From 83e52853ec33d21232b2649a8d0f1e6c40393a12 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 10:34:53 +0200 Subject: [PATCH 01/47] chore(deps): pin npm version to remove lockfile drift Contributors on npm 10 (Node 22's bundled default) vs npm 11 reconcile the app-builder-lib.minimatch override differently, rewriting package-lock.json on npm install. Pin npm 11.7.0 via the standard packageManager field (Corepack) and complete the existing volta block so volta users pick it up automatically. Ref: discussion #8869 --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 501c7d51f7..62db9fe4f6 100644 --- a/package.json +++ b/package.json @@ -361,7 +361,9 @@ "owner": "johannesjo" } ], + "packageManager": "npm@11.7.0", "volta": { - "node": "22.18.0" + "node": "22.18.0", + "npm": "11.7.0" } } From 6e20c09e255fef758c11920673c07c34ee9aacd5 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 10:46:20 +0200 Subject: [PATCH 02/47] chore: stop tracking .claude/skills symlink (runtime-managed) The Claude Code runtime scaffolds .claude/ each session and materializes .claude/skills as a real directory, clobbering the committed symlink (-> ../.agents/skills). Git then reported a perpetual 'D .claude/skills' in every worktree/session. Skill sources remain tracked under .agents/skills/; let the runtime own .claude/skills and ignore it via the existing /.claude/* rule. --- .claude/skills | 1 - .gitignore | 1 - 2 files changed, 2 deletions(-) delete mode 120000 .claude/skills diff --git a/.claude/skills b/.claude/skills deleted file mode 120000 index 2b7a412b8f..0000000000 --- a/.claude/skills +++ /dev/null @@ -1 +0,0 @@ -../.agents/skills \ No newline at end of file diff --git a/.gitignore b/.gitignore index 23fa10dae8..6aba5beeb3 100644 --- a/.gitignore +++ b/.gitignore @@ -98,7 +98,6 @@ electron-builder.win-store.yaml /src/environments/environment.js /src/environments/environment.js.map /.claude/* -!/.claude/skills .aider* From 79f91e36fe3a5e69e13dc44ad446840397850e70 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 13:25:30 +0200 Subject: [PATCH 03/47] fix(sync): remediate sync-correctness review findings Address findings from a full sync-system review (blockers + high/medium): - USE_REMOTE ("Use Server Data") is now a true download-first rebuild: fetches the complete server history (incl. own/already-known ops via a raw-download mode) and validates it before any local mutation; aborts untouched on download failure, empty remote, or newer-schema ops. - Widen the incoming full-state conflict gate to treat all user work as meaningful (not just TASK/PROJECT/TAG/NOTE CUD); fix the piggyback race by judging against a pre-upload pending snapshot. - Stop advancing the server cursor past ops blocked by schema/migration failures so they are retried after an app update; block any op from a newer schema version (no forward-compat band). - Retry of failed remote ops runs archive side effects only, never re-dispatching reducers whose effects already committed (fixes additive double-apply of time-tracking/counter deltas across hydration+retry). - Use local monotonic seq (not lexical UUIDv7) to decide which ops are covered by an uploaded snapshot, preventing dropped unsynced ops under clock rollback. - Server: include entityIds in duplicate-operation equality. - Capture buffer no longer silently drops accepted actions at 100 (warns to reload); drop only past a 5000 pathological cap. Adds regression coverage for each fix. sync-core 209/209, super-sync-server 817/817, and all touched Angular specs pass. --- .gitignore | 2 +- .../super-sync-server/src/sync/conflict.ts | 4 + .../super-sync-server/src/sync/sync.types.ts | 2 + .../super-sync-server/tests/conflict.spec.ts | 52 ++++ .../duplicate-operation-precheck.spec.ts | 2 + .../tests/sync-compressed-body.routes.spec.ts | 1 + packages/sync-core/src/apply.types.ts | 15 +- packages/sync-core/src/remote-apply.ts | 4 +- packages/sync-core/src/replay-coordinator.ts | 11 +- packages/sync-core/src/upload-planning.ts | 25 +- .../tests/replay-coordinator.spec.ts | 82 ++++++ .../sync-core/tests/upload-planning.spec.ts | 23 +- .../apply/operation-applier.service.spec.ts | 43 ++- .../op-log/apply/operation-applier.service.ts | 5 +- .../operation-capture.meta-reducer.spec.ts | 82 ++++++ .../capture/operation-capture.meta-reducer.ts | 51 +++- src/app/op-log/core/types/apply.types.ts | 14 + ...ion-log-hydrator.retry.integration.spec.ts | 6 + .../operation-log-hydrator.service.spec.ts | 64 +++++ .../operation-log-hydrator.service.ts | 39 ++- .../sync/operation-log-download.service.ts | 23 +- .../sync/operation-log-sync.service.spec.ts | 264 ++++++++++++++++-- .../op-log/sync/operation-log-sync.service.ts | 212 +++++++++----- ...g-upload-piggyback-seq.integration.spec.ts | 2 + .../sync/operation-log-upload.service.spec.ts | 40 ++- .../sync/operation-log-upload.service.ts | 14 +- .../remote-ops-processing.service.spec.ts | 111 ++++---- .../sync/remote-ops-processing.service.ts | 142 +++++----- .../sync-import-conflict-gate.service.spec.ts | 127 +++++++++ .../sync/sync-import-conflict-gate.service.ts | 50 +++- .../migration-handling.integration.spec.ts | 36 +-- 31 files changed, 1232 insertions(+), 316 deletions(-) diff --git a/.gitignore b/.gitignore index 6aba5beeb3..50c7f99c48 100644 --- a/.gitignore +++ b/.gitignore @@ -45,7 +45,7 @@ android/.idea # misc \\backups /backups -/.angular/cache +/.angular /.sass-cache /connect.lock /coverage diff --git a/packages/super-sync-server/src/sync/conflict.ts b/packages/super-sync-server/src/sync/conflict.ts index 668d382ef8..1aad6b9d99 100644 --- a/packages/super-sync-server/src/sync/conflict.ts +++ b/packages/super-sync-server/src/sync/conflict.ts @@ -246,6 +246,10 @@ export const isSameDuplicateOperation = ( existingOp.opType === op.opType && existingOp.entityType === op.entityType && existingOp.entityId === (op.entityId ?? null) && + // Compare against the same normalization the row was persisted with + // (getStoredEntityIds: single-entity sets collapse to []), so a genuine + // retry matches while a batch op differing only in entityIds does not. + areJsonValuesEqual(existingOp.entityIds, getStoredEntityIds(op)) && payloadsMatch && areJsonValuesEqual(existingOp.vectorClock, storedVectorClock) && existingOp.schemaVersion === op.schemaVersion && diff --git a/packages/super-sync-server/src/sync/sync.types.ts b/packages/super-sync-server/src/sync/sync.types.ts index 0fdc3d5e32..50950b75b6 100644 --- a/packages/super-sync-server/src/sync/sync.types.ts +++ b/packages/super-sync-server/src/sync/sync.types.ts @@ -184,6 +184,7 @@ export interface DuplicateOperationCandidate { opType: string; entityType: string; entityId: string | null; + entityIds: string[]; payload: unknown; vectorClock: unknown; schemaVersion: number; @@ -207,6 +208,7 @@ export const DUPLICATE_OP_SELECT = { opType: true, entityType: true, entityId: true, + entityIds: true, payload: true, vectorClock: true, schemaVersion: true, diff --git a/packages/super-sync-server/tests/conflict.spec.ts b/packages/super-sync-server/tests/conflict.spec.ts index 87f104103b..d0824ddbc0 100644 --- a/packages/super-sync-server/tests/conflict.spec.ts +++ b/packages/super-sync-server/tests/conflict.spec.ts @@ -36,6 +36,7 @@ const duplicateCandidate = ( opType: 'CRT', entityType: 'TASK', entityId: 'task-1', + entityIds: [], payload: { title: 'A' }, vectorClock: { 'client-a': 1 }, schemaVersion: 1, @@ -66,6 +67,57 @@ describe('conflict helpers', () => { ); }); + it('accepts batch retries with identical entityIds', () => { + const incoming = op({ entityIds: ['task-1', 'task-2'] }); + const existing = duplicateCandidate({ entityIds: ['task-1', 'task-2'] }); + + expect(isSameDuplicateOperation(existing, 1, incoming, 60_000)).toBe(true); + }); + + it('rejects duplicate ids when entityIds differ', () => { + const incoming = op({ entityIds: ['task-1', 'task-3'] }); + const existing = duplicateCandidate({ entityIds: ['task-1', 'task-2'] }); + + expect(isSameDuplicateOperation(existing, 1, incoming, 60_000)).toBe(false); + }); + + it('rejects duplicate ids when only one side has entityIds', () => { + const incomingWithBatch = op({ entityIds: ['task-1', 'task-2'] }); + expect( + isSameDuplicateOperation(duplicateCandidate(), 1, incomingWithBatch, 60_000), + ).toBe(false); + + const existingWithBatch = duplicateCandidate({ entityIds: ['task-1', 'task-2'] }); + expect(isSameDuplicateOperation(existingWithBatch, 1, op(), 60_000)).toBe(false); + }); + + it('accepts single-entity retries whose entityIds collapse to the scalar entityId', () => { + // getStoredEntityIds persists [] when entityIds is exactly [entityId], so a + // retry that re-sends that redundant array must still match the stored row. + const incoming = op({ entityIds: ['task-1'] }); + + expect(isSameDuplicateOperation(duplicateCandidate(), 1, incoming, 60_000)).toBe( + true, + ); + }); + + it('rejects encrypted retries when entityIds differ', () => { + // With both sides encrypted the payload comparison is skipped, so entityIds + // must independently block a batch-op id collision. + const incoming = op({ + payload: 'BASE64-CIPHERTEXT-A', + isPayloadEncrypted: true, + entityIds: ['task-1', 'task-3'], + }); + const existing = duplicateCandidate({ + payload: 'BASE64-CIPHERTEXT-B', + isPayloadEncrypted: true, + entityIds: ['task-1', 'task-2'], + }); + + expect(isSameDuplicateOperation(existing, 1, incoming, 60_000)).toBe(false); + }); + it('accepts encrypted retries whose ciphertext differs from the stored payload', () => { // Regression: when encryption is on, encrypt() generates a fresh random IV // per call, so a retry of the same logical op produces different ciphertext. diff --git a/packages/super-sync-server/tests/duplicate-operation-precheck.spec.ts b/packages/super-sync-server/tests/duplicate-operation-precheck.spec.ts index 42ad121a8a..e82171f538 100644 --- a/packages/super-sync-server/tests/duplicate-operation-precheck.spec.ts +++ b/packages/super-sync-server/tests/duplicate-operation-precheck.spec.ts @@ -441,6 +441,7 @@ describe('Duplicate Operation Pre-check', () => { opType: 'UPD', entityType: 'TASK', entityId: 'task-race', + entityIds: [], payload: { foo: 'bar' }, vectorClock: { 'client-1': 1 }, schemaVersion: 1, @@ -516,6 +517,7 @@ describe('Duplicate Operation Pre-check', () => { opType: 'UPD', entityType: 'TASK', entityId: 'task-other-user', + entityIds: [], payload: { foo: 'bar' }, vectorClock: { 'client-1': 1 }, schemaVersion: 1, diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index 3796f83fc9..fd60cfd4a2 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -93,6 +93,7 @@ const createStoredDuplicateOp = (op: ReturnType) => ({ opType: op.opType, entityType: op.entityType, entityId: op.entityId, + entityIds: [], payload: op.payload, vectorClock: op.vectorClock, schemaVersion: op.schemaVersion, diff --git a/packages/sync-core/src/apply.types.ts b/packages/sync-core/src/apply.types.ts index 8efbbe4024..7c79608c03 100644 --- a/packages/sync-core/src/apply.types.ts +++ b/packages/sync-core/src/apply.types.ts @@ -12,7 +12,11 @@ export interface ApplyOperationsResult = Op /** * If an error occurred, this contains the failed operation and the error. - * Operations after this one in the batch were NOT applied. + * The failed op and the operations after it in the batch did NOT complete + * their archive side effects — but their reducer effects DID commit: the + * bulk dispatch is all-or-nothing and runs before archive handling, so + * archive side effects are the only per-op failure point. Retry paths must + * therefore use `skipReducerDispatch` to avoid double-applying reducers. */ failedOp?: { op: TOperation; @@ -27,4 +31,13 @@ export interface ApplyOperationsOptions { * local operations during hydration. */ isLocalHydration?: boolean; + + /** + * When true, skip the bulk reducer dispatch and run only the post-dispatch + * archive side effects. Used to retry operations whose reducer effects + * already committed in an earlier batch (see `failedOp`): re-dispatching on + * retry would double-apply additive reducers such as time-tracking deltas + * and counter increments. + */ + skipReducerDispatch?: boolean; } diff --git a/packages/sync-core/src/remote-apply.ts b/packages/sync-core/src/remote-apply.ts index a47fe72c97..2546770de6 100644 --- a/packages/sync-core/src/remote-apply.ts +++ b/packages/sync-core/src/remote-apply.ts @@ -74,7 +74,9 @@ const emptyRemoteApplyResult = < * 3. mark successfully applied seqs; * 4. merge applied remote vector clocks; * 5. retain only the newest applied full-state ops when configured; - * 6. mark the failed op and remaining unapplied ops as failed on partial error. + * 6. mark the failed op and remaining ops as failed on partial error — their + * reducer effects committed with the bulk dispatch; only their archive side + * effects are outstanding (see ApplyOperationsResult.failedOp). */ export const applyRemoteOperations = async < TOperation extends Operation = Operation, diff --git a/packages/sync-core/src/replay-coordinator.ts b/packages/sync-core/src/replay-coordinator.ts index 6cc02b825a..567c5374af 100644 --- a/packages/sync-core/src/replay-coordinator.ts +++ b/packages/sync-core/src/replay-coordinator.ts @@ -71,7 +71,9 @@ export const yieldToEventLoop = (): Promise => * package. This coordinator only owns the generic ordering: * * 1. open the remote-apply window; - * 2. dispatch the host's bulk replay action; + * 2. dispatch the host's bulk replay action (skipped when the caller marks + * reducers as already committed via `skipReducerDispatch` — retry paths + * must not double-apply additive reducers); * 3. yield once so host state reducers finish before side effects; * 4. run remote archive side effects after dispatch, when configured; * 5. start post-sync cooldown before closing the remote-apply window; @@ -103,6 +105,7 @@ export const replayOperationBatch = async < } const isLocalHydration = applyOptions.isLocalHydration ?? false; + const skipReducerDispatch = applyOptions.skipReducerDispatch ?? false; if (!isLocalHydration && archiveSideEffects !== undefined && !operationToAction) { throw new Error( @@ -112,8 +115,10 @@ export const replayOperationBatch = async < remoteApplyWindow.startApplyingRemoteOps(); try { - dispatcher.dispatch(createBulkApplyAction(ops)); - await waitForEventLoop(); + if (!skipReducerDispatch) { + dispatcher.dispatch(createBulkApplyAction(ops)); + await waitForEventLoop(); + } if (!isLocalHydration && archiveSideEffects !== undefined && operationToAction) { const archiveResult = await processArchiveSideEffects({ diff --git a/packages/sync-core/src/upload-planning.ts b/packages/sync-core/src/upload-planning.ts index 5be41ce62b..74663d52e3 100644 --- a/packages/sync-core/src/upload-planning.ts +++ b/packages/sync-core/src/upload-planning.ts @@ -11,24 +11,33 @@ export interface PlanRegularOpsAfterFullStateUploadOptions< TEntry extends OperationLogEntry> = OperationLogEntry, > { regularOps: TEntry[]; - lastUploadedFullStateOpId?: string; + lastUploadedFullStateOpSeq?: number; } /** * Splits regular operations around an uploaded full-state snapshot. * - * UUIDv7 operation IDs are time-ordered in Super Productivity. Core only - * depends on lexical ID ordering supplied by the host. Ops before the full-state - * operation are already represented in the snapshot and can be marked synced; - * later ops still need the normal upload path. + * Classification uses the local op-log `seq` (monotonic append order on this + * client), NOT the UUIDv7 op id: lexical id order follows the wall clock, which + * can roll back (e.g. NTP correction, restart), so a post-snapshot op could get + * a smaller id, be treated as "included in snapshot", and silently never reach + * the server. `seq` is exactly creation order, which is what "captured in the + * frozen snapshot payload" means. Ops appended before the full-state op are + * already represented in the snapshot and can be marked synced; later ops still + * need the normal upload path. + * + * When no seq is available (no full-state op uploaded, or its local entry is + * unknown, e.g. the id came from a remote source), everything stays on the + * upload path — uploading an op that is also in the snapshot is safe (server + * dedups by op id), whereas skipping one that is not would lose data. */ export const planRegularOpsAfterFullStateUpload = < TEntry extends OperationLogEntry> = OperationLogEntry, >({ regularOps, - lastUploadedFullStateOpId, + lastUploadedFullStateOpSeq, }: PlanRegularOpsAfterFullStateUploadOptions): RegularOpsAfterFullStateUploadPlan => { - if (!lastUploadedFullStateOpId) { + if (lastUploadedFullStateOpSeq === undefined) { return { opsIncludedInSnapshot: [], opsAfterSnapshot: regularOps, @@ -39,7 +48,7 @@ export const planRegularOpsAfterFullStateUpload = < const opsAfterSnapshot: TEntry[] = []; for (const entry of regularOps) { - if (entry.op.id < lastUploadedFullStateOpId) { + if (entry.seq < lastUploadedFullStateOpSeq) { opsIncludedInSnapshot.push(entry); } else { opsAfterSnapshot.push(entry); diff --git a/packages/sync-core/tests/replay-coordinator.spec.ts b/packages/sync-core/tests/replay-coordinator.spec.ts index faa8cd087e..a94dc95621 100644 --- a/packages/sync-core/tests/replay-coordinator.spec.ts +++ b/packages/sync-core/tests/replay-coordinator.spec.ts @@ -264,6 +264,88 @@ describe('replayOperationBatch', () => { ]); }); + it('runs only archive side effects when skipReducerDispatch is set (retry path)', async () => { + const callOrder: string[] = []; + const ops = [createOperation('op-1'), createOperation('op-2')]; + const dispatcher: ActionDispatchPort = { + dispatch: vi.fn(() => { + callOrder.push('dispatchBulk'); + }), + }; + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async (action) => { + callOrder.push(`archive:${action.opId}`); + }), + }; + + const result = await replayOperationBatch({ + ops, + applyOptions: { skipReducerDispatch: true }, + dispatcher, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + archiveSideEffects, + operationToAction: (op) => ({ type: '[Test] Action', opId: op.id }), + yieldToEventLoop: vi.fn(async () => { + callOrder.push('yield'); + }), + }); + + expect(result).toEqual({ appliedOps: ops }); + expect(dispatcher.dispatch).not.toHaveBeenCalled(); + expect(callOrder).toEqual([ + 'startApplyingRemoteOps', + 'archive:op-1', + 'archive:op-2', + 'yield', + 'startPostSyncCooldown', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + + it('reports archive failures without re-dispatching when skipReducerDispatch is set', async () => { + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const archiveError = new Error('archive failed again'); + const dispatcher: ActionDispatchPort = { dispatch: vi.fn() }; + const archiveSideEffects: ArchiveSideEffectPort = { + handleOperation: vi.fn(async (action) => { + if (action.opId === 'op-2') { + throw archiveError; + } + }), + }; + + const result = await replayOperationBatch({ + ops: [op1, op2], + applyOptions: { skipReducerDispatch: true }, + dispatcher, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow([]), + deferredLocalActions: createDeferredLocalActions([]), + archiveSideEffects, + operationToAction: (op) => ({ type: '[Test] Action', opId: op.id }), + yieldToEventLoop: vi.fn(async () => undefined), + }); + + expect(dispatcher.dispatch).not.toHaveBeenCalled(); + expect(result).toEqual({ + appliedOps: [op1], + failedOp: { + op: op2, + error: archiveError, + }, + }); + }); + it('fails fast when archive side effects are configured without operation conversion', async () => { const callOrder: string[] = []; const archiveSideEffects: ArchiveSideEffectPort = { diff --git a/packages/sync-core/tests/upload-planning.spec.ts b/packages/sync-core/tests/upload-planning.spec.ts index a9855f3669..9201a59283 100644 --- a/packages/sync-core/tests/upload-planning.spec.ts +++ b/packages/sync-core/tests/upload-planning.spec.ts @@ -32,7 +32,7 @@ describe('planRegularOpsAfterFullStateUpload', () => { expect( planRegularOpsAfterFullStateUpload({ regularOps: entries, - lastUploadedFullStateOpId: undefined, + lastUploadedFullStateOpSeq: undefined, }), ).toEqual({ opsIncludedInSnapshot: [], @@ -40,7 +40,7 @@ describe('planRegularOpsAfterFullStateUpload', () => { }); }); - it('splits regular ops before and after the uploaded full-state op id', () => { + it('splits regular ops before and after the uploaded full-state op seq', () => { const before = createEntry(1, '001'); const same = createEntry(2, '010'); const after = createEntry(3, '011'); @@ -48,13 +48,30 @@ describe('planRegularOpsAfterFullStateUpload', () => { expect( planRegularOpsAfterFullStateUpload({ regularOps: [before, same, after], - lastUploadedFullStateOpId: '010', + lastUploadedFullStateOpSeq: 2, }), ).toEqual({ opsIncludedInSnapshot: [before], opsAfterSnapshot: [same, after], }); }); + + it('uploads an op created after the snapshot even when its id sorts before the full-state op id (clock rollback)', () => { + // Wall-clock rollback: the post-snapshot op got a lexically SMALLER UUIDv7 + // id than the full-state op. Local seq order must win — the op is NOT in + // the frozen snapshot payload and must be uploaded, not marked synced. + const afterSnapshotWithSmallerId = createEntry(5, '001'); + + expect( + planRegularOpsAfterFullStateUpload({ + regularOps: [afterSnapshotWithSmallerId], + lastUploadedFullStateOpSeq: 4, + }), + ).toEqual({ + opsIncludedInSnapshot: [], + opsAfterSnapshot: [afterSnapshotWithSmallerId], + }); + }); }); describe('planUploadLastServerSeqUpdate', () => { diff --git a/src/app/op-log/apply/operation-applier.service.spec.ts b/src/app/op-log/apply/operation-applier.service.spec.ts index b0ea2e7d7e..c923934728 100644 --- a/src/app/op-log/apply/operation-applier.service.spec.ts +++ b/src/app/op-log/apply/operation-applier.service.spec.ts @@ -554,7 +554,10 @@ describe('OperationApplierService', () => { const result = await service.applyOperations(ops); - // Bulk dispatch succeeded (all ops applied to NgRx state) + // Bulk dispatch succeeded: the reducer effects of ALL FIVE ops are + // committed to NgRx state, including the "failed" ones — failedOp only + // means their archive side effects are outstanding. That is why retry + // paths must use skipReducerDispatch (see test below). expect(mockStore.dispatch).toHaveBeenCalledTimes(1); // But archive handling failed on op-3 @@ -566,6 +569,44 @@ describe('OperationApplierService', () => { // Archive handler was called 3 times (op-1, op-2, op-3) expect(mockArchiveOperationHandler.handleOperation).toHaveBeenCalledTimes(3); }); + + it('should not re-run reducers when retrying the failed slice with skipReducerDispatch', async () => { + const ops = [ + createMockOperation('op-1', 'TASK', OpType.Update, { title: 'First' }), + createMockOperation('op-2', 'TASK', OpType.Update, { title: 'Second' }), + createMockOperation('op-3', 'TASK', OpType.Update, { title: 'Third' }), + createMockOperation('op-4', 'TASK', OpType.Update, { title: 'Fourth' }), + createMockOperation('op-5', 'TASK', OpType.Update, { title: 'Fifth' }), + ]; + + let callCount = 0; + mockArchiveOperationHandler.handleOperation.and.callFake(() => { + callCount++; + return callCount === 3 + ? Promise.reject(new Error('Archive write failed on op-3')) + : Promise.resolve(); + }); + + const firstPass = await service.applyOperations(ops); + expect(firstPass.failedOp!.op.id).toBe('op-3'); + expect(mockStore.dispatch).toHaveBeenCalledTimes(1); + + mockStore.dispatch.calls.reset(); + mockArchiveOperationHandler.handleOperation.calls.reset(); + mockArchiveOperationHandler.handleOperation.and.returnValue(Promise.resolve()); + + // Retry the failed slice the way retryFailedRemoteOps does: archive only. + const retry = await service.applyOperations(ops.slice(2), { + skipReducerDispatch: true, + }); + + // No reducer re-dispatch — additive reducers (e.g. syncTimeSpent, + // increaseSimpleCounterCounterToday) cannot double-apply on retry. + expect(mockStore.dispatch).not.toHaveBeenCalled(); + expect(mockArchiveOperationHandler.handleOperation).toHaveBeenCalledTimes(3); + expect(retry.appliedOps).toEqual(ops.slice(2)); + expect(retry.failedOp).toBeUndefined(); + }); }); describe('effects isolation (key architectural benefit)', () => { diff --git a/src/app/op-log/apply/operation-applier.service.ts b/src/app/op-log/apply/operation-applier.service.ts index 0d7df161c4..4f0dc8b4c1 100644 --- a/src/app/op-log/apply/operation-applier.service.ts +++ b/src/app/op-log/apply/operation-applier.service.ts @@ -106,7 +106,10 @@ export class OperationApplierService implements OperationApplyPort { const result = await replayOperationBatch({ ops, - applyOptions: { isLocalHydration }, + applyOptions: { + isLocalHydration, + skipReducerDispatch: options.skipReducerDispatch, + }, dispatcher: this.store, createBulkApplyAction: (operations) => bulkApplyOperations({ operations, localClientId }), diff --git a/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts b/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts index 0f0adab79d..21d9b4d1ea 100644 --- a/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts +++ b/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts @@ -7,6 +7,8 @@ import { bufferDeferredAction, getDeferredActions, clearDeferredActions, + DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD, + DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP, } from './operation-capture.meta-reducer'; import { OperationCaptureService } from './operation-capture.service'; import { Action } from '@ngrx/store'; @@ -269,6 +271,86 @@ describe('operationCaptureMetaReducer', () => { expect(getDeferredActions()).toEqual([]); }); }); + + describe('buffer limits', () => { + // devError shows a native alert + confirm (and throws if confirm returns + // true), so force confirm to return false. src/test.ts installs a + // PERMANENT global confirm spy (jasmine.createSpy, never auto-restored), + // so reset its accumulated calls per test and restore the global + // returnValue(true) default afterwards. + const spyNativeDialogs = (): jasmine.Spy => { + if (!jasmine.isSpy(window.alert)) { + spyOn(window, 'alert'); + } + const confirmSpy = jasmine.isSpy(window.confirm) + ? (window.confirm as jasmine.Spy) + : spyOn(window, 'confirm'); + confirmSpy.calls.reset(); + confirmSpy.and.returnValue(false); + return confirmSpy; + }; + + afterEach(() => { + if (jasmine.isSpy(window.confirm)) { + (window.confirm as jasmine.Spy).and.returnValue(true); + } + }); + + const createManyActions = (count: number): PersistentAction[] => + Array.from({ length: count }, (_, i) => + createMockAction({ type: `[Test] Action ${i}` }), + ); + + it('should preserve ALL actions in order past the reload-warning threshold', () => { + spyNativeDialogs(); + const actions = createManyActions(DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD + 50); + + actions.forEach((a) => bufferDeferredAction(a)); + + // Nothing may be dropped: each buffered action's state change was + // already accepted into NgRx — dropping = permanent unsyncable divergence. + expect(getDeferredActions()).toEqual(actions); + }); + + it('should fire devError at the reload-warning threshold without dropping', () => { + const confirmSpy = spyNativeDialogs(); + const actions = createManyActions(DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD); + const reloadWarningCalls = (): unknown[][] => + confirmSpy.calls + .allArgs() + .filter((args) => /consider reloading/.test(String(args[0]))); + + actions.slice(0, -1).forEach((a) => bufferDeferredAction(a)); + expect(reloadWarningCalls().length).toBe(0); + + bufferDeferredAction(actions[actions.length - 1]); + expect(reloadWarningCalls().length).toBe(1); + + expect(getDeferredActions()).toEqual(actions); + }); + + it('should drop the oldest action only past the pathological hard cap', () => { + const confirmSpy = spyNativeDialogs(); + const actions = createManyActions(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP); + const dropErrorCalls = (): unknown[][] => + confirmSpy.calls + .allArgs() + .filter((args) => /pathological cap/.test(String(args[0]))); + + actions.forEach((a) => bufferDeferredAction(a)); + expect(dropErrorCalls().length).toBe(0); + + const extraAction = createMockAction({ type: '[Test] Extra Action' }); + bufferDeferredAction(extraAction); + expect(dropErrorCalls().length).toBe(1); + + const buffered = getDeferredActions(); + expect(buffered.length).toBe(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP); + // Oldest action was dropped, remaining order intact + expect(buffered[0]).toBe(actions[1]); + expect(buffered[buffered.length - 1]).toBe(extraAction); + }); + }); }); describe('sync buffering (user interaction during sync)', () => { diff --git a/src/app/op-log/capture/operation-capture.meta-reducer.ts b/src/app/op-log/capture/operation-capture.meta-reducer.ts index 54cfaf89c0..31a49d714c 100644 --- a/src/app/op-log/capture/operation-capture.meta-reducer.ts +++ b/src/app/op-log/capture/operation-capture.meta-reducer.ts @@ -133,31 +133,48 @@ export const getIsApplyingRemoteOps = (): boolean => { }; /** - * Maximum number of deferred actions before warning. + * Soft-warning threshold for the deferred actions buffer. * If exceeded, sync may be stuck or taking too long. */ -const MAX_DEFERRED_ACTIONS_WARNING = 10; +const DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD = 10; /** - * Hard limit for deferred actions buffer. - * If reached, oldest actions are dropped to prevent unbounded memory growth. + * Reload-warning threshold: a devError advises the user to reload, but + * NOTHING is dropped. Every buffered action's state change was already + * accepted into NgRx; dropping it would mean no operation ever represents + * it — a permanent, unsyncable local divergence. + * Exported for tests. */ -const MAX_DEFERRED_ACTIONS_HARD_LIMIT = 100; +export const DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD = 100; + +/** + * Pathological hard cap for the deferred actions buffer — purely a memory + * backstop against a runaway dispatch loop. ~100 buffered actions are + * plausibly reachable by a real user during a stuck/very long remote-apply + * window, so we must never drop there (see reload-warning threshold above, + * which tells the user to reload long before this cap can be hit through + * real interaction). Only past this cap is drop-oldest the lesser evil vs + * unbounded memory growth. + * Exported for tests. + */ +export const DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP = 5000; /** * Buffers an action for processing after sync completes. * Called by the meta-reducer when a persistent action arrives during sync. */ export const bufferDeferredAction = (action: PersistentAction): void => { - // Hard limit: drop oldest action if buffer is full (sync stuck scenario). - // NOTE: The shifted action remains in deferredActionSet (WeakSet has no delete-by-value). - // The effect filters it via isDeferredAction(), and getDeferredActions() won't return it, - // so it is silently lost. This is acceptable: the hard limit is itself an error condition - // (sync stuck), and dropping the oldest action is the lesser evil vs unbounded growth. - if (deferredActions.length >= MAX_DEFERRED_ACTIONS_HARD_LIMIT) { + // Pathological cap only: dropping an accepted persistent action means its + // state mutation survives in NgRx while no operation will ever represent + // it → permanent, unsyncable divergence. So this drop exists solely as a + // memory backstop against a runaway dispatch loop, never as flow control. + // NOTE: The shifted action remains in deferredActionSet (WeakSet has no + // delete-by-value). The effect filters it via isDeferredAction(), and + // getDeferredActions() won't return it, so it is silently lost. + if (deferredActions.length >= DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP) { devError( - `[operationCaptureMetaReducer] Deferred actions buffer exceeded ${MAX_DEFERRED_ACTIONS_HARD_LIMIT} items. ` + - `Dropping oldest action. Sync may be stuck - consider reloading the app.`, + `[operationCaptureMetaReducer] Deferred actions buffer exceeded pathological cap of ${DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP} items. ` + + `Dropping oldest action - this local change will NOT sync. Likely a runaway dispatch loop; reload the app.`, ); deferredActions.shift(); } @@ -165,8 +182,12 @@ export const bufferDeferredAction = (action: PersistentAction): void => { deferredActions.push(action); deferredActionSet.add(action); - // Soft warning at 10 items - if (deferredActions.length > MAX_DEFERRED_ACTIONS_WARNING) { + if (deferredActions.length >= DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD) { + devError( + `[operationCaptureMetaReducer] Deferred actions buffer has ${deferredActions.length} items. ` + + `Sync may be stuck - consider reloading the app. Nothing is dropped; actions remain buffered.`, + ); + } else if (deferredActions.length > DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD) { devError( `[operationCaptureMetaReducer] Deferred actions buffer has ${deferredActions.length} items - sync may be stuck or taking too long`, ); diff --git a/src/app/op-log/core/types/apply.types.ts b/src/app/op-log/core/types/apply.types.ts index 3e80976225..7c2e72f24b 100644 --- a/src/app/op-log/core/types/apply.types.ts +++ b/src/app/op-log/core/types/apply.types.ts @@ -7,6 +7,11 @@ import type { Operation } from '../operation.types'; export interface ApplyOperationsResult { /** Operations that were successfully applied. */ appliedOps: Operation[]; + /** + * First op whose archive side effect threw. Its reducer effect (and that of + * every op after it) DID commit — the bulk dispatch is all-or-nothing and + * precedes archive handling — so retry paths must pass `skipReducerDispatch`. + */ failedOp?: { op: Operation; error: Error; @@ -27,4 +32,13 @@ export interface ApplyOperationsOptions { * their vector clocks. */ skipDeferredLocalActions?: boolean; + + /** + * When true, skip the bulk reducer dispatch and run only the archive side + * effects. Used to retry ops marked `failed`, whose reducer effects already + * committed (see `ApplyOperationsResult.failedOp`): re-dispatching would + * double-apply additive reducers such as syncTimeSpent or + * increaseSimpleCounterCounterToday. + */ + skipReducerDispatch?: boolean; } diff --git a/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts index bec40e8500..aea87db639 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts @@ -132,6 +132,12 @@ describe('OperationLogHydratorService retryFailedRemoteOps (integration, real st expect(applier.applyOperations).toHaveBeenCalledTimes(1); const passed = applier.applyOperations.calls.argsFor(0)[0] as Operation[]; expect(passed.length).toBe(3); + // Archive side effects only: the failed ops' reducers committed in the + // batch that marked them failed, so a reducer re-dispatch would + // double-apply additive reducers. + expect(applier.applyOperations.calls.argsFor(0)[1]).toEqual({ + skipReducerDispatch: true, + }); }); it('on partial failure marks the failed op and every op after it as still-failing', async () => { diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts index 3ba60e317a..6c77d85278 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts @@ -410,6 +410,50 @@ describe('OperationLogHydratorService', () => { expect(mockHydrationStateService.endApplyingRemoteOps).toHaveBeenCalled(); }); + it('should apply a failed tail op exactly once across hydration replay + retry (one boot)', async () => { + // Regression: a remote op marked 'failed' (archive side effect threw + // after its reducer committed) with seq > lastAppliedOpSeq used to get + // its reducer applied TWICE per boot — once by the status-blind tail + // replay and once more by retryFailedRemoteOps re-dispatching. The + // retry must complete it with archive side effects only. + const snapshot = createMockSnapshot({ lastAppliedOpSeq: 5 }); + const failedOp = createMockOperation('op-failed'); + const failedTailEntry: OperationLogEntry = { + seq: 6, + op: failedOp, + appliedAt: Date.now(), + source: 'remote', + applicationStatus: 'failed', + retryCount: 1, + }; + mockOpLogStore.loadStateCache.and.returnValue(Promise.resolve(snapshot)); + mockOpLogStore.getOpsAfterSeq.and.returnValue(Promise.resolve([failedTailEntry])); + mockOpLogStore.getFailedRemoteOps.and.returnValue( + Promise.resolve([failedTailEntry]), + ); + mockOperationApplierService.applyOperations.and.callFake((ops: Operation[]) => + Promise.resolve({ appliedOps: ops }), + ); + + await service.hydrateStore(); + + // Reducer application happens exactly once: the tail replay bulk dispatch. + const bulkDispatches = mockStore.dispatch.calls + .allArgs() + .map((args) => args[0] as unknown as { type: string; operations?: Operation[] }) + .filter((action) => action.type === bulkApplyHydrationOperations.type); + expect(bulkDispatches.length).toBe(1); + expect(bulkDispatches[0].operations!.map((o) => o.id)).toEqual(['op-failed']); + + // The retry completes the op WITHOUT re-dispatching its reducer. + expect(mockOperationApplierService.applyOperations).toHaveBeenCalledTimes(1); + const [retriedOps, retryOptions] = + mockOperationApplierService.applyOperations.calls.argsFor(0); + expect(retriedOps.map((o: Operation) => o.id)).toEqual(['op-failed']); + expect(retryOptions).toEqual({ skipReducerDispatch: true }); + expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([6]); + }); + it('should request ops after snapshot sequence', async () => { const snapshot = createMockSnapshot({ lastAppliedOpSeq: 42 }); mockOpLogStore.loadStateCache.and.returnValue(Promise.resolve(snapshot)); @@ -1364,6 +1408,26 @@ describe('OperationLogHydratorService', () => { expect(passedOps.map((o: Operation) => o.id)).toEqual(['op-a', 'op-b', 'op-c']); expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([40, 41, 42]); expect(mockOpLogStore.markFailed).not.toHaveBeenCalled(); + // Applied-op clocks are merged on completion (parity with the primary + // remote-apply path, which only merges the clocks of ops it marked applied). + expect(mockOpLogStore.mergeRemoteOpClocks).toHaveBeenCalledWith(passedOps); + }); + + it('should retry archive side effects only — never re-dispatch reducers', async () => { + // Failed ops had their reducers committed by the bulk dispatch of the + // batch that marked them failed; re-dispatching on retry double-applies + // additive reducers (syncTimeSpent, increaseSimpleCounterCounterToday). + mockOpLogStore.getFailedRemoteOps.and.returnValue( + Promise.resolve([failedEntry(40, 'op-a')]), + ); + mockOperationApplierService.applyOperations.and.callFake((ops: Operation[]) => + Promise.resolve({ appliedOps: ops }), + ); + + await service.retryFailedRemoteOps(); + + const options = mockOperationApplierService.applyOperations.calls.argsFor(0)[1]; + expect(options).toEqual({ skipReducerDispatch: true }); }); it('should apply failed ops in ascending seq order regardless of store order', async () => { diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.ts b/src/app/op-log/persistence/operation-log-hydrator.service.ts index c2b092fac4..e35345841e 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.ts @@ -208,6 +208,21 @@ export class OperationLogHydratorService { ); // 4. Replay tail operations (A.7.13: with operation migration) + // + // Replay is deliberately status-blind (getOpsAfterSeq has no status + // filter) — every entry's reducer effect belongs in state exactly once: + // - applied ops: their effect is state history by definition. + // - failed ops (remote, archive side effect threw): their reducers DID + // commit before the failure (bulk dispatch precedes archive handling), + // so replay restores that effect; retryFailedRemoteOps() below then + // re-runs ONLY the outstanding archive side effects. + // - rejected ops: every rejection path appends its compensation AFTER + // them in seq order, so replay converges to post-resolution runtime + // state — server-rejected local ops are followed by merged ops + // (SupersededOperationResolver) or keep their effect (permanent + // rejections never revert state), and LWW-losing remote ops are + // followed by the local-win op that overwrites them + // (ConflictResolutionService). const tailOps = await this.opLogStore.getOpsAfterSeq(snapshot.lastAppliedOpSeq); if (tailOps.length > 0) { @@ -293,6 +308,8 @@ export class OperationLogHydratorService { ); // No snapshot means we might be in a fresh install state or post-migration-check with no legacy data. // We must replay ALL operations from the beginning of the log. + // Status-blind on purpose — see the replay-policy note in the snapshot + // branch above. const allOps = await this.opLogStore.getOpsAfterSeq(0); if (allOps.length === 0) { @@ -557,8 +574,10 @@ export class OperationLogHydratorService { * Called after hydration to give failed ops another chance to apply now that * more state might be available (e.g., dependencies resolved by sync). * - * Failed ops are ops that previously failed during conflict resolution - * but may succeed now that more state has been loaded. + * Failed ops are ops whose archive side effect threw after their reducers + * committed, so the retry runs archive side effects ONLY + * (`skipReducerDispatch`) — hydration replay / the snapshot already carry + * their reducer effects. */ async retryFailedRemoteOps(): Promise { const failedOps = await this.opLogStore.getFailedRemoteOps(); @@ -583,7 +602,17 @@ export class OperationLogHydratorService { const opsToApply = orderedFailedOps.map((e) => e.op); const opIdToSeq = new Map(orderedFailedOps.map((e) => [e.op.id, e.seq])); - const result = await this.operationApplierService.applyOperations(opsToApply); + // `failed` can only be set AFTER a bulk dispatch committed (archive side + // effects run after the dispatch and are the only per-op failure point), + // so every failed op's reducer effect is already in state — via the + // snapshot when its seq <= lastAppliedOpSeq, via the status-blind tail + // replay above otherwise. Skip the reducer dispatch and re-run only the + // outstanding archive side effects: re-dispatching would double-apply + // additive reducers (syncTimeSpent, increaseSimpleCounterCounterToday) + // on every retry attempt. + const result = await this.operationApplierService.applyOperations(opsToApply, { + skipReducerDispatch: true, + }); // Mark successfully applied ops. const appliedSeqs = result.appliedOps @@ -591,6 +620,10 @@ export class OperationLogHydratorService { .filter((seq): seq is number => seq !== undefined); if (appliedSeqs.length > 0) { await this.opLogStore.markApplied(appliedSeqs); + // Parity with the primary remote-apply path: applyRemoteOperations only + // merges clocks for ops it marked applied, so a failed op's clock is + // merged here, when its application completes. + await this.opLogStore.mergeRemoteOpClocks(result.appliedOps); OpLog.normal( `OperationLogHydratorService: Successfully retried ${appliedSeqs.length} failed ops`, ); diff --git a/src/app/op-log/sync/operation-log-download.service.ts b/src/app/op-log/sync/operation-log-download.service.ts index c551689a84..daf055fb21 100644 --- a/src/app/op-log/sync/operation-log-download.service.ts +++ b/src/app/op-log/sync/operation-log-download.service.ts @@ -72,9 +72,18 @@ export class OperationLogDownloadService implements OnDestroy { this.clockDriftRetryServerTimestamp = null; } + /** + * @param options.includeOwnAndAppliedOps - Raw-rebuild mode for USE_REMOTE: + * skips the appliedOpIds filter AND downloads ops authored by this + * client (no excludeClient param). Required to reconstruct the full + * server history — the normal filters would drop everything the local + * store already knows, leaving a destructive replace with nothing to + * replay. Callers are expected to clear the local op store before + * appending the result. + */ async downloadRemoteOps( syncProvider: OperationSyncCapable, - options?: { forceFromSeq0?: boolean }, + options?: { forceFromSeq0?: boolean; includeOwnAndAppliedOps?: boolean }, ): Promise { if (!syncProvider) { OpLog.warn( @@ -88,7 +97,7 @@ export class OperationLogDownloadService implements OnDestroy { private async _downloadRemoteOpsViaApi( syncProvider: OperationSyncCapable, - options?: { forceFromSeq0?: boolean }, + options?: { forceFromSeq0?: boolean; includeOwnAndAppliedOps?: boolean }, ): Promise { const forceFromSeq0 = options?.forceFromSeq0 ?? false; OpLog.normal( @@ -117,8 +126,14 @@ export class OperationLogDownloadService implements OnDestroy { await this.lockService.request(LOCK_NAMES.DOWNLOAD, async () => { const lastServerSeq = forceFromSeq0 ? 0 : await syncProvider.getLastServerSeq(); - const appliedOpIds = await this.opLogStore.getAppliedOpIds(); - const clientId = await this.clientIdProvider.loadClientId(); + // Raw-rebuild mode: an empty applied set disables the duplicate filter + // below; a missing clientId makes the server include this client's own ops. + const appliedOpIds = options?.includeOwnAndAppliedOps + ? new Set() + : await this.opLogStore.getAppliedOpIds(); + const clientId = options?.includeOwnAndAppliedOps + ? null + : await this.clientIdProvider.loadClientId(); OpLog.verbose( `OperationLogDownloadService: [DEBUG] Starting download. ` + `lastServerSeq=${lastServerSeq}, appliedOpIds.size=${appliedOpIds.size}, clientId=${clientId}`, diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index dde580c6bc..f0cc485793 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -107,6 +107,7 @@ describe('OperationLogSyncService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }); rejectedOpsHandlerServiceSpy = jasmine.createSpyObj('RejectedOpsHandlerService', [ @@ -327,6 +328,7 @@ describe('OperationLogSyncService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }); const mockProvider = { @@ -439,6 +441,7 @@ describe('OperationLogSyncService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }; }); const setLastServerSeqSpy = jasmine @@ -690,6 +693,7 @@ describe('OperationLogSyncService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }); // handleRejectedOps returns 3 merged ops created @@ -877,6 +881,7 @@ describe('OperationLogSyncService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }); const mockProvider = { @@ -892,6 +897,55 @@ describe('OperationLogSyncService', () => { } }); + it('should NOT advance lastServerSeq when processing blocked at an incompatible op', async () => { + const remoteOp: Operation = { + id: 'op-future', + clientId: 'client-B', + actionType: 'test' as ActionType, + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-1', + payload: {}, + vectorClock: { clientB: 1 }, + timestamp: Date.now(), + schemaVersion: 99, + }; + + downloadServiceSpy.downloadRemoteOps.and.returnValue( + Promise.resolve({ + newOps: [remoteOp], + hasMore: false, + latestSeq: 5, + latestServerSeq: 5, + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + }), + ); + + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + }); + + const setLastServerSeqSpy = jasmine.createSpy('setLastServerSeq').and.resolveTo(); + const mockProvider = { + isReady: () => Promise.resolve(true), + setLastServerSeq: setLastServerSeqSpy, + } as any; + + const result = await service.downloadRemoteOps(mockProvider); + + // Cursor stays behind the blocked op so it is re-downloaded and retried + // after an app update instead of skipped forever. + expect(result.kind).toBe('ops_processed'); + expect(setLastServerSeqSpy).not.toHaveBeenCalled(); + }); + it('should return localWinOpsCreated: 0 and newOpsCount: 0 on server migration', async () => { downloadServiceSpy.downloadRemoteOps.and.returnValue( Promise.resolve({ @@ -955,6 +1009,7 @@ describe('OperationLogSyncService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }; }); @@ -2318,29 +2373,44 @@ describe('OperationLogSyncService', () => { describe('forceDownloadRemoteState', () => { let downloadServiceSpy: jasmine.SpyObj; + const makeRemoteOp = (id: string = 'op1'): Operation => ({ + id, + actionType: 'ACTION' as ActionType, + opType: 'UPDATE' as OpType, + entityType: 'TASK', + entityId: 'task1', + payload: {}, + clientId: 'remote', + vectorClock: { remote: 1 }, + timestamp: Date.now(), + schemaVersion: 1, + }); + beforeEach(() => { downloadServiceSpy = TestBed.inject( OperationLogDownloadService, ) as jasmine.SpyObj; - opLogStoreSpy.clearUnsyncedOps = jasmine - .createSpy('clearUnsyncedOps') + opLogStoreSpy.clearAllOperations = jasmine + .createSpy('clearAllOperations') .and.resolveTo(); + opLogStoreSpy.saveStateCache = jasmine.createSpy('saveStateCache').and.resolveTo(); }); - it('should clear unsynced ops before downloading', async () => { + it('should download BEFORE any destructive local mutation', async () => { const callOrder: string[] = []; - opLogStoreSpy.clearUnsyncedOps.and.callFake(async () => { - callOrder.push('clearUnsyncedOps'); + opLogStoreSpy.clearAllOperations.and.callFake(async () => { + callOrder.push('clearAllOperations'); }); downloadServiceSpy.downloadRemoteOps.and.callFake(async () => { callOrder.push('downloadRemoteOps'); return { - newOps: [], + newOps: [makeRemoteOp()], needsFullStateUpload: false, success: true, providerMode: 'superSyncOps', failedFileCount: 0, + latestServerSeq: 1, }; }); @@ -2351,24 +2421,25 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); - expect(callOrder[0]).toBe('clearUnsyncedOps'); + expect(callOrder).toEqual(['downloadRemoteOps', 'clearAllOperations']); }); - it('should capture a safety backup BEFORE clearing unsynced ops (#8107)', async () => { + it('should capture a safety backup BEFORE clearing local data (#8107)', async () => { const callOrder: string[] = []; backupServiceSpy.captureImportBackup.and.callFake(async () => { callOrder.push('captureImportBackup'); return 1; }); - opLogStoreSpy.clearUnsyncedOps.and.callFake(async () => { - callOrder.push('clearUnsyncedOps'); + opLogStoreSpy.clearAllOperations.and.callFake(async () => { + callOrder.push('clearAllOperations'); }); downloadServiceSpy.downloadRemoteOps.and.resolveTo({ - newOps: [], + newOps: [makeRemoteOp()], needsFullStateUpload: false, success: true, providerMode: 'superSyncOps', failedFileCount: 0, + latestServerSeq: 1, }); const mockProvider = { supportsOperationSync: true, @@ -2377,7 +2448,7 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); - expect(callOrder).toEqual(['captureImportBackup', 'clearUnsyncedOps']); + expect(callOrder).toEqual(['captureImportBackup', 'clearAllOperations']); }); it('should ABORT without wiping local data if the safety backup fails (#8107)', async () => { @@ -2389,18 +2460,18 @@ describe('OperationLogSyncService', () => { await expectAsync(service.forceDownloadRemoteState(mockProvider)).toBeRejected(); - expect(opLogStoreSpy.clearUnsyncedOps).not.toHaveBeenCalled(); - expect(opLogStoreSpy.clearFullStateOps).not.toHaveBeenCalled(); + expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); }); it('should offer to restore the previous data after replacing (#8107)', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ - newOps: [], + newOps: [makeRemoteOp()], needsFullStateUpload: false, success: true, providerMode: 'superSyncOps', failedFileCount: 0, + latestServerSeq: 1, }); const mockProvider = { supportsOperationSync: true, @@ -2419,11 +2490,12 @@ describe('OperationLogSyncService', () => { it('should reset lastServerSeq to 0', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ - newOps: [], + newOps: [makeRemoteOp()], needsFullStateUpload: false, success: true, providerMode: 'superSyncOps', failedFileCount: 0, + latestServerSeq: 1, }); const setLastServerSeqSpy = jasmine.createSpy('setLastServerSeq').and.resolveTo(); @@ -2438,13 +2510,14 @@ describe('OperationLogSyncService', () => { expect(setLastServerSeqSpy).toHaveBeenCalledWith(0); }); - it('should download ops with forceFromSeq0 option', async () => { + it('should download raw history: forceFromSeq0 AND includeOwnAndAppliedOps', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ - newOps: [], + newOps: [makeRemoteOp()], needsFullStateUpload: false, success: true, providerMode: 'superSyncOps', failedFileCount: 0, + latestServerSeq: 1, }); const mockProvider = { @@ -2454,13 +2527,13 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); - expect(downloadServiceSpy.downloadRemoteOps).toHaveBeenCalledWith( - mockProvider, - jasmine.objectContaining({ forceFromSeq0: true }), - ); + expect(downloadServiceSpy.downloadRemoteOps).toHaveBeenCalledWith(mockProvider, { + forceFromSeq0: true, + includeOwnAndAppliedOps: true, + }); }); - it('should throw when force download fails', async () => { + it('should throw when force download fails and leave local data untouched', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [], needsFullStateUpload: false, @@ -2468,6 +2541,30 @@ describe('OperationLogSyncService', () => { failedFileCount: 1, }); + const setLastServerSeqSpy = jasmine.createSpy('setLastServerSeq').and.resolveTo(); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: setLastServerSeqSpy, + } as any; + + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/Download failed/); + expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); + expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); + expect(setLastServerSeqSpy).not.toHaveBeenCalled(); + }); + + it('should refuse to rebuild from ops with a newer schema version BEFORE destroying anything', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [{ ...makeRemoteOp('op-future'), schemaVersion: 99 }], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), @@ -2475,10 +2572,41 @@ describe('OperationLogSyncService', () => { await expectAsync( service.forceDownloadRemoteState(mockProvider), - ).toBeRejectedWithError(/Download failed/); + ).toBeRejectedWithError(/newer schema version/); + expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); }); + it('should NOT advance the cursor past 0 when replay blocks on a migration failure', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 50, + }); + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + }); + + const setLastServerSeqSpy = jasmine.createSpy('setLastServerSeq').and.resolveTo(); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: setLastServerSeqSpy, + } as any; + + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/USE_REMOTE incomplete/); + expect(setLastServerSeqSpy).toHaveBeenCalledWith(0); + expect(setLastServerSeqSpy).not.toHaveBeenCalledWith(50); + }); + it('should process downloaded ops without confirmation', async () => { const mockOps: Operation[] = [ { @@ -2554,7 +2682,9 @@ describe('OperationLogSyncService', () => { expect(setLastServerSeqSpy).toHaveBeenCalledWith(50); }); - it('should handle empty remote state gracefully', async () => { + it('should REJECT an empty remote instead of silently succeeding', async () => { + // An empty remote is not a state to adopt: succeeding here used to wipe + // the local op-log bookkeeping while leaving live state unchanged. downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [], needsFullStateUpload: false, @@ -2563,13 +2693,18 @@ describe('OperationLogSyncService', () => { failedFileCount: 0, }); + const setLastServerSeqSpy = jasmine.createSpy('setLastServerSeq').and.resolveTo(); const mockProvider = { supportsOperationSync: true, - setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + setLastServerSeq: setLastServerSeqSpy, } as any; - await expectAsync(service.forceDownloadRemoteState(mockProvider)).toBeResolved(); + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/no data to rebuild from/); expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); + expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); + expect(setLastServerSeqSpy).not.toHaveBeenCalled(); }); it('should hydrate from snapshotState when present (file-based sync)', async () => { @@ -2618,12 +2753,21 @@ describe('OperationLogSyncService', () => { expect(setLastServerSeqSpy).toHaveBeenCalledWith(1); }); - it('should propagate errors from clearUnsyncedOps', async () => { + it('should propagate errors from clearAllOperations', async () => { const error = new Error('Failed to clear ops'); - opLogStoreSpy.clearUnsyncedOps.and.rejectWith(error); + opLogStoreSpy.clearAllOperations.and.rejectWith(error); + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); const mockProvider = { supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), } as any; await expectAsync(service.forceDownloadRemoteState(mockProvider)).toBeRejectedWith( @@ -3568,6 +3712,68 @@ describe('OperationLogSyncService', () => { expect(result.kind).toBe('cancelled'); }); + it('should show conflict dialog for local work accepted in the SAME upload round (pre-upload snapshot race)', async () => { + const piggybackedSyncImport: Operation = { + id: 'import-1', + clientId: 'client-B', + actionType: ActionType.LOAD_ALL_DATA, + opType: OpType.SyncImport, + entityType: 'ALL', + payload: { task: { ids: ['remote-task'] } }, + vectorClock: { clientB: 5 }, + timestamp: Date.now(), + schemaVersion: 1, + }; + + uploadServiceSpy.uploadPendingOps.and.resolveTo({ + uploadedCount: 1, + piggybackedOps: [piggybackedSyncImport], + rejectedCount: 0, + rejectedOps: [], + }); + + const pendingEntry: OperationLogEntry = { + seq: 1, + op: { + id: 'local-op-1', + clientId: 'client-A', + actionType: 'test' as ActionType, + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-1', + payload: { title: 'Local Title' }, + vectorClock: { clientA: 1 }, + timestamp: Date.now(), + schemaVersion: 1, + }, + appliedAt: Date.now(), + source: 'local', + }; + // The op is pending BEFORE the upload but marked synced DURING it (server + // accepted it in the same round that piggybacked the import) — a live + // post-upload read no longer sees it. + let getUnsyncedCalls = 0; + opLogStoreSpy.getUnsynced.and.callFake(async () => { + getUnsyncedCalls++; + return getUnsyncedCalls === 1 ? [pendingEntry] : []; + }); + + syncImportConflictDialogServiceSpy.showConflictDialog.and.resolveTo('CANCEL'); + + const mockProvider = { + isReady: () => Promise.resolve(true), + } as any; + + const result = await service.uploadPendingOps(mockProvider); + + // Without the pre-upload snapshot, the gate would read the (now empty) + // live pending set and silently accept the import over the local edit. + expect(syncImportConflictDialogServiceSpy.showConflictDialog).toHaveBeenCalledWith( + jasmine.objectContaining({ scenario: 'INCOMING_IMPORT' }), + ); + expect(result.kind).toBe('cancelled'); + }); + it('should process piggybacked SYNC_IMPORT silently when client only has already-synced meaningful data', async () => { const piggybackedSyncImport: Operation = { id: 'import-1', diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 61801c3758..7fd655e5f8 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -10,7 +10,6 @@ import { import { FILE_BASED_SYNC_CONSTANTS } from '../sync-providers/file-based/file-based-sync.types'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { BackupService } from '../backup/backup.service'; -import { FULL_STATE_OP_TYPES } from '../core/operation.types'; import { OpLog } from '../../core/log'; import { OperationSyncCapable, @@ -39,6 +38,7 @@ import { SyncImportConflictResolution, } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { SyncImportConflictGateService } from './sync-import-conflict-gate.service'; +import { CURRENT_SCHEMA_VERSION } from '../persistence/schema-migration.service'; import { SyncProviderManager } from '../sync-providers/provider-manager.service'; import { getDefaultMainModelData } from '../model/model-config'; import { loadAllData } from '../../root-store/meta/load-all-data.action'; @@ -209,6 +209,14 @@ export class OperationLogSyncService { return { kind: 'blocked_fresh_client' }; } + // Capture pending ops BEFORE the upload round (the flush at the top of this + // method already drained in-flight writes). The piggyback conflict gate below + // must judge "would this import discard local work?" against this snapshot: + // ops accepted by the server during the upload are marked synced per chunk, + // so a post-upload getUnsynced() read no longer sees local work that a + // piggybacked import is about to replace. + const pendingOpsAtUploadStart = await this.opLogStore.getUnsynced(); + // SERVER MIGRATION CHECK: Passed as callback to execute INSIDE the upload lock. // This prevents race conditions where multiple tabs could both detect migration // and create duplicate SYNC_IMPORT operations. @@ -248,7 +256,10 @@ export class OperationLogSyncService { const piggybackedConflict = await this.syncImportConflictGateService.checkIncomingFullStateConflict( result.piggybackedOps, - { isNeverSynced: isNeverSyncedAtSyncStart }, + { + isNeverSynced: isNeverSyncedAtSyncStart, + preCapturedPendingOps: pendingOpsAtUploadStart, + }, ); if (piggybackedConflict.fullStateOp) { const { fullStateOp, pendingOps, dialogData } = piggybackedConflict; @@ -313,7 +324,12 @@ export class OperationLogSyncService { // so that a crash — or a cancelled/USE_REMOTE/USE_LOCAL SYNC_IMPORT dialog, all of // which return early ABOVE without reaching here — cannot advance the seq past ops // that were never stored. Mirrors the download path's invariant. - if (result.lastServerSeqToPersist !== undefined) { + // A version/migration block keeps the cursor behind the blocked op so it is + // re-downloaded and retried after an app update instead of skipped forever. + if ( + result.lastServerSeqToPersist !== undefined && + !processResult.blockedByIncompatibleOp + ) { await syncProvider.setLastServerSeq(result.lastServerSeqToPersist); } } @@ -915,7 +931,9 @@ export class OperationLogSyncService { // This ensures localStorage and IndexedDB stay in sync. If we crash before this point, // lastServerSeq won't be updated, and the client will re-download the ops on next sync. // This is the correct behavior - better to re-download than to skip ops. - if (result.latestServerSeq !== undefined) { + // A version/migration block keeps the cursor behind the blocked op so it is + // re-downloaded and retried after an app update instead of skipped forever. + if (result.latestServerSeq !== undefined && !processResult.blockedByIncompatibleOp) { await syncProvider.setLastServerSeq(result.latestServerSeq); } @@ -1059,8 +1077,9 @@ export class OperationLogSyncService { ); // Persist the cursor only AFTER the ops are applied, matching the normal - // incremental path's crash-safety ordering. - if (result.latestServerSeq !== undefined) { + // incremental path's crash-safety ordering. A version/migration block keeps + // the cursor behind the blocked op (retried after an app update). + if (result.latestServerSeq !== undefined && !processResult.blockedByIncompatibleOp) { await syncProvider.setLastServerSeq(result.latestServerSeq); } @@ -1120,17 +1139,21 @@ export class OperationLogSyncService { * Force download all remote state, replacing local data. * This is used when user explicitly chooses "USE_REMOTE" in conflict resolution. * - * Clears all local unsynced operations and downloads from seq 0 to get - * the complete remote state including any SYNC_IMPORT. + * Download-first rebuild: the COMPLETE server history (including this + * client's own ops and ops already known locally) is downloaded and + * validated BEFORE any local mutation. Only then is the local op-log + * replaced wholesale with the server history and replayed from a defaults + * baseline. Aborts without touching local state on download failure, an + * empty remote, or a remote containing newer-schema ops. * - * IMPORTANT: This also resets the vector clock to the remote snapshot's clock + * IMPORTANT: This also resets the vector clock to the remote's rebuilt clock * to ensure rejected local ops don't pollute the causal history. * * @param syncProvider - The sync provider to download from */ async forceDownloadRemoteState(syncProvider: OperationSyncCapable): Promise { OpLog.warn( - 'OperationLogSyncService: Force downloading remote state - clearing local import and unsynced ops.', + 'OperationLogSyncService: Force downloading remote state for a full rebuild.', ); // Safety net (#8107): snapshot current local state before we destroy it, so an @@ -1151,25 +1174,21 @@ export class OperationLogSyncService { ); } - // IMPORTANT: Clear local full-state ops (SYNC_IMPORT, BACKUP_IMPORT, REPAIR) - // This is critical - if the user chose USE_REMOTE, we must not filter incoming - // ops against the local import that we're discarding. - const clearedFullStateOps = await this.opLogStore.clearFullStateOps(); - if (clearedFullStateOps > 0) { - OpLog.normal( - `OperationLogSyncService: Cleared ${clearedFullStateOps} local full-state op(s).`, - ); - } - - // Clear all unsynced local ops - we're replacing them with remote state - await this.opLogStore.clearUnsyncedOps(); - - // Reset lastServerSeq to 0 so we download everything - await syncProvider.setLastServerSeq(0); - - // Download all remote ops from the beginning + // ───────────────────────────────────────────────────────────────────────── + // PHASE 1 — download and validate. Nothing local is mutated until the + // complete server history is in memory: a network failure here must leave + // local state untouched (previously the op-log was cleared and the cursor + // reset BEFORE the download, so a failed download stranded the client with + // destroyed local bookkeeping). + // + // Raw-rebuild mode: includes ops authored by this client and ops already + // known locally. The normal download filters drop both, which made the old + // USE_REMOTE unable to rebuild a server history this client had (fully or + // partially) produced itself. + // ───────────────────────────────────────────────────────────────────────── const result = await this.downloadService.downloadRemoteOps(syncProvider, { forceFromSeq0: true, + includeOwnAndAppliedOps: true, }); if (!result.success) { @@ -1179,20 +1198,60 @@ export class OperationLogSyncService { ); } - // Reset the vector clock to the remote snapshot's clock. - // This removes entries from rejected local ops that would otherwise - // pollute the causal history and cause incorrect conflict detection. - if (result.snapshotVectorClock) { - await this.opLogStore.setVectorClock(result.snapshotVectorClock); - OpLog.normal( - 'OperationLogSyncService: Reset vector clock to remote snapshot clock.', + const hasSnapshotState = + result.providerMode === 'fileSnapshotOps' && !!result.snapshotState; + if (!hasSnapshotState && result.newOps.length === 0) { + // An empty remote is not a state to adopt. Silently succeeding here used + // to leave live NgRx state untouched while the op-log bookkeeping was + // already wiped — state and log permanently disagreeing. + throw new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'); + } + + // Refuse BEFORE destroying anything if the remote history contains ops this + // client cannot interpret (newer schema) — replay would block midway and + // leave a partial rebuild. Migration exceptions can still surface during + // replay; those are handled after processRemoteOps below. + const tooNewOp = result.newOps.find( + (op) => (op.schemaVersion ?? 1) > CURRENT_SCHEMA_VERSION, + ); + if (tooNewOp) { + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.VERSION_TOO_OLD, + actionStr: T.PS.UPDATE_APP, + actionFn: () => window.open('https://super-productivity.com/download', '_blank'), + }); + throw new Error( + 'USE_REMOTE aborted: remote history contains ops from a newer schema version — update the app first.', ); } + // ───────────────────────────────────────────────────────────────────────── + // PHASE 2 — destructive replace (local mutations only, no network left). + // The local op-log becomes exactly the server history: clearing ALL ops + // (not just unsynced/full-state ones) lets the raw replay below re-append + // and re-apply ops this client already knew. The old keep-synced-ops + // variant made the duplicate filter skip them, so a "rebuild" replayed only + // an unseen suffix onto a defaults reset. + // ───────────────────────────────────────────────────────────────────────── + await this.opLogStore.clearAllOperations(); + await syncProvider.setLastServerSeq(0); + + // Reset the vector clock to the remote's causal knowledge (snapshot clock + // merged with every downloaded op clock). This also drops entries from + // rejected local ops that would otherwise pollute conflict detection. + let rebuiltClock: VectorClock = { ...(result.snapshotVectorClock ?? {}) }; + for (const opClock of result.allOpClocks ?? []) { + rebuiltClock = mergeVectorClocks(rebuiltClock, opClock); + } + await this.opLogStore.setVectorClock(rebuiltClock); + OpLog.normal('OperationLogSyncService: Reset vector clock to rebuilt remote clock.'); + // FILE-BASED SYNC: Handle snapshot state from force download. // When downloading from seq 0 on file-based providers, we may receive a // snapshotState instead of incremental ops. This happens when the remote // has a SYNC_IMPORT (full state snapshot) with empty recentOps. + // hydrateFromRemoteSync persists its own state cache + vector clock. if (result.providerMode === 'fileSnapshotOps' && result.snapshotState) { OpLog.normal( 'OperationLogSyncService: Force download received snapshotState. Hydrating...', @@ -1235,48 +1294,59 @@ export class OperationLogSyncService { return; } - if (result.newOps.length > 0) { - // Check if there's a full-state op in the batch (SYNC_IMPORT would replace state) - const hasFullStateOp = result.newOps.some((op) => - FULL_STATE_OP_TYPES.has(op.opType), + // Crash safety: persist a defaults baseline so an interrupted replay + // hydrates as defaults + appended prefix on next boot (a consistent prefix + // of server state, completed by the next sync from cursor 0) instead of + // layering the new history onto the stale pre-replace snapshot. + await this.opLogStore.saveStateCache({ + state: getDefaultMainModelData(), + lastAppliedOpSeq: 0, + vectorClock: rebuiltClock, + compactedAt: Date.now(), + snapshotEntityKeys: [], + }); + + // Reset live state to defaults, then replay the COMPLETE server history on + // top. A full-state op in the history replaces state again by its own + // semantics; a purely incremental history rebuilds from this baseline. + const defaultData = getDefaultMainModelData(); + this.store.dispatch( + loadAllData({ + appDataComplete: defaultData as Parameters< + typeof loadAllData + >[0]['appDataComplete'], + }), + ); + // Brief yield to let NgRx process the state reset + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). + // Skip conflict detection because the NgRx store was just reset to empty state, + // which causes all entities to appear missing and CONCURRENT ops to be discarded. + // Validation failure is surfaced via the session-validation latch. (#7330) + const processResult = await this.remoteOpsProcessingService.processRemoteOps( + result.newOps, + { skipConflictDetection: true }, + ); + + if (processResult.blockedByIncompatibleOp) { + // Version blocks were pre-checked above; only a migration exception lands + // here. The rebuild is partial: keep the cursor at 0 so the next sync + // retries the remainder, and surface the failure — the Undo snack still + // offers the pre-replace backup. + this._showRestorePreviousDataSnack(backupSavedAt); + throw new Error( + 'USE_REMOTE incomplete: an op failed schema migration during replay.', ); + } - // If no full-state op, we need to reset state before applying incremental ops. - // This is because USE_REMOTE should REPLACE local state with remote state, - // not merge remote ops into existing local state. - if (!hasFullStateOp) { - OpLog.normal( - 'OperationLogSyncService: No full-state op in remote. Resetting state before applying incremental ops.', - ); - // Reset to default/empty state so incremental ops build fresh state - const defaultData = getDefaultMainModelData(); - this.store.dispatch( - loadAllData({ - appDataComplete: defaultData as Parameters< - typeof loadAllData - >[0]['appDataComplete'], - }), - ); - // Brief yield to let NgRx process the state reset - await new Promise((resolve) => setTimeout(resolve, 0)); - } - - // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). - // Skip conflict detection because the NgRx store was just reset to empty state, - // which causes all entities to appear missing and CONCURRENT ops to be discarded. - // Validation failure is surfaced via the session-validation latch. (#7330) - await this.remoteOpsProcessingService.processRemoteOps(result.newOps, { - skipConflictDetection: true, - }); - - // Update lastServerSeq - if (result.latestServerSeq !== undefined) { - await syncProvider.setLastServerSeq(result.latestServerSeq); - } + // Update lastServerSeq + if (result.latestServerSeq !== undefined) { + await syncProvider.setLastServerSeq(result.latestServerSeq); } OpLog.normal( - `OperationLogSyncService: Force download complete. Processed ${result.newOps.length} ops.`, + `OperationLogSyncService: Force download complete. Rebuilt from ${result.newOps.length} ops.`, ); this._showRestorePreviousDataSnack(backupSavedAt); } diff --git a/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts b/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts index 453032f51e..11a1badebb 100644 --- a/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts +++ b/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts @@ -191,6 +191,7 @@ describe('OperationLogSyncService + OperationLogUploadService — piggyback seq allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }); dialogServiceSpy = jasmine.createSpyObj('SyncImportConflictDialogService', [ @@ -393,6 +394,7 @@ describe('OperationLogSyncService + OperationLogUploadService — piggyback seq allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, }; }); setLastServerSeqSpy.and.callFake(async (n: number) => { diff --git a/src/app/op-log/sync/operation-log-upload.service.spec.ts b/src/app/op-log/sync/operation-log-upload.service.spec.ts index 46dbe09b30..8a9a5fac18 100644 --- a/src/app/op-log/sync/operation-log-upload.service.spec.ts +++ b/src/app/op-log/sync/operation-log-upload.service.spec.ts @@ -1016,7 +1016,7 @@ describe('OperationLogUploadService', () => { }); it('should mark regular ops as synced when full-state op is uploaded (ops before snapshot)', async () => { - // Regular op id 'op-0' sorts BEFORE full-state op id 'op-1', + // Regular op seq 1 is BEFORE full-state op seq 2, // meaning the regular op was created before the snapshot and is included in it. const regularEntry = createMockEntry(1, 'op-0', 'client-1'); const fullStateEntry = createFullStateEntry( @@ -1042,7 +1042,7 @@ describe('OperationLogUploadService', () => { }); it('should mark regular ops as synced when Repair op is uploaded (ops before snapshot)', async () => { - // Regular op id 'op-0' sorts BEFORE full-state op id 'op-1' + // Regular op seq 1 is BEFORE full-state op seq 2 const regularEntry = createMockEntry(1, 'op-0', 'client-1'); const fullStateEntry = createFullStateEntry(2, 'op-1', 'client-1', OpType.Repair); mockOpLogStore.getUnsynced.and.returnValue( @@ -1062,7 +1062,7 @@ describe('OperationLogUploadService', () => { }); it('should upload regular ops created AFTER full-state snapshot', async () => { - // Full-state op id 'op-1' sorts BEFORE regular op id 'op-2', + // Full-state op seq 1 is BEFORE regular op seq 2, // meaning the regular op was created AFTER the snapshot and is NOT included in it. const fullStateEntry = createFullStateEntry( 1, @@ -1094,6 +1094,40 @@ describe('OperationLogUploadService', () => { expect(mockOpLogStore.markSynced).toHaveBeenCalledWith([2]); }); + it('should upload a post-snapshot op even when its UUIDv7 id sorts before the full-state op id (clock rollback)', async () => { + // Wall-clock rollback regression: the regular op was created AFTER the + // snapshot (seq 2 > seq 1) but got a lexically SMALLER UUIDv7 id + // ('op-0' < 'op-1'). It is NOT in the frozen snapshot payload, so it + // must be uploaded — never just marked synced. + const fullStateEntry = createFullStateEntry( + 1, + 'op-1', + 'client-1', + OpType.BackupImport, + ); + const regularEntry = createMockEntry(2, 'op-0', 'client-1'); + mockOpLogStore.getUnsynced.and.returnValue( + Promise.resolve([fullStateEntry, regularEntry]), + ); + mockApiProvider.uploadOps.and.returnValue( + Promise.resolve({ + results: [{ opId: 'op-0', accepted: true }], + latestSeq: 2, + newOps: [], + }), + ); + + const result = await service.uploadPendingOps(mockApiProvider); + + expect(mockApiProvider.uploadSnapshot).toHaveBeenCalled(); + expect(mockApiProvider.uploadOps).toHaveBeenCalled(); + const uploadedOpIds = mockApiProvider.uploadOps.calls + .mostRecent() + .args[0].map((op) => op.id); + expect(uploadedOpIds).toEqual(['op-0']); + expect(result.uploadedCount).toBe(2); + }); + it('should NOT auto-set isCleanSlate for SyncImport unlike BackupImport/Repair', async () => { const entry = createFullStateEntry(1, 'op-1', 'client-1', OpType.SyncImport); mockOpLogStore.getUnsynced.and.returnValue(Promise.resolve([entry])); diff --git a/src/app/op-log/sync/operation-log-upload.service.ts b/src/app/op-log/sync/operation-log-upload.service.ts index a84d79d977..93bef79ca4 100644 --- a/src/app/op-log/sync/operation-log-upload.service.ts +++ b/src/app/op-log/sync/operation-log-upload.service.ts @@ -182,7 +182,10 @@ export class OperationLogUploadService { // Upload full-state operations via snapshot endpoint let fullStateOpUploaded = false; - let lastUploadedFullStateOpId: string | undefined; + // Local append order (seq), not op id: UUIDv7 ids follow the wall clock and + // can order a post-snapshot op BEFORE the full-state op after a clock + // rollback, which would mark it synced without ever uploading it. + let lastUploadedFullStateOpSeq: number | undefined; for (const entry of fullStateOps) { // BackupImport/Repair: always wipe server (recovery operations replace all state) // SyncImport: only wipe when explicitly requested (preserves SYNC_IMPORT_EXISTS check) @@ -202,9 +205,10 @@ export class OperationLogUploadService { lastKnownServerSeq = result.serverSeq; highestReceivedSeq = Math.max(highestReceivedSeq, result.serverSeq); } - // Track that a full-state op was uploaded - regular ops before it are already included + // Track that a full-state op was uploaded - regular ops appended before it + // are already included in its frozen snapshot payload fullStateOpUploaded = true; - lastUploadedFullStateOpId = entry.op.id; + lastUploadedFullStateOpSeq = entry.seq; } else { // Special handling for SYNC_IMPORT_EXISTS: another client already uploaded // a SYNC_IMPORT. We should delete our local SYNC_IMPORT and let the normal @@ -245,11 +249,11 @@ export class OperationLogUploadService { return; } - if (fullStateOpUploaded && lastUploadedFullStateOpId) { + if (fullStateOpUploaded && lastUploadedFullStateOpSeq !== undefined) { const { opsIncludedInSnapshot, opsAfterSnapshot } = planRegularOpsAfterFullStateUpload({ regularOps, - lastUploadedFullStateOpId, + lastUploadedFullStateOpSeq, }); if (opsIncludedInSnapshot.length > 0) { diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index 0392b7ae3d..c59653fc0f 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -4,7 +4,6 @@ import { of } from 'rxjs'; import { RemoteOpsProcessingService } from './remote-ops-processing.service'; import { SchemaMigrationService, - MAX_VERSION_SKIP, MIN_SUPPORTED_SCHEMA_VERSION, } from '../persistence/schema-migration.service'; import { SnackService } from '../../core/snack/snack.service'; @@ -400,7 +399,7 @@ describe('RemoteOpsProcessingService', () => { ); }); - it('should skip ops that throw during migration but continue processing others', async () => { + it('should stop the batch at the first op that throws during migration and only process the prefix', async () => { const remoteOps: Operation[] = [ { id: 'op1', schemaVersion: 1 } as Operation, { id: 'throws', schemaVersion: 1 } as Operation, @@ -419,17 +418,26 @@ describe('RemoteOpsProcessingService', () => { vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); - await service.processRemoteOps(remoteOps); + const result = await service.processRemoteOps(remoteOps); - // op1 and op3 should be processed (appendBatchSkipDuplicates called with array of both) + // Only the prefix before the blocked op is processed. op3 must NOT be + // applied: it may depend on the blocked op, and the un-advanced cursor + // will re-deliver it after the migration issue is fixed. expect(opLogStoreSpy.appendBatchSkipDuplicates).toHaveBeenCalledWith( - [remoteOps[0], remoteOps[2]], + [remoteOps[0]], 'remote', { pendingApply: true }, ); + expect(result.blockedByIncompatibleOp).toBe(true); + expect(snackServiceSpy.open).toHaveBeenCalledWith( + jasmine.objectContaining({ + type: 'ERROR', + msg: T.F.SYNC.S.MIGRATION_FAILED, + }), + ); }); - it('should return early when all ops fail migration', async () => { + it('should block without processing anything when the first op fails migration', async () => { const remoteOps: Operation[] = [ { id: 'op1', schemaVersion: 1 } as Operation, { id: 'op2', schemaVersion: 1 } as Operation, @@ -446,6 +454,7 @@ describe('RemoteOpsProcessingService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, }); }); @@ -487,12 +496,13 @@ describe('RemoteOpsProcessingService', () => { ); }); - it('should show error snackbar and abort if version is too new', async () => { - const remoteOps: Operation[] = [ - { id: 'op1', schemaVersion: 1 + MAX_VERSION_SKIP + 1 } as Operation, - ]; + it('should block any op from a newer schema version (no forward-compat band)', async () => { + // Current version is 1 (set in beforeEach). Even version 2 — one ahead — + // must block: real migrations rename/split fields, so a future op applied + // verbatim corrupts state. + const remoteOps: Operation[] = [{ id: 'op1', schemaVersion: 2 } as Operation]; - await service.processRemoteOps(remoteOps); + const result = await service.processRemoteOps(remoteOps); expect(snackServiceSpy.open).toHaveBeenCalledWith( jasmine.objectContaining({ @@ -503,6 +513,30 @@ describe('RemoteOpsProcessingService', () => { // Should not proceed to apply ops expect(opLogStoreSpy.getUnsynced).not.toHaveBeenCalled(); + expect(result.blockedByIncompatibleOp).toBe(true); + }); + + it('should process the prefix before a too-new op but flag the block', async () => { + const remoteOps: Operation[] = [ + { id: 'op1', schemaVersion: 1 } as Operation, + { id: 'future', schemaVersion: 2 } as Operation, + { id: 'op3', schemaVersion: 1 } as Operation, + ]; + + opLogStoreSpy.getUnsynced.and.returnValue(Promise.resolve([])); + opLogStoreSpy.getUnsyncedByEntity.and.returnValue(Promise.resolve(new Map())); + vectorClockServiceSpy.getEntityFrontier.and.returnValue(Promise.resolve(new Map())); + vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); + opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); + + const result = await service.processRemoteOps(remoteOps); + + expect(opLogStoreSpy.appendBatchSkipDuplicates).toHaveBeenCalledWith( + [remoteOps[0]], + 'remote', + { pendingApply: true }, + ); + expect(result.blockedByIncompatibleOp).toBe(true); }); it('should show error snackbar and abort if version is below minimum supported', async () => { @@ -526,69 +560,26 @@ describe('RemoteOpsProcessingService', () => { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, }); }); - it('should show warning once per session when receiving ops from newer version', async () => { - // Current version is 1 (set in beforeEach) - const remoteOps: Operation[] = [ - { id: 'op1', schemaVersion: 2 } as Operation, - { id: 'op2', schemaVersion: 2 } as Operation, - ]; - - // Setup for processing - opLogStoreSpy.getUnsynced.and.returnValue(Promise.resolve([])); - opLogStoreSpy.getUnsyncedByEntity.and.returnValue(Promise.resolve(new Map())); - vectorClockServiceSpy.getEntityFrontier.and.returnValue(Promise.resolve(new Map())); - vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); - opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); - opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - - await service.processRemoteOps(remoteOps); - - // Should show warning exactly once (not twice for two ops) - expect(snackServiceSpy.open).toHaveBeenCalledTimes(1); - expect(snackServiceSpy.open).toHaveBeenCalledWith( - jasmine.objectContaining({ - type: 'WARNING', - msg: T.F.SYNC.S.NEWER_VERSION_AVAILABLE, - }), - ); - - // Should still process the ops (appendBatchSkipDuplicates called with both ops) - expect(opLogStoreSpy.appendBatchSkipDuplicates).toHaveBeenCalledWith( - remoteOps, - 'remote', - { - pendingApply: true, - }, - ); - }); - - it('should not show newer version warning again in same session', async () => { + it('should show the version-block error only once per session', async () => { // Current version is 1 (set in beforeEach) const remoteOps1: Operation[] = [{ id: 'op1', schemaVersion: 2 } as Operation]; const remoteOps2: Operation[] = [{ id: 'op2', schemaVersion: 2 } as Operation]; - // Setup for processing - opLogStoreSpy.getUnsynced.and.returnValue(Promise.resolve([])); - opLogStoreSpy.getUnsyncedByEntity.and.returnValue(Promise.resolve(new Map())); - vectorClockServiceSpy.getEntityFrontier.and.returnValue(Promise.resolve(new Map())); - vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); - opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); - opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - // First call await service.processRemoteOps(remoteOps1); - // Second call (same session) + // Second call (same session — periodic sync retries re-hit the block) await service.processRemoteOps(remoteOps2); - // Warning should only be shown once across both calls + // Error snack should only be shown once across both calls expect(snackServiceSpy.open).toHaveBeenCalledTimes(1); expect(snackServiceSpy.open).toHaveBeenCalledWith( jasmine.objectContaining({ - type: 'WARNING', - msg: T.F.SYNC.S.NEWER_VERSION_AVAILABLE, + type: 'ERROR', + msg: T.F.SYNC.S.VERSION_TOO_OLD, }), ); }); diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index 576e0e5b61..d1e910f3c9 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -19,7 +19,6 @@ import { ValidateStateService } from '../validation/validate-state.service'; import { SyncSessionValidationService } from './sync-session-validation.service'; import { VectorClockService } from './vector-clock.service'; import { - MAX_VERSION_SKIP, MIN_SUPPORTED_SCHEMA_VERSION, SchemaMigrationService, } from '../persistence/schema-migration.service'; @@ -69,8 +68,8 @@ export class RemoteOpsProcessingService { private writeFlushService = inject(OperationWriteFlushService); private injector = inject(Injector); - /** Flag to show newer version warning only once per session */ - private _hasWarnedNewerVersionThisSession = false; + /** Flag to show version-incompatibility warnings only once per session */ + private _hasWarnedVersionBlockThisSession = false; /** Flag to show migration failure warning only once per session */ private _hasWarnedMigrationFailureThisSession = false; @@ -100,6 +99,7 @@ export class RemoteOpsProcessingService { filteredOpCount: number; filteringImport?: Operation; isLocalUnsyncedImport: boolean; + blockedByIncompatibleOp: boolean; }> { // Validation failure surfaces via the SyncSessionValidationService latch // (#7330). `validateAfterSync` and the conflict-resolution validation path @@ -109,52 +109,41 @@ export class RemoteOpsProcessingService { // ───────────────────────────────────────────────────────────────────────── // STEP 1: Schema Migration (Receiver-Side) // Migrate ops from older schema versions to current version. - // - Ops below MIN_SUPPORTED_SCHEMA_VERSION: error, stop sync - // - Ops beyond MAX_VERSION_SKIP: error, stop sync - // - Ops from newer version (within skip): warning once per session, continue + // + // Blocking semantics: an op that cannot be terminally processed here — + // below MIN_SUPPORTED_SCHEMA_VERSION, from a NEWER schema version (real + // migrations rename/split fields, so a future op applied verbatim corrupts + // state), or throwing during migration — STOPS the batch at that op. Ops + // before the block are processed normally; the blocked op and everything + // after are neither stored nor applied. Callers must NOT advance the + // server cursor when `blockedByIncompatibleOp` is true, so the blocked op + // is re-downloaded and retried after an app update / migration fix instead + // of being skipped forever (already-processed prefix ops are deduplicated + // by the appliedOpIds download filter). Ops migrated to `null` are an + // intentional terminal drop and do NOT block. // ───────────────────────────────────────────────────────────────────────── const currentVersion = this.schemaMigrationService.getCurrentVersion(); const migratedOps: Operation[] = []; const droppedEntityIds = new Set(); - const failedMigrationOpIds: string[] = []; - let updateRequired = false; + let blockReason: 'VERSION_UNSUPPORTED' | 'VERSION_TOO_NEW' | 'MIGRATION_FAILED' = + 'MIGRATION_FAILED'; + let blockedOp: Operation | null = null; for (const op of remoteOps) { const opVersion = op.schemaVersion ?? 1; - // Check if remote op is too old (below minimum supported) + // Op below minimum supported version: no migration path exists. if (opVersion < MIN_SUPPORTED_SCHEMA_VERSION) { - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.VERSION_UNSUPPORTED, - actionStr: T.PS.UPDATE_APP, - actionFn: () => - window.open('https://super-productivity.com/download', '_blank'), - }); - return { - localWinOpsCreated: 0, - allOpsFilteredBySyncImport: false, - filteredOpCount: 0, - isLocalUnsyncedImport: false, - }; - } - - // Check if remote op is too new (exceeds supported skip) - if (opVersion > currentVersion + MAX_VERSION_SKIP) { - updateRequired = true; + blockedOp = op; + blockReason = 'VERSION_UNSUPPORTED'; break; } - // Warn once per session if receiving ops from a newer version - if (opVersion > currentVersion && !this._hasWarnedNewerVersionThisSession) { - this._hasWarnedNewerVersionThisSession = true; - this.snackService.open({ - type: 'WARNING', - msg: T.F.SYNC.S.NEWER_VERSION_AVAILABLE, - actionStr: T.PS.UPDATE_APP, - actionFn: () => - window.open('https://super-productivity.com/download', '_blank'), - }); + // Op from a newer schema version: this client cannot interpret it safely. + if (opVersion > currentVersion) { + blockedOp = op; + blockReason = 'VERSION_TOO_NEW'; + break; } try { @@ -178,44 +167,23 @@ export class RemoteOpsProcessingService { } } catch (e) { OpLog.err(`RemoteOpsProcessingService: Migration failed for op ${op.id}`, e); - // Track failed migrations to notify user. If ops are from a compatible version, - // this indicates a bug or data corruption. - failedMigrationOpIds.push(op.id); + blockedOp = op; + blockReason = 'MIGRATION_FAILED'; + break; } } - // Notify user if any migrations failed (once per session to avoid spam) - if (failedMigrationOpIds.length > 0) { - OpLog.warn( - `RemoteOpsProcessingService: ${failedMigrationOpIds.length} op(s) failed migration`, - { failedOpIds: failedMigrationOpIds }, + if (blockedOp) { + OpLog.err( + `RemoteOpsProcessingService: Blocked at op ${blockedOp.id} (${blockReason}, ` + + `schemaVersion=${blockedOp.schemaVersion ?? 1}, current=${currentVersion}). ` + + 'Processing the batch prefix only; cursor must not advance past this op.', ); - if (!this._hasWarnedMigrationFailureThisSession) { - this._hasWarnedMigrationFailureThisSession = true; - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.MIGRATION_FAILED, - }); - } - } - - if (updateRequired) { - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.VERSION_TOO_OLD, - actionStr: T.PS.UPDATE_APP, - actionFn: () => window.open('https://super-productivity.com/download', '_blank'), - }); - return { - localWinOpsCreated: 0, - allOpsFilteredBySyncImport: false, - filteredOpCount: 0, - isLocalUnsyncedImport: false, - }; + this._notifyBlockedOp(blockReason); } if (migratedOps.length === 0) { - if (remoteOps.length > 0) { + if (remoteOps.length > 0 && !blockedOp) { OpLog.normal( 'RemoteOpsProcessingService: All remote ops were dropped during migration.', ); @@ -225,8 +193,10 @@ export class RemoteOpsProcessingService { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp: blockedOp !== null, }; } + const blockedByIncompatibleOp = blockedOp !== null; // ───────────────────────────────────────────────────────────────────────── // STEP 2: Filter ops invalidated by SYNC_IMPORT @@ -258,6 +228,7 @@ export class RemoteOpsProcessingService { filteredOpCount: invalidatedOps.length, filteringImport, isLocalUnsyncedImport, + blockedByIncompatibleOp, }; } @@ -285,6 +256,7 @@ export class RemoteOpsProcessingService { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp, }; } @@ -316,6 +288,7 @@ export class RemoteOpsProcessingService { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp, }; } @@ -376,9 +349,42 @@ export class RemoteOpsProcessingService { allOpsFilteredBySyncImport: false, filteredOpCount: 0, isLocalUnsyncedImport: false, + blockedByIncompatibleOp, }; } + /** + * User notification for a version/migration block, once per session per + * category to avoid snack spam from periodic sync retries (the block persists + * until an app update or migration fix, and every retry re-hits it). + */ + private _notifyBlockedOp( + reason: 'VERSION_UNSUPPORTED' | 'VERSION_TOO_NEW' | 'MIGRATION_FAILED', + ): void { + if (reason === 'MIGRATION_FAILED') { + if (!this._hasWarnedMigrationFailureThisSession) { + this._hasWarnedMigrationFailureThisSession = true; + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.MIGRATION_FAILED, + }); + } + return; + } + if (!this._hasWarnedVersionBlockThisSession) { + this._hasWarnedVersionBlockThisSession = true; + this.snackService.open({ + type: 'ERROR', + msg: + reason === 'VERSION_UNSUPPORTED' + ? T.F.SYNC.S.VERSION_UNSUPPORTED + : T.F.SYNC.S.VERSION_TOO_OLD, + actionStr: T.PS.UPDATE_APP, + actionFn: () => window.open('https://super-productivity.com/download', '_blank'), + }); + } + } + /** * Applies non-conflicting operations with crash-safe tracking. * diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts index d833226618..2718eeddb4 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts @@ -333,4 +333,131 @@ describe('SyncImportConflictGateService', () => { expect(writeFlushServiceSpy.flushPendingWrites).not.toHaveBeenCalled(); expect(opLogStoreSpy.getUnsynced).toHaveBeenCalled(); }); + + describe('meaningful-work coverage beyond TASK/PROJECT/TAG/NOTE CUD', () => { + it('should treat a pending MOV op as meaningful', async () => { + const pendingMove = createEntry( + createOperation({ + id: 'local-move', + actionType: '[Task] Move task' as ActionType, + opType: OpType.Move, + entityType: 'TASK', + entityId: 'task-1', + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ); + opLogStoreSpy.getUnsynced.and.resolveTo([pendingMove]); + + const result = await service.checkIncomingFullStateConflict([createOperation()]); + + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); + }); + + it('should treat pending non-task entity work (TIME_TRACKING, SIMPLE_COUNTER) as meaningful', async () => { + const pendingTimeTracking = createEntry( + createOperation({ + id: 'local-tt', + actionType: '[TimeTracking] Update whole day' as ActionType, + opType: OpType.Update, + entityType: 'TIME_TRACKING', + entityId: 'tt-1', + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ); + const pendingCounter = createEntry( + createOperation({ + id: 'local-counter', + actionType: '[SimpleCounter] Increase Counter Today' as ActionType, + opType: OpType.Update, + entityType: 'SIMPLE_COUNTER', + entityId: 'counter-1', + clientId: 'client-A', + vectorClock: { clientA: 2 }, + }), + ); + + expect(service.hasMeaningfulPendingOps([pendingTimeTracking])).toBeTrue(); + expect(service.hasMeaningfulPendingOps([pendingCounter])).toBeTrue(); + }); + + it('should still treat MIGRATION/RECOVERY bookkeeping ops as not meaningful', async () => { + const pendingMigration = createEntry( + createOperation({ + id: 'local-genesis', + actionType: '[Migration] Genesis' as ActionType, + opType: OpType.Create, + entityType: 'MIGRATION', + entityId: 'genesis', + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ); + + expect(service.hasMeaningfulPendingOps([pendingMigration])).toBeFalse(); + }); + }); + + describe('preCapturedPendingOps (piggyback-upload race)', () => { + it('should judge meaningfulness against the pre-upload snapshot, not the live pending set', async () => { + // Live pending set is empty — the op was accepted and marked synced during + // the same upload round that piggybacked the import. + opLogStoreSpy.getUnsynced.and.resolveTo([]); + + const acceptedThisRound = createEntry( + createOperation({ + id: 'accepted-op', + actionType: '[Task] Update task' as ActionType, + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-1', + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ); + + const result = await service.checkIncomingFullStateConflict([createOperation()], { + preCapturedPendingOps: [acceptedThisRound], + }); + + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); + }); + + it('should derive discardable example-task ids from the LIVE pending set', async () => { + const exampleCreate = (id: string): OperationLogEntry => + createEntry( + createOperation({ + id, + actionType: ActionType.TASK_SHARED_ADD, + opType: OpType.Create, + entityType: 'TASK', + entityId: `task-${id}`, + payload: { + actionPayload: { + task: { id: `task-${id}` }, + isExampleTask: true, + }, + entityChanges: [], + }, + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ); + + // Pre-captured snapshot has two example ops; one was accepted (synced) + // during the round, so only the still-pending one may be discarded. + const stillPending = exampleCreate('example-still-pending'); + const acceptedAlready = exampleCreate('example-accepted'); + opLogStoreSpy.getUnsynced.and.resolveTo([stillPending]); + + const result = await service.checkIncomingFullStateConflict([createOperation()], { + preCapturedPendingOps: [stillPending, acceptedAlready], + }); + + expect(result.discardablePendingOpIds).toEqual(['example-still-pending']); + }); + }); }); diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index b8013aee3a..bc255685de 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -10,7 +10,16 @@ import { OperationWriteFlushService } from './operation-write-flush.service'; import { SyncImportConflictData } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; -const USER_ENTITY_TYPES = new Set(['TASK', 'PROJECT', 'TAG', 'NOTE']); +/** + * Pending ops on these entity types are startup/system noise, not user work: + * config writes happen automatically (defaults, migrations) and sync settings + * are intentionally local; MIGRATION/RECOVERY are bookkeeping genesis ops. + * Everything else — including MOV/BATCH ops and entities like TIME_TRACKING, + * SIMPLE_COUNTER, TASK_REPEAT_CFG, PLANNER, BOARD — is user work an incoming + * full-state import would silently discard (imports drop concurrent ops by + * design), so it must count as meaningful and trigger the conflict dialog. + */ +const DISCARDABLE_ENTITY_TYPES = new Set(['GLOBAL_CONFIG', 'MIGRATION', 'RECOVERY']); export interface IncomingFullStateConflictGateResult { fullStateOp?: Operation; @@ -36,9 +45,10 @@ export class SyncImportConflictGateService { private writeFlushService = inject(OperationWriteFlushService); /** - * Config-only pending ops are not considered user work for this conflict gate. - * Full-state ops are always meaningful because applying a newer full-state op can - * invalidate their local import/repair semantics. + * Every pending op is user work unless it is on a DISCARDABLE_ENTITY_TYPES + * entity or is an onboarding example-task create. Full-state ops are always + * meaningful because applying a newer full-state op can invalidate their + * local import/repair semantics. */ hasMeaningfulPendingOps(ops: OperationLogEntry[]): boolean { return ops.some((entry) => { @@ -48,18 +58,24 @@ export class SyncImportConflictGateService { if (isExampleTaskCreateOp(entry)) { return false; } - return ( - USER_ENTITY_TYPES.has(entry.op.entityType) && - (entry.op.opType === OpType.Create || - entry.op.opType === OpType.Update || - entry.op.opType === OpType.Delete) - ); + return !DISCARDABLE_ENTITY_TYPES.has(entry.op.entityType); }); } + /** + * @param options.preCapturedPendingOps - Pending ops captured BEFORE the upload + * round started. The piggyback-upload path MUST pass this: by the time + * its gate runs, ops accepted in the same round were already marked + * synced, so a live getUnsynced() read would no longer see local work + * that the piggybacked import is about to discard. + */ async checkIncomingFullStateConflict( incomingOps: Operation[], - options: { flushPendingWrites?: boolean; isNeverSynced?: boolean } = {}, + options: { + flushPendingWrites?: boolean; + isNeverSynced?: boolean; + preCapturedPendingOps?: OperationLogEntry[]; + } = {}, ): Promise { const fullStateOp = incomingOps.find((op) => FULL_STATE_OP_TYPES.has(op.opType)); @@ -78,13 +94,21 @@ export class SyncImportConflictGateService { await this.writeFlushService.flushPendingWrites(); } - const pendingOps = await this.opLogStore.getUnsynced(); + const pendingOps = + options.preCapturedPendingOps ?? (await this.opLogStore.getUnsynced()); const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); // Example-task ops that the caller may reject when it accepts the import silently. // When `hasMeaningfulPending` is true (real work pending alongside example tasks), // the conflict dialog is shown instead and these are intentionally left untouched: // if the user keeps local state, their example tasks ride along with the rest. - const discardablePendingOpIds = pendingOps + // + // These must come from a LIVE read: with a pre-captured snapshot, example ops + // accepted earlier in the same upload round are already marked synced and must + // not be re-marked rejected by the caller. + const livePendingOps = options.preCapturedPendingOps + ? await this.opLogStore.getUnsynced() + : pendingOps; + const discardablePendingOpIds = livePendingOps .filter(isExampleTaskCreateOp) .map((entry) => entry.op.id); diff --git a/src/app/op-log/testing/integration/migration-handling.integration.spec.ts b/src/app/op-log/testing/integration/migration-handling.integration.spec.ts index aaf7460e00..f6d303695c 100644 --- a/src/app/op-log/testing/integration/migration-handling.integration.spec.ts +++ b/src/app/op-log/testing/integration/migration-handling.integration.spec.ts @@ -1,10 +1,7 @@ import { TestBed } from '@angular/core/testing'; import { RemoteOpsProcessingService } from '../../sync/remote-ops-processing.service'; import { OperationLogStoreService } from '../../persistence/operation-log-store.service'; -import { - SchemaMigrationService, - MAX_VERSION_SKIP, -} from '../../persistence/schema-migration.service'; +import { SchemaMigrationService } from '../../persistence/schema-migration.service'; import { SnackService } from '../../../core/snack/snack.service'; import { VectorClockService } from '../../sync/vector-clock.service'; import { OperationApplierService } from '../../apply/operation-applier.service'; @@ -146,28 +143,14 @@ describe('Migration Handling Integration', () => { ); }); - it('should accept operation with compatible future version (within skip limit)', async () => { - // Logic: if opVersion <= current + MAX_VERSION_SKIP, it's accepted - // MAX_VERSION_SKIP is 3. So current + 3 should still be accepted. - // Note: A WARNING snackbar may be shown for newer versions, but no ERROR. - const compatibleVersion = CURRENT_SCHEMA_VERSION + MAX_VERSION_SKIP; - const op = createOp(compatibleVersion); + it('should block operation from any future version (no forward-compat band)', async () => { + // Forward-compatible migrations are not implemented: real migrations + // rename/split fields, so a future op applied verbatim corrupts state. + // Even one version ahead must block until the app is updated. + const futureVersion = CURRENT_SCHEMA_VERSION + 1; + const op = createOp(futureVersion); - await service.processRemoteOps([op]); - - // Should NOT show error (operation is accepted) - expect(snackServiceSpy.open).not.toHaveBeenCalledWith( - jasmine.objectContaining({ type: 'ERROR' }), - ); - expect(operationApplierSpy.applyOperations).toHaveBeenCalled(); - }); - - it('should reject operation with incompatible future version', async () => { - // Logic: if opVersion > current + MAX_VERSION_SKIP, update required - const incompatibleVersion = CURRENT_SCHEMA_VERSION + MAX_VERSION_SKIP + 1; - const op = createOp(incompatibleVersion); - - await service.processRemoteOps([op]); + const result = await service.processRemoteOps([op]); // Should trigger error snackbar expect(snackServiceSpy.open).toHaveBeenCalledWith( @@ -177,8 +160,9 @@ describe('Migration Handling Integration', () => { }), ); - // Should NOT apply operation + // Should NOT apply operation, and callers must hold the cursor expect(operationApplierSpy.applyOperations).not.toHaveBeenCalled(); + expect(result.blockedByIncompatibleOp).toBe(true); }); it('should handle operations missing schemaVersion (default to 1)', async () => { From 6e6ce34c15c90e850c80d331c00f7159ea594a07 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 13:36:52 +0200 Subject: [PATCH 04/47] test(e2e): incoming SYNC_IMPORT must prompt on pending non-task work Regression test for the widened incoming-full-state conflict gate and the piggyback pre-upload pending-snapshot fix: a pending SIMPLE_COUNTER change (non-TASK/PROJECT/TAG/NOTE) must trigger the conflict dialog instead of being silently discarded by an incoming SYNC_IMPORT from another client. Not yet run (SuperSync docker stack is unavailable in this sandbox); run via npm run e2e:supersync:file or the E2E Tests (Scheduled) workflow. --- ...ersync-conflict-gate-non-task-work.spec.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts new file mode 100644 index 0000000000..314f7e08bc --- /dev/null +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -0,0 +1,146 @@ +import { test, expect } from '../../fixtures/supersync.fixture'; +import { + createTestUser, + getSuperSyncConfig, + createSimulatedClient, + closeClient, + type SimulatedE2EClient, +} from '../../utils/supersync-helpers'; +import { ImportPage } from '../../pages/import.page'; + +/** + * SuperSync: incoming SYNC_IMPORT must not silently discard NON-task local work. + * + * Regression test for the widened incoming-full-state conflict gate + the + * piggyback pre-upload pending-snapshot fix (sync-import-conflict-gate.service). + * + * Before the fix the gate only treated CRUD ops on TASK/PROJECT/TAG/NOTE as + * "meaningful", so a pending SIMPLE_COUNTER change (and MOV/BATCH, time + * tracking, planner, boards, ...) was silently overwritten when an incoming + * SYNC_IMPORT from another client arrived — no conflict dialog, no choice. + * The gate now treats every synced entity as user work, and the piggyback path + * judges against the pending set captured BEFORE the upload round (ops accepted + * mid-round are marked synced and would otherwise vanish from a live re-read). + * + * Scenario: + * 1. Client A + B set up sync; A creates a click counter, increments it, syncs. + * 2. Client B syncs and receives the counter (B now has synced history). + * 3. Client B increments the counter locally — a pending SIMPLE_COUNTER op it + * does NOT sync. + * 4. Client A imports a backup (creates a SYNC_IMPORT) and syncs it. + * 5. Client B triggers a sync. The SYNC_IMPORT conflict dialog MUST appear so + * the user can choose, instead of the counter change being discarded. + * + * Run with: npm run e2e:supersync:file e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts + */ + +const createClickCounter = async ( + client: SimulatedE2EClient, + title: string, +): Promise => { + await client.page.goto('/#/habits'); + await client.page.waitForURL(/habits/); + await client.page.waitForTimeout(500); + + const addBtn = client.page.locator('.add-habit-btn'); + await addBtn.waitFor({ state: 'visible', timeout: 10000 }); + await addBtn.click(); + + const dialog = client.page.locator('dialog-simple-counter-edit-settings'); + await dialog.waitFor({ state: 'visible', timeout: 10000 }); + + const titleInput = dialog.locator('formly-form input').first(); + await titleInput.waitFor({ state: 'visible', timeout: 5000 }); + await titleInput.fill(title); + + const typeSelect = dialog.locator('mat-select').first(); + await typeSelect.waitFor({ state: 'visible', timeout: 5000 }); + await client.page.waitForTimeout(500); + await typeSelect.click(); + await client.page.waitForTimeout(300); + await client.page.locator('mat-option:has-text("Click Counter")').click(); + await client.page.waitForTimeout(300); + + await dialog.locator('button[type="submit"]').click(); + await dialog.waitFor({ state: 'hidden', timeout: 10000 }); + await client.page.waitForTimeout(500); + + await client.page.goto('/#/tag/TODAY/tasks'); + await client.page.waitForURL(/(active\/tasks|tag\/TODAY\/tasks)/); + await client.page.waitForTimeout(500); +}; + +const incrementClickCounter = async (client: SimulatedE2EClient): Promise => { + const counter = client.page + .locator('.counters-action-group simple-counter-button') + .last(); + await counter.waitFor({ state: 'visible', timeout: 15000 }); + await counter.locator('.main-btn').click(); + await client.page.waitForTimeout(200); +}; + +test.describe.configure({ mode: 'serial' }); + +test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', () => { + test('a pending simple-counter change prompts the conflict dialog instead of being discarded', async ({ + browser, + baseURL, + testRunId, + }) => { + test.slow(); + + const counterTitle = `GateCounter-${testRunId}-${Date.now()}`; + let clientA: SimulatedE2EClient | null = null; + let clientB: SimulatedE2EClient | null = null; + + try { + const user = await createTestUser(testRunId); + const syncConfig = getSuperSyncConfig(user); + + // ===== PHASE 1: A creates + increments a counter, syncs ===== + clientA = await createSimulatedClient(browser, baseURL!, 'A', testRunId); + await clientA.sync.setupSuperSync(syncConfig); + await createClickCounter(clientA, counterTitle); + await incrementClickCounter(clientA); + await clientA.sync.syncAndWait(); + console.log('[Gate] Client A created + synced a simple counter'); + + // ===== PHASE 2: B joins, receives the counter (now non-fresh) ===== + clientB = await createSimulatedClient(browser, baseURL!, 'B', testRunId); + await clientB.sync.setupSuperSync(syncConfig); + await clientB.sync.syncAndWait(); + await clientB.page.goto('/#/tag/TODAY/tasks'); + await clientB.page.waitForLoadState('networkidle'); + console.log('[Gate] Client B synced and received the counter'); + + // ===== PHASE 3: B makes a pending NON-task change (does NOT sync) ===== + await incrementClickCounter(clientB); + console.log('[Gate] Client B incremented the counter locally (pending, unsynced)'); + + // ===== PHASE 4: A imports a backup (SYNC_IMPORT) and syncs it ===== + const importPageA = new ImportPage(clientA.page); + await importPageA.navigateToImportPage(); + await importPageA.importBackupFile(ImportPage.getFixturePath('test-backup.json')); + await clientA.sync.syncAndWait(); + console.log('[Gate] Client A imported a backup and synced the SYNC_IMPORT'); + + // ===== PHASE 5: B syncs — the conflict dialog MUST appear ===== + // triggerSync() does NOT auto-resolve dialogs, so we can assert on it. + await clientB.sync.triggerSync(); + + await expect(clientB.sync.syncImportConflictDialog).toBeVisible({ timeout: 30000 }); + console.log('[Gate] ✓ Conflict dialog shown for pending non-task work'); + + // Resolving with "Use My Data" keeps B's local work; the point of the test + // is that B was given the choice at all. + await clientB.sync.syncImportUseLocalBtn.click(); + await clientB.sync.syncImportConflictDialog.waitFor({ + state: 'hidden', + timeout: 10000, + }); + } finally { + if (clientA) await closeClient(clientA); + if (clientB) await closeClient(clientB); + } + }); +}); From f38342eeb9d540bea183723051d9a8458c204e6a Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 15:24:51 +0200 Subject: [PATCH 05/47] fix(sync): harden replay and conflict recovery Make remote replay and authoritative state replacement crash-resumable and atomic. Preserve pending changes until piggyback processing commits, and stop cleanly on incompatible operations. Validate sync identities and schemas while strengthening request deduplication and privacy-safe diagnostics. --- ...ersync-conflict-gate-non-task-work.spec.ts | 56 ++- ...sc-to-tasks-settings-migration-v1-to-v2.ts | 2 + .../src/supersync-http-contract.ts | 4 +- .../tests/supersync-http-contract.spec.ts | 27 ++ .../super-sync-server/src/sync/conflict.ts | 28 ++ .../services/operation-download.service.ts | 3 + .../sync/services/operation-upload.service.ts | 29 +- .../services/request-deduplication.service.ts | 59 ++- .../src/sync/services/validation.service.ts | 6 +- .../src/sync/sync.routes.ops-handler.ts | 12 +- .../src/sync/sync.routes.snapshot-handler.ts | 28 +- .../src/sync/sync.service.ts | 26 +- .../tests/operation-download.service.spec.ts | 28 ++ ...quest-dedup-failure-caching.routes.spec.ts | 3 + .../request-deduplication.service.spec.ts | 52 ++- .../tests/sync-compressed-body.routes.spec.ts | 2 + .../tests/sync.service.spec.ts | 34 +- .../tests/validation.service.spec.ts | 10 + packages/sync-core/src/operation.types.ts | 2 +- packages/sync-core/src/ports.ts | 5 +- packages/sync-core/src/remote-apply.ts | 35 +- packages/sync-core/src/replay-coordinator.ts | 3 + packages/sync-core/tests/remote-apply.spec.ts | 34 +- .../tests/replay-coordinator.spec.ts | 5 + .../imex/sync/sync-wrapper.service.spec.ts | 24 ++ src/app/imex/sync/sync-wrapper.service.ts | 21 + .../op-log/apply/operation-applier.service.ts | 1 + .../backup/state-snapshot.service.spec.ts | 27 ++ .../op-log/backup/state-snapshot.service.ts | 19 +- .../operation-capture.meta-reducer.spec.ts | 11 +- .../capture/operation-capture.meta-reducer.ts | 48 ++- .../capture/operation-log.effects.spec.ts | 28 +- .../op-log/capture/operation-log.effects.ts | 27 +- src/app/op-log/core/types/apply.types.ts | 7 + .../op-log/core/types/sync-results.types.ts | 24 +- .../compact/compact-operation.types.ts | 2 +- .../operation-log-hydrator.service.spec.ts | 7 +- .../operation-log-hydrator.service.ts | 17 +- .../operation-log-recovery.service.spec.ts | 20 +- .../operation-log-recovery.service.ts | 12 +- .../operation-log-store.service.spec.ts | 159 +++++++ .../operation-log-store.service.ts | 136 +++++- .../schema-migration.service.spec.ts | 34 ++ .../persistence/schema-migration.service.ts | 24 +- .../sync/conflict-resolution.service.spec.ts | 30 +- .../sync/conflict-resolution.service.ts | 28 +- .../sync/immediate-upload.service.spec.ts | 13 + .../op-log/sync/immediate-upload.service.ts | 8 + .../operation-log-download.service.spec.ts | 45 ++ .../sync/operation-log-download.service.ts | 20 +- .../sync/operation-log-sync.service.spec.ts | 385 ++++++++++++----- .../op-log/sync/operation-log-sync.service.ts | 403 +++++++++++------- .../sync/operation-log-upload.service.spec.ts | 24 ++ .../sync/operation-log-upload.service.ts | 30 +- .../remote-ops-processing.service.spec.ts | 114 ++++- .../sync/remote-ops-processing.service.ts | 50 ++- .../sync/server-migration.service.spec.ts | 70 ++- .../op-log/sync/server-migration.service.ts | 222 +++++----- .../sync-import-conflict-gate.service.spec.ts | 100 ++++- .../sync/sync-import-conflict-gate.service.ts | 57 +-- .../ws-triggered-download.service.spec.ts | 21 +- .../sync/ws-triggered-download.service.ts | 8 + ...emote-apply-store-port.integration.spec.ts | 22 +- 63 files changed, 2170 insertions(+), 621 deletions(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index 314f7e08bc..5c38ccea49 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -40,7 +40,6 @@ const createClickCounter = async ( ): Promise => { await client.page.goto('/#/habits'); await client.page.waitForURL(/habits/); - await client.page.waitForTimeout(500); const addBtn = client.page.locator('.add-habit-btn'); await addBtn.waitFor({ state: 'visible', timeout: 10000 }); @@ -55,28 +54,51 @@ const createClickCounter = async ( const typeSelect = dialog.locator('mat-select').first(); await typeSelect.waitFor({ state: 'visible', timeout: 5000 }); - await client.page.waitForTimeout(500); await typeSelect.click(); - await client.page.waitForTimeout(300); - await client.page.locator('mat-option:has-text("Click Counter")').click(); - await client.page.waitForTimeout(300); + const clickCounterOption = client.page.locator('mat-option:has-text("Click Counter")'); + await clickCounterOption.waitFor({ state: 'visible', timeout: 5000 }); + await clickCounterOption.click(); await dialog.locator('button[type="submit"]').click(); await dialog.waitFor({ state: 'hidden', timeout: 10000 }); - await client.page.waitForTimeout(500); await client.page.goto('/#/tag/TODAY/tasks'); await client.page.waitForURL(/(active\/tasks|tag\/TODAY\/tasks)/); - await client.page.waitForTimeout(500); }; -const incrementClickCounter = async (client: SimulatedE2EClient): Promise => { +const getNamedClickCounter = async ( + client: SimulatedE2EClient, + title: string, +): Promise> => { const counter = client.page .locator('.counters-action-group simple-counter-button') .last(); await counter.waitFor({ state: 'visible', timeout: 15000 }); + + // Counter buttons render only their initial, so verify the exact counter via + // its accessible Material tooltip before interacting with it. + await counter.hover(); + await expect(client.page.getByRole('tooltip').filter({ hasText: title })).toBeVisible(); + return counter; +}; + +const expectClickCounterValue = async ( + client: SimulatedE2EClient, + title: string, + expectedValue: number, +): Promise => { + const counter = await getNamedClickCounter(client, title); + await expect(counter.locator('.label')).toHaveText(String(expectedValue)); +}; + +const incrementClickCounter = async ( + client: SimulatedE2EClient, + title: string, + expectedValue: number, +): Promise => { + const counter = await getNamedClickCounter(client, title); await counter.locator('.main-btn').click(); - await client.page.waitForTimeout(200); + await expect(counter.locator('.label')).toHaveText(String(expectedValue)); }; test.describe.configure({ mode: 'serial' }); @@ -101,7 +123,7 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( clientA = await createSimulatedClient(browser, baseURL!, 'A', testRunId); await clientA.sync.setupSuperSync(syncConfig); await createClickCounter(clientA, counterTitle); - await incrementClickCounter(clientA); + await incrementClickCounter(clientA, counterTitle, 1); await clientA.sync.syncAndWait(); console.log('[Gate] Client A created + synced a simple counter'); @@ -111,10 +133,11 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( await clientB.sync.syncAndWait(); await clientB.page.goto('/#/tag/TODAY/tasks'); await clientB.page.waitForLoadState('networkidle'); + await expectClickCounterValue(clientB, counterTitle, 1); console.log('[Gate] Client B synced and received the counter'); // ===== PHASE 3: B makes a pending NON-task change (does NOT sync) ===== - await incrementClickCounter(clientB); + await incrementClickCounter(clientB, counterTitle, 2); console.log('[Gate] Client B incremented the counter locally (pending, unsynced)'); // ===== PHASE 4: A imports a backup (SYNC_IMPORT) and syncs it ===== @@ -138,6 +161,17 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( state: 'hidden', timeout: 10000, }); + await clientB.sync.waitForSyncToComplete({ timeout: 30000 }); + + // The dialog itself is not the guarantee: prove the exact pending counter + // value won, then run another sync and prove it remains durable. + await clientB.page.goto('/#/tag/TODAY/tasks'); + await clientB.page.waitForURL(/(active\/tasks|tag\/TODAY\/tasks)/); + await expectClickCounterValue(clientB, counterTitle, 2); + + await clientB.sync.syncAndWait(); + await expectClickCounterValue(clientB, counterTitle, 2); + console.log('[Gate] ✓ Pending counter value survived resolution and re-sync'); } finally { if (clientA) await closeClient(clientA); if (clientB) await closeClient(clientB); diff --git a/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts b/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts index 9a3158ba86..9a0d35b9b9 100644 --- a/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts +++ b/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts @@ -97,6 +97,7 @@ export const MiscToTasksSettingsMigration_v1v2: SchemaMigration = { result.push({ ...op, id: `${op.id}_misc`, + entityIds: ['misc'], payload: buildPayload(miscCfg, 'misc'), }); } @@ -106,6 +107,7 @@ export const MiscToTasksSettingsMigration_v1v2: SchemaMigration = { ...op, id: `${op.id}_tasks`, entityId: 'tasks', + entityIds: ['tasks'], payload: buildPayload(tasksCfg, 'tasks'), }); } diff --git a/packages/shared-schema/src/supersync-http-contract.ts b/packages/shared-schema/src/supersync-http-contract.ts index f9ac187e8e..b128400cbf 100644 --- a/packages/shared-schema/src/supersync-http-contract.ts +++ b/packages/shared-schema/src/supersync-http-contract.ts @@ -69,7 +69,7 @@ export const SuperSyncOperationSchema = z.object({ payload: z.unknown(), vectorClock: SuperSyncVectorClockSchema, timestamp: z.number(), - schemaVersion: z.number(), + schemaVersion: z.number().int().min(1).max(100), isPayloadEncrypted: z.boolean().optional(), syncImportReason: z.enum(SUPER_SYNC_IMPORT_REASONS).optional(), }); @@ -92,7 +92,7 @@ export const SuperSyncUploadSnapshotRequestSchema = z.object({ clientId: SuperSyncClientIdSchema, reason: z.enum(SUPER_SYNC_SNAPSHOT_REASONS), vectorClock: SuperSyncVectorClockSchema, - schemaVersion: z.number().optional(), + schemaVersion: z.number().int().min(1).max(100).optional(), isPayloadEncrypted: z.boolean().optional(), syncImportReason: z.enum(SUPER_SYNC_IMPORT_REASONS).optional(), opId: z.string().uuid().optional(), diff --git a/packages/shared-schema/tests/supersync-http-contract.spec.ts b/packages/shared-schema/tests/supersync-http-contract.spec.ts index 65faa56474..c0f725540b 100644 --- a/packages/shared-schema/tests/supersync-http-contract.spec.ts +++ b/packages/shared-schema/tests/supersync-http-contract.spec.ts @@ -72,6 +72,33 @@ describe('SuperSync HTTP contract schemas', () => { ).toThrow(); }); + it.each([0, 1.5, -1, 101])( + 'rejects malformed operation schema version %s', + (schemaVersion) => { + expect(() => + SuperSyncUploadOpsRequestSchema.parse({ + ops: [{ ...createValidOperation(), schemaVersion }], + clientId: 'client_1', + }), + ).toThrow(); + }, + ); + + it.each([0, 1.5, -1, 101])( + 'rejects malformed snapshot schema version %s', + (schemaVersion) => { + expect(() => + SuperSyncUploadSnapshotRequestSchema.parse({ + state: {}, + clientId: 'client_1', + reason: 'recovery', + vectorClock: { client_1: 1 }, + schemaVersion, + }), + ).toThrow(); + }, + ); + it('coerces download query numbers like the route-level schema', () => { const parsed = SuperSyncDownloadOpsQuerySchema.parse({ sinceSeq: '5', diff --git a/packages/super-sync-server/src/sync/conflict.ts b/packages/super-sync-server/src/sync/conflict.ts index 1aad6b9d99..008733516a 100644 --- a/packages/super-sync-server/src/sync/conflict.ts +++ b/packages/super-sync-server/src/sync/conflict.ts @@ -265,6 +265,34 @@ export const isSameDuplicateOperation = ( ); }; +/** Complete identity comparison for two validated operations in one request. */ +export const isSameIncomingOperation = ( + first: Operation, + second: Operation, + firstOriginalTimestamp: number = first.timestamp, + secondOriginalTimestamp: number = second.timestamp, +): boolean => { + const bothEncrypted = + (first.isPayloadEncrypted ?? false) && (second.isPayloadEncrypted ?? false); + return ( + first.clientId === second.clientId && + first.actionType === second.actionType && + first.opType === second.opType && + first.entityType === second.entityType && + first.entityId === second.entityId && + areJsonValuesEqual(getStoredEntityIds(first), getStoredEntityIds(second)) && + (bothEncrypted || areJsonValuesEqual(first.payload, second.payload)) && + areJsonValuesEqual( + limitVectorClockSize(first.vectorClock, [first.clientId]), + limitVectorClockSize(second.vectorClock, [second.clientId]), + ) && + first.schemaVersion === second.schemaVersion && + firstOriginalTimestamp === secondOriginalTimestamp && + (first.isPayloadEncrypted ?? false) === (second.isPayloadEncrypted ?? false) && + (first.syncImportReason ?? null) === (second.syncImportReason ?? null) + ); +}; + export const isSameDuplicateTimestamp = ( existingTimestamp: bigint | number | string, existingReceivedAt: bigint | number | string, diff --git a/packages/super-sync-server/src/sync/services/operation-download.service.ts b/packages/super-sync-server/src/sync/services/operation-download.service.ts index 0a718ddd8c..724608e353 100644 --- a/packages/super-sync-server/src/sync/services/operation-download.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-download.service.ts @@ -21,6 +21,7 @@ const OPERATION_DOWNLOAD_SELECT = { opType: true, entityType: true, entityId: true, + entityIds: true, payload: true, vectorClock: true, schemaVersion: true, @@ -54,6 +55,7 @@ type OperationDownloadRow = { opType: string; entityType: string; entityId: string | null; + entityIds: string[]; payload: unknown; vectorClock: unknown; schemaVersion: number; @@ -72,6 +74,7 @@ const mapOperationRow = (row: OperationDownloadRow): ServerOperation => ({ opType: row.opType as Operation['opType'], entityType: row.entityType, entityId: row.entityId ?? undefined, + entityIds: row.entityIds.length > 0 ? row.entityIds : undefined, payload: row.payload, vectorClock: row.vectorClock as VectorClock, schemaVersion: row.schemaVersion, diff --git a/packages/super-sync-server/src/sync/services/operation-upload.service.ts b/packages/super-sync-server/src/sync/services/operation-upload.service.ts index 00c69cc7b4..550b4e4276 100644 --- a/packages/super-sync-server/src/sync/services/operation-upload.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-upload.service.ts @@ -24,6 +24,7 @@ import { getStoredEntityIds, getEntityConflictKey, isSameDuplicateOperation, + isSameIncomingOperation, prefetchLatestEntityOpsForBatch, pruneVectorClockForStorage, resolveConflictForExistingOp, @@ -317,10 +318,9 @@ export class OperationUploadService { } /** - * Stage 2: within a single batch, accept the first op for an id and reject - * every later op sharing that id as DUPLICATE_OPERATION (by id, not content - * — see plan §1a step 2 / the C4 divergence note). Must run before sequence - * reservation so a duplicate never consumes a server_seq. + * Stage 2: within a single batch, accept the first op for an id. Exact retries + * are DUPLICATE_OPERATION; same-id operations with different complete identity + * are INVALID_OP_ID. Must run before sequence reservation. */ private rejectIntraBatchDuplicates( userId: number, @@ -328,20 +328,31 @@ export class OperationUploadService { validatedCandidates: BatchUploadCandidate[], results: UploadResult[], ): BatchUploadCandidate[] { - const seenOpIds = new Set(); + const firstCandidateByOpId = new Map(); const uniqueCandidates: BatchUploadCandidate[] = []; for (const candidate of validatedCandidates) { - if (seenOpIds.has(candidate.op.id)) { + const firstCandidate = firstCandidateByOpId.get(candidate.op.id); + if (firstCandidate) { + const isExactRetry = isSameIncomingOperation( + firstCandidate.op, + candidate.op, + firstCandidate.originalTimestamp, + candidate.originalTimestamp, + ); results[candidate.resultIndex] = this.rejectedUploadResult( userId, clientId, candidate.op, - 'Duplicate operation ID', - SYNC_ERROR_CODES.DUPLICATE_OPERATION, + isExactRetry + ? 'Duplicate operation ID' + : 'Operation ID already belongs to a different operation', + isExactRetry + ? SYNC_ERROR_CODES.DUPLICATE_OPERATION + : SYNC_ERROR_CODES.INVALID_OP_ID, ); continue; } - seenOpIds.add(candidate.op.id); + firstCandidateByOpId.set(candidate.op.id, candidate); uniqueCandidates.push(candidate); } return uniqueCandidates; diff --git a/packages/super-sync-server/src/sync/services/request-deduplication.service.ts b/packages/super-sync-server/src/sync/services/request-deduplication.service.ts index 6c45ca235f..2ea3b1e496 100644 --- a/packages/super-sync-server/src/sync/services/request-deduplication.service.ts +++ b/packages/super-sync-server/src/sync/services/request-deduplication.service.ts @@ -8,6 +8,9 @@ * Entries expire after 5 minutes. */ import type { UploadResult } from '../sync.types'; +import type { Operation } from '../sync.types'; +import { createHash } from 'node:crypto'; +import { stableJsonStringify } from '../conflict'; /** * Maximum entries in deduplication cache to prevent unbounded memory growth. @@ -38,8 +41,18 @@ export interface SnapshotDedupResponse { * even if the same `requestId` string is reused across namespaces. */ type RequestDeduplicationEntry = - | { namespace: 'ops'; processedAt: number; results: UploadResult[] } - | { namespace: 'snapshot'; processedAt: number; results: SnapshotDedupResponse }; + | { + namespace: 'ops'; + processedAt: number; + results: UploadResult[]; + fingerprint?: string; + } + | { + namespace: 'snapshot'; + processedAt: number; + results: SnapshotDedupResponse; + fingerprint?: string; + }; type DedupPayload = N extends 'ops' ? UploadResult[] @@ -56,6 +69,7 @@ export class RequestDeduplicationService { userId: number, namespace: N, requestId: string, + fingerprint?: string, ): DedupPayload | null { const key = this._key(userId, namespace, requestId); const entry = this.cache.get(key); @@ -66,6 +80,7 @@ export class RequestDeduplicationService { } // Defensive: keying already isolates namespaces, but verify before casting. if (entry.namespace !== namespace) return null; + if (fingerprint !== undefined && entry.fingerprint !== fingerprint) return null; return entry.results as DedupPayload; } @@ -77,6 +92,7 @@ export class RequestDeduplicationService { namespace: N, requestId: string, results: DedupPayload, + fingerprint?: string, ): void { const key = this._key(userId, namespace, requestId); if (this.cache.size >= MAX_CACHE_SIZE) { @@ -88,6 +104,7 @@ export class RequestDeduplicationService { namespace, processedAt: Date.now(), results, + fingerprint, } as RequestDeduplicationEntry); } @@ -138,3 +155,41 @@ export class RequestDeduplicationService { return `${userId}:${namespace}:${requestId}`; } } + +export const createOpsRequestFingerprint = ( + clientId: string, + ops: Operation[], +): string => { + const logicalOps = ops.map((op) => ({ + ...op, + payload: op.isPayloadEncrypted ? '[encrypted-payload]' : op.payload, + })); + return createHash('sha256') + .update(stableJsonStringify({ clientId, ops: logicalOps })) + .digest('base64url'); +}; + +export interface SnapshotRequestFingerprintInput { + state: unknown; + clientId: string; + reason: string; + vectorClock: Record; + schemaVersion?: number; + isPayloadEncrypted?: boolean; + syncImportReason?: string; + opId?: string; + isCleanSlate?: boolean; + snapshotOpType?: string; +} + +export const createSnapshotRequestFingerprint = ( + request: SnapshotRequestFingerprintInput, +): string => { + const logicalRequest = { + ...request, + state: request.isPayloadEncrypted ? '[encrypted-state]' : request.state, + }; + return createHash('sha256') + .update(stableJsonStringify(logicalRequest)) + .digest('base64url'); +}; diff --git a/packages/super-sync-server/src/sync/services/validation.service.ts b/packages/super-sync-server/src/sync/services/validation.service.ts index 382467ee12..971bece805 100644 --- a/packages/super-sync-server/src/sync/services/validation.service.ts +++ b/packages/super-sync-server/src/sync/services/validation.service.ts @@ -158,7 +158,11 @@ export class ValidationService { }; } if (op.schemaVersion !== undefined) { - if (op.schemaVersion < 1 || op.schemaVersion > 100) { + if ( + !Number.isInteger(op.schemaVersion) || + op.schemaVersion < 1 || + op.schemaVersion > 100 + ) { return { valid: false, error: `Invalid schema version: ${op.schemaVersion}`, diff --git a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts index 0024759d59..24ed6eed35 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts @@ -30,6 +30,7 @@ import { getRawOpsCount, sendOpsBatchTooLargeReply, } from './sync.routes.quota'; +import { createOpsRequestFingerprint } from './services/request-deduplication.service'; export const uploadOpsHandler = async ( req: FastifyRequest<{ Body: UploadOpsRequest }>, @@ -86,6 +87,9 @@ export const uploadOpsHandler = async ( } const { ops, clientId, lastKnownServerSeq, requestId } = parseResult.data; + const requestFingerprint = requestId + ? createOpsRequestFingerprint(clientId, ops as unknown as Operation[]) + : undefined; const syncService = getSyncService(); Logger.info( @@ -112,7 +116,11 @@ export const uploadOpsHandler = async ( // This ensures retries after successful uploads don't fail with 413 // if the original upload pushed the user over quota if (requestId) { - const cachedResults = syncService.checkOpsRequestDedup(userId, requestId); + const cachedResults = syncService.checkOpsRequestDedup( + userId, + requestId, + requestFingerprint, + ); if (cachedResults) { Logger.info(`[user:${userId}] Returning cached results for request ${requestId}`); @@ -214,7 +222,7 @@ export const uploadOpsHandler = async ( requestId && !results.some((r) => r.errorCode === SYNC_ERROR_CODES.INTERNAL_ERROR) ) { - syncService.cacheOpsRequestResults(userId, requestId, results); + syncService.cacheOpsRequestResults(userId, requestId, results, requestFingerprint); } const accepted = results.filter((r) => r.accepted).length; diff --git a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts index 2c01f0daa8..8f59879597 100644 --- a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts @@ -29,6 +29,7 @@ import { sendQuotaExceededReply, sendSyncImportExistsReply, } from './sync.routes.quota'; +import { createSnapshotRequestFingerprint } from './services/request-deduplication.service'; export const uploadSnapshotHandler = async ( req: FastifyRequest<{ Body: unknown }>, @@ -99,9 +100,27 @@ export const uploadSnapshotHandler = async ( requestId, } = snapshotRequest; const syncService = getSyncService(); + const requestFingerprint = requestId + ? createSnapshotRequestFingerprint({ + state, + clientId, + reason, + vectorClock, + schemaVersion, + isPayloadEncrypted, + syncImportReason, + opId, + isCleanSlate, + snapshotOpType, + }) + : undefined; if (requestId) { - const cachedResponse = syncService.checkSnapshotRequestDedup(userId, requestId); + const cachedResponse = syncService.checkSnapshotRequestDedup( + userId, + requestId, + requestFingerprint, + ); if (cachedResponse) { Logger.info( `[user:${userId}] Returning cached snapshot result for request ${requestId}`, @@ -340,7 +359,12 @@ export const uploadSnapshotHandler = async ( finalResult.errorCode !== SYNC_ERROR_CODES.DUPLICATE_OPERATION && finalResult.errorCode !== SYNC_ERROR_CODES.INTERNAL_ERROR ) { - syncService.cacheSnapshotRequestResult(userId, requestId, responseBody); + syncService.cacheSnapshotRequestResult( + userId, + requestId, + responseBody, + requestFingerprint, + ); } return reply.send(responseBody); diff --git a/packages/super-sync-server/src/sync/sync.service.ts b/packages/super-sync-server/src/sync/sync.service.ts index 9ac22cd7ca..d4ac70c58d 100644 --- a/packages/super-sync-server/src/sync/sync.service.ts +++ b/packages/super-sync-server/src/sync/sync.service.ts @@ -454,26 +454,44 @@ export class SyncService { return this.rateLimitService.cleanupExpiredCounters(); } - checkOpsRequestDedup(userId: number, requestId: string): UploadResult[] | null { - return this.requestDeduplicationService.checkDeduplication(userId, 'ops', requestId); + checkOpsRequestDedup( + userId: number, + requestId: string, + fingerprint?: string, + ): UploadResult[] | null { + return this.requestDeduplicationService.checkDeduplication( + userId, + 'ops', + requestId, + fingerprint, + ); } cacheOpsRequestResults( userId: number, requestId: string, results: UploadResult[], + fingerprint?: string, ): void { - this.requestDeduplicationService.cacheResults(userId, 'ops', requestId, results); + this.requestDeduplicationService.cacheResults( + userId, + 'ops', + requestId, + results, + fingerprint, + ); } checkSnapshotRequestDedup( userId: number, requestId: string, + fingerprint?: string, ): SnapshotDedupResponse | null { return this.requestDeduplicationService.checkDeduplication( userId, 'snapshot', requestId, + fingerprint, ); } @@ -481,12 +499,14 @@ export class SyncService { userId: number, requestId: string, response: SnapshotDedupResponse, + fingerprint?: string, ): void { this.requestDeduplicationService.cacheResults( userId, 'snapshot', requestId, response, + fingerprint, ); } diff --git a/packages/super-sync-server/tests/operation-download.service.spec.ts b/packages/super-sync-server/tests/operation-download.service.spec.ts index c53c2e667a..7c71cb113e 100644 --- a/packages/super-sync-server/tests/operation-download.service.spec.ts +++ b/packages/super-sync-server/tests/operation-download.service.spec.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import type { Prisma } from '@prisma/client'; import { OperationDownloadService } from '../src/sync/services/operation-download.service'; // Mock prisma @@ -35,6 +36,7 @@ const EXPECTED_OPERATION_DOWNLOAD_SELECT = { opType: true, entityType: true, entityId: true, + entityIds: true, payload: true, vectorClock: true, schemaVersion: true, @@ -54,6 +56,7 @@ const createMockOpRow = ( actionType: string; entityType: string; entityId: string | null; + entityIds: string[]; payload: unknown; vectorClock: Record; schemaVersion: number; @@ -71,6 +74,7 @@ const createMockOpRow = ( entityType: overrides.entityType ?? 'Task', // Use 'in' check to allow null to be explicitly set entityId: 'entityId' in overrides ? overrides.entityId : `task-${serverSeq}`, + entityIds: overrides.entityIds ?? [], payload: overrides.payload ?? { title: `Task ${serverSeq}` }, vectorClock: overrides.vectorClock ?? { [clientId]: serverSeq }, schemaVersion: overrides.schemaVersion ?? 1, @@ -358,6 +362,30 @@ describe('OperationDownloadService', () => { ); }); + it('should round-trip batch entityIds in downloaded operations', async () => { + vi.mocked(prisma.$transaction).mockImplementation( + async (fn: (tx: Prisma.TransactionClient) => Promise) => + fn({ + operation: { + findFirst: vi.fn().mockResolvedValue(null), + findMany: vi.fn().mockResolvedValue([ + createMockOpRow(1, 'batch-client', { + entityId: 'task-1', + entityIds: ['task-1', 'task-2'], + }), + ]), + }, + userSyncState: { + findUnique: vi.fn().mockResolvedValue({ lastSeq: 1 }), + }, + } as unknown as Prisma.TransactionClient), + ); + + const result = await service.getOpsSinceWithSeq(1, 0); + + expect(result.ops[0].op.entityIds).toEqual(['task-1', 'task-2']); + }); + it('should detect gap when client is ahead of server', async () => { vi.mocked(prisma.$transaction).mockImplementation(async (fn: any) => { const mockTx = { diff --git a/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts b/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts index fae2d6bac2..862ad0ebe7 100644 --- a/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts +++ b/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts @@ -196,6 +196,7 @@ describe('Request dedup — transaction-failure results are not cached (#8332)', userId, 'ops-v1-success', results, + expect.any(String), ); }); @@ -218,6 +219,7 @@ describe('Request dedup — transaction-failure results are not cached (#8332)', userId, 'ops-v1-conflict', results, + expect.any(String), ); }); }); @@ -245,6 +247,7 @@ describe('Request dedup — transaction-failure results are not cached (#8332)', userId, 'snapshot-v1-success', { accepted: true, serverSeq: 5, error: undefined }, + expect.any(String), ); }); }); diff --git a/packages/super-sync-server/tests/request-deduplication.service.spec.ts b/packages/super-sync-server/tests/request-deduplication.service.spec.ts index a4cfa87dec..41acd782df 100644 --- a/packages/super-sync-server/tests/request-deduplication.service.spec.ts +++ b/packages/super-sync-server/tests/request-deduplication.service.spec.ts @@ -1,5 +1,8 @@ import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; -import { RequestDeduplicationService } from '../src/sync/services/request-deduplication.service'; +import { + RequestDeduplicationService, + createSnapshotRequestFingerprint, +} from '../src/sync/services/request-deduplication.service'; import { UploadResult } from '../src/sync/sync.types'; describe('RequestDeduplicationService', () => { @@ -36,6 +39,18 @@ describe('RequestDeduplicationService', () => { expect(result).toEqual(mockResults); }); + it('should treat the same requestId with a different body fingerprint as a cache miss', () => { + const mockResults = createMockResults(1); + service.cacheResults(1, 'ops', 'request-123', mockResults, 'fingerprint-a'); + + expect( + service.checkDeduplication(1, 'ops', 'request-123', 'fingerprint-a'), + ).toEqual(mockResults); + expect( + service.checkDeduplication(1, 'ops', 'request-123', 'fingerprint-b'), + ).toBeNull(); + }); + it('should return null for expired request', () => { const mockResults = createMockResults(1); service.cacheResults(1, 'ops', 'request-123', mockResults); @@ -114,6 +129,41 @@ describe('RequestDeduplicationService', () => { }); }); + describe('snapshot request fingerprints', () => { + const request = { + state: { task: { ids: ['task-1'] } }, + clientId: 'client-1', + reason: 'recovery', + vectorClock: { 'client-1': 1 }, + schemaVersion: 4, + }; + + it('changes when an unencrypted snapshot body changes', () => { + expect(createSnapshotRequestFingerprint(request)).not.toBe( + createSnapshotRequestFingerprint({ + ...request, + state: { task: { ids: ['task-2'] } }, + }), + ); + }); + + it('stays stable across encrypted snapshot IV changes', () => { + expect( + createSnapshotRequestFingerprint({ + ...request, + state: 'ciphertext-a', + isPayloadEncrypted: true, + }), + ).toBe( + createSnapshotRequestFingerprint({ + ...request, + state: 'ciphertext-b', + isPayloadEncrypted: true, + }), + ); + }); + }); + describe('cleanupExpiredEntries', () => { it('should remove all expired entries', () => { service.cacheResults(1, 'ops', 'request-1', createMockResults(1)); diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index fd60cfd4a2..60d282be76 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -810,6 +810,7 @@ describe('Sync compressed body routes', () => { expect(mocks.syncService.checkSnapshotRequestDedup).toHaveBeenCalledWith( 1, 'snapshot-v1-retry', + expect.any(String), ); expect(mocks.syncService.checkOpsRequestDedup).not.toHaveBeenCalled(); expect(mocks.prisma.operation.findFirst).not.toHaveBeenCalled(); @@ -879,6 +880,7 @@ describe('Sync compressed body routes', () => { 1, 'snapshot-v1-dup-test', { accepted: true, serverSeq: 77 }, + expect.any(String), ); }); diff --git a/packages/super-sync-server/tests/sync.service.spec.ts b/packages/super-sync-server/tests/sync.service.spec.ts index 7a46555be9..bcb192276a 100644 --- a/packages/super-sync-server/tests/sync.service.spec.ts +++ b/packages/super-sync-server/tests/sync.service.spec.ts @@ -708,15 +708,7 @@ describe('SyncService', () => { expect(testState.operations.size).toBe(25); }); - it('rejects an intra-batch same-id op as DUPLICATE_OPERATION even when its content differs (deliberate divergence from the legacy per-op path, which yields INVALID_OP_ID)', async () => { - // C4: the batch path dedups intra-batch purely by op.id (plan §1a step 2) - // and rejects the later op as DUPLICATE_OPERATION regardless of content. - // The legacy per-op path inserts the first then catches the second at the - // DB and returns INVALID_OP_ID for differing content. Both are terminal - // rejections with no row and no sequence gap, so sync invariants hold; - // the client treats DUPLICATE_OPERATION as a silent success and - // INVALID_OP_ID as a hard rejection. This test pins the chosen batch - // semantics so the divergence stays intentional, not accidental drift. + it('rejects an intra-batch same-id collision as INVALID_OP_ID', async () => { const service = new SyncService({ batchUpload: true }); const opId = uuidv7(); const first = makeOp({ @@ -747,7 +739,7 @@ describe('SyncService', () => { expect(results[1]).toEqual( expect.objectContaining({ accepted: false, - errorCode: SYNC_ERROR_CODES.DUPLICATE_OPERATION, + errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, }), ); // No sequence gap: lastSeq advanced by exactly 1, exactly one row. @@ -755,6 +747,28 @@ describe('SyncService', () => { expect(testState.operations.size).toBe(1); }); + it('preserves an exact intra-batch retry as DUPLICATE_OPERATION', async () => { + const service = new SyncService({ batchUpload: true }); + const retry = makeOp({ + id: uuidv7(), + entityId: 'task-1', + entityIds: ['task-1', 'task-2'], + vectorClock: { [clientId]: 1 }, + }); + + const results = await service.uploadOps(userId, clientId, [retry, { ...retry }]); + + expect(results[0]).toEqual( + expect.objectContaining({ accepted: true, serverSeq: 1 }), + ); + expect(results[1]).toEqual( + expect.objectContaining({ + accepted: false, + errorCode: SYNC_ERROR_CODES.DUPLICATE_OPERATION, + }), + ); + }); + it('should reject intra-batch entity conflicts in order', async () => { const service = new SyncService({ batchUpload: true }); const ops: Operation[] = [ diff --git a/packages/super-sync-server/tests/validation.service.spec.ts b/packages/super-sync-server/tests/validation.service.spec.ts index b2e65c4c31..f98939be11 100644 --- a/packages/super-sync-server/tests/validation.service.spec.ts +++ b/packages/super-sync-server/tests/validation.service.spec.ts @@ -315,6 +315,16 @@ describe('ValidationService', () => { expect(result.errorCode).toBe(SYNC_ERROR_CODES.INVALID_SCHEMA_VERSION); }); + it('should reject non-integer schema versions', () => { + const result = validationService.validateOp( + createValidOp({ schemaVersion: 1.5 }), + clientId, + ); + + expect(result.valid).toBe(false); + expect(result.errorCode).toBe(SYNC_ERROR_CODES.INVALID_SCHEMA_VERSION); + }); + it('should accept valid schema versions 1-100', () => { for (const schemaVersion of [1, 50, 100]) { const op = createValidOp({ schemaVersion }); diff --git a/packages/sync-core/src/operation.types.ts b/packages/sync-core/src/operation.types.ts index ceb37f8edb..67e35f4ee1 100644 --- a/packages/sync-core/src/operation.types.ts +++ b/packages/sync-core/src/operation.types.ts @@ -151,7 +151,7 @@ export interface OperationLogEntry = Operat * For remote ops only: tracks whether the op was successfully applied to local state. * Used for crash recovery: on startup, any 'pending' or 'failed' remote ops are re-dispatched. */ - applicationStatus?: 'pending' | 'applied' | 'failed'; + applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed'; /** * For 'failed' ops: number of retry attempts. diff --git a/packages/sync-core/src/ports.ts b/packages/sync-core/src/ports.ts index 3aac4c98d3..76396607a4 100644 --- a/packages/sync-core/src/ports.ts +++ b/packages/sync-core/src/ports.ts @@ -20,7 +20,10 @@ export interface SyncActionLike { export interface OperationApplyPort = Operation> { applyOperations( ops: TOperation[], - options?: ApplyOperationsOptions, + options?: ApplyOperationsOptions & { + /** Persist reducer-commit bookkeeping before post-dispatch side effects start. */ + onReducersCommitted?: (ops: TOperation[]) => Promise; + }, ): Promise>; } diff --git a/packages/sync-core/src/remote-apply.ts b/packages/sync-core/src/remote-apply.ts index 2546770de6..3cc8f32f77 100644 --- a/packages/sync-core/src/remote-apply.ts +++ b/packages/sync-core/src/remote-apply.ts @@ -24,6 +24,7 @@ export interface RemoteOperationApplyStorePort< source: 'remote', options: { pendingApply: true }, ): Promise>; + markArchivePending(seqs: number[]): Promise; markApplied(seqs: number[]): Promise; markFailed(opIds: string[]): Promise; mergeRemoteOpClocks(ops: TOperation[]): Promise; @@ -111,7 +112,25 @@ export const applyRemoteOperations = async < } }); - const applyResult = await applier.applyOperations(appendResult.writtenOps); + const applyResult = await applier.applyOperations(appendResult.writtenOps, { + onReducersCommitted: async (reducerCommittedOps) => { + const reducerCommittedSeqs = reducerCommittedOps + .map((op) => opIdToSeq.get(op.id)) + .filter((seq): seq is number => seq !== undefined); + if (reducerCommittedSeqs.length !== reducerCommittedOps.length) { + throw new Error( + 'applyRemoteOperations: reducer commit contained an operation outside the appended batch.', + ); + } + await store.markArchivePending(reducerCommittedSeqs); + await store.mergeRemoteOpClocks(reducerCommittedOps); + }, + }); + if (applyResult.failedOp && !opIdToSeq.has(applyResult.failedOp.op.id)) { + throw new Error( + `applyRemoteOperations: applier reported unknown failed op ${applyResult.failedOp.op.id}.`, + ); + } const appliedSeqs = applyResult.appliedOps .map((op) => opIdToSeq.get(op.id)) .filter((seq): seq is number => seq !== undefined); @@ -120,7 +139,6 @@ export const applyRemoteOperations = async < if (appliedSeqs.length > 0) { await store.markApplied(appliedSeqs); - await store.mergeRemoteOpClocks(applyResult.appliedOps); const appliedFullStateOpIds = applyResult.appliedOps .filter(isFullStateOperation) @@ -132,9 +150,7 @@ export const applyRemoteOperations = async < } } - const failedOpIds = applyResult.failedOp - ? getFailedOpIds(appendResult.writtenOps, applyResult.failedOp.op) - : []; + const failedOpIds = applyResult.failedOp ? [applyResult.failedOp.op.id] : []; if (failedOpIds.length > 0) { await store.markFailed(failedOpIds); @@ -150,12 +166,3 @@ export const applyRemoteOperations = async < failedOpIds, }; }; - -const getFailedOpIds = >( - opsToApply: TOperation[], - failedOp: TOperation, -): string[] => { - const failedOpIndex = opsToApply.findIndex((op) => op.id === failedOp.id); - const failedOps = failedOpIndex === -1 ? [failedOp] : opsToApply.slice(failedOpIndex); - return failedOps.map((op) => op.id); -}; diff --git a/packages/sync-core/src/replay-coordinator.ts b/packages/sync-core/src/replay-coordinator.ts index 567c5374af..57792e3d47 100644 --- a/packages/sync-core/src/replay-coordinator.ts +++ b/packages/sync-core/src/replay-coordinator.ts @@ -35,6 +35,7 @@ export interface OperationReplayCoordinatorOptions< operationToAction?: (op: TOperation) => TReplayAction; isArchiveAffectingAction?: (action: TReplayAction) => boolean; onRemoteArchiveDataApplied?: () => void; + onReducersCommitted?: (ops: TOperation[]) => Promise; onArchiveSideEffectError?: ( context: OperationReplayArchiveFailureContext, ) => void; @@ -94,6 +95,7 @@ export const replayOperationBatch = async < operationToAction, isArchiveAffectingAction = () => false, onRemoteArchiveDataApplied, + onReducersCommitted, onArchiveSideEffectError, onPostSyncCooldownError, yieldToEventLoop: waitForEventLoop = yieldToEventLoop, @@ -118,6 +120,7 @@ export const replayOperationBatch = async < if (!skipReducerDispatch) { dispatcher.dispatch(createBulkApplyAction(ops)); await waitForEventLoop(); + await onReducersCommitted?.(ops); } if (!isLocalHydration && archiveSideEffects !== undefined && operationToAction) { diff --git a/packages/sync-core/tests/remote-apply.spec.ts b/packages/sync-core/tests/remote-apply.spec.ts index b2aca4f786..9aa8e29a06 100644 --- a/packages/sync-core/tests/remote-apply.spec.ts +++ b/packages/sync-core/tests/remote-apply.spec.ts @@ -22,6 +22,7 @@ const createStore = (appendResult: { skippedCount: number; }): RemoteOperationApplyStorePort> => ({ appendBatchSkipDuplicates: vi.fn().mockResolvedValue(appendResult), + markArchivePending: vi.fn().mockResolvedValue(undefined), markApplied: vi.fn().mockResolvedValue(undefined), markFailed: vi.fn().mockResolvedValue(undefined), mergeRemoteOpClocks: vi.fn().mockResolvedValue(undefined), @@ -31,7 +32,10 @@ const createStore = (appendResult: { const createApplier = ( result: Awaited>['applyOperations']>>, ): OperationApplyPort> => ({ - applyOperations: vi.fn().mockResolvedValue(result), + applyOperations: vi.fn(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return result; + }), }); describe('applyRemoteOperations', () => { @@ -50,7 +54,11 @@ describe('applyRemoteOperations', () => { expect(store.appendBatchSkipDuplicates).toHaveBeenCalledWith([op1, op2], 'remote', { pendingApply: true, }); - expect(applier.applyOperations).toHaveBeenCalledWith([op2]); + expect(applier.applyOperations).toHaveBeenCalledWith( + [op2], + jasmineLikeObjectContainingFunction(), + ); + expect(store.markArchivePending).toHaveBeenCalledWith([10]); expect(store.markApplied).toHaveBeenCalledWith([10]); expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([op2]); expect(result).toEqual({ @@ -104,7 +112,7 @@ describe('applyRemoteOperations', () => { expect(result.clearedFullStateOpCount).toBe(3); }); - it('marks the failed op and remaining unapplied ops as failed', async () => { + it('checkpoints the whole reducer batch but charges only the attempted archive failure', async () => { const op1 = createOperation('op-1'); const op2 = createOperation('op-2'); const op3 = createOperation('op-3'); @@ -126,13 +134,14 @@ describe('applyRemoteOperations', () => { }); expect(store.markApplied).toHaveBeenCalledWith([1]); - expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([op1]); - expect(store.markFailed).toHaveBeenCalledWith(['op-2', 'op-3']); + expect(store.markArchivePending).toHaveBeenCalledWith([1, 2, 3]); + expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([op1, op2, op3]); + expect(store.markFailed).toHaveBeenCalledWith(['op-2']); expect(result.failedOp).toEqual({ op: op2, error }); - expect(result.failedOpIds).toEqual(['op-2', 'op-3']); + expect(result.failedOpIds).toEqual(['op-2']); }); - it('marks only the reported failed op if it is not in the appended batch', async () => { + it('rejects an applier result whose failed op is not in the appended batch', async () => { const op1 = createOperation('op-1'); const unknownFailedOp = createOperation('unknown-failed-op'); const store = createStore({ seqs: [1], writtenOps: [op1], skippedCount: 0 }); @@ -141,9 +150,12 @@ describe('applyRemoteOperations', () => { failedOp: { op: unknownFailedOp, error: new Error('unexpected') }, }); - const result = await applyRemoteOperations({ ops: [op1], store, applier }); - - expect(store.markFailed).toHaveBeenCalledWith(['unknown-failed-op']); - expect(result.failedOpIds).toEqual(['unknown-failed-op']); + await expect(applyRemoteOperations({ ops: [op1], store, applier })).rejects.toThrow( + 'unknown-failed-op', + ); + expect(store.markFailed).not.toHaveBeenCalled(); }); }); + +const jasmineLikeObjectContainingFunction = (): object => + expect.objectContaining({ onReducersCommitted: expect.any(Function) }); diff --git a/packages/sync-core/tests/replay-coordinator.spec.ts b/packages/sync-core/tests/replay-coordinator.spec.ts index a94dc95621..0b866de933 100644 --- a/packages/sync-core/tests/replay-coordinator.spec.ts +++ b/packages/sync-core/tests/replay-coordinator.spec.ts @@ -69,6 +69,9 @@ describe('replayOperationBatch', () => { const onRemoteArchiveDataApplied = vi.fn(() => { callOrder.push('remoteArchiveDataApplied'); }); + const onReducersCommitted = vi.fn(async () => { + callOrder.push('reducersCommitted'); + }); const result = await replayOperationBatch({ ops, @@ -87,6 +90,7 @@ describe('replayOperationBatch', () => { }), isArchiveAffectingAction: (action) => action.archiveAffecting === true, onRemoteArchiveDataApplied, + onReducersCommitted, yieldToEventLoop, }); @@ -101,6 +105,7 @@ describe('replayOperationBatch', () => { 'startApplyingRemoteOps', 'dispatchBulk', 'yield', + 'reducersCommitted', 'archive:op-1', 'archive:op-2', 'yield', diff --git a/src/app/imex/sync/sync-wrapper.service.spec.ts b/src/app/imex/sync/sync-wrapper.service.spec.ts index 01562c659f..cc09584fdf 100644 --- a/src/app/imex/sync/sync-wrapper.service.spec.ts +++ b/src/app/imex/sync/sync-wrapper.service.spec.ts @@ -583,6 +583,19 @@ describe('SyncWrapperService', () => { expect(mockProviderManager.setLastSyncedProviderId).not.toHaveBeenCalled(); }); + + it('should stop with ERROR when download is blocked by an incompatible op', async () => { + mockSyncService.downloadRemoteOps.and.resolveTo({ + kind: 'blocked_incompatible', + }); + + const result = await service.sync(); + + expect(result).toBe('HANDLED_ERROR'); + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockProviderManager.setLastSyncedProviderId).not.toHaveBeenCalled(); + expect(mockSyncService.uploadPendingOps).not.toHaveBeenCalled(); + }); }); describe('_sync() - Sync flow', () => { @@ -1000,6 +1013,17 @@ describe('SyncWrapperService', () => { expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('IN_SYNC'); }); + it('should stop with ERROR when upload piggyback is blocked by an incompatible op', async () => { + mockSyncService.uploadPendingOps.and.resolveTo({ + kind: 'blocked_incompatible', + }); + + const result = await service.sync(); + + expect(result).toBe('HANDLED_ERROR'); + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + }); + it('should set IN_SYNC when permanentRejectionCount is 0 even with empty rejectedOps array', async () => { mockSyncService.uploadPendingOps.and.returnValue( Promise.resolve({ diff --git a/src/app/imex/sync/sync-wrapper.service.ts b/src/app/imex/sync/sync-wrapper.service.ts index d98d8e213f..0815ca6cbf 100644 --- a/src/app/imex/sync/sync-wrapper.service.ts +++ b/src/app/imex/sync/sync-wrapper.service.ts @@ -506,6 +506,13 @@ export class SyncWrapperService { this._providerManager.setSyncStatus('UNKNOWN_OR_CHANGED'); return 'HANDLED_ERROR'; } + if (downloadResult.kind === 'blocked_incompatible') { + SyncLog.warn( + 'SyncWrapperService: Download blocked by an incompatible operation.', + ); + this._providerManager.setSyncStatus('ERROR'); + return 'HANDLED_ERROR'; + } // Track the successfully synced provider for switch detection on next sync this._providerManager.setLastSyncedProviderId(providerId); @@ -515,6 +522,13 @@ export class SyncWrapperService { syncCapableProvider, { isNeverSynced: isNeverSyncedAtSyncStart }, ); + if (uploadResult.kind === 'blocked_incompatible') { + SyncLog.warn( + 'SyncWrapperService: Upload piggyback blocked by an incompatible operation.', + ); + this._providerManager.setSyncStatus('ERROR'); + return 'HANDLED_ERROR'; + } const completedUploadResults: CompletedUploadOutcome[] = uploadResult.kind === 'completed' ? [uploadResult] : []; if (uploadResult.kind === 'completed') { @@ -582,6 +596,13 @@ export class SyncWrapperService { this._providerManager.setSyncStatus('UNKNOWN_OR_CHANGED'); return 'HANDLED_ERROR'; } + if (reuploadResult.kind === 'blocked_incompatible') { + SyncLog.warn( + 'SyncWrapperService: LWW re-upload blocked by an incompatible operation.', + ); + this._providerManager.setSyncStatus('ERROR'); + return 'HANDLED_ERROR'; + } if (reuploadResult.kind === 'completed') { completedUploadResults.push(reuploadResult); } diff --git a/src/app/op-log/apply/operation-applier.service.ts b/src/app/op-log/apply/operation-applier.service.ts index 4f0dc8b4c1..186b235e4c 100644 --- a/src/app/op-log/apply/operation-applier.service.ts +++ b/src/app/op-log/apply/operation-applier.service.ts @@ -129,6 +129,7 @@ export class OperationApplierService implements OperationApplyPort { // this action is now disabled to prevent UI freezes during bulk archive sync. this.store.dispatch(remoteArchiveDataApplied()); }, + onReducersCommitted: options.onReducersCommitted, onArchiveSideEffectError: ({ op, processedCount, error }) => { OpLog.err( `OperationApplierService: Failed archive handling for operation ${op.id}. ` + diff --git a/src/app/op-log/backup/state-snapshot.service.spec.ts b/src/app/op-log/backup/state-snapshot.service.spec.ts index c0eddc9437..0421722ee5 100644 --- a/src/app/op-log/backup/state-snapshot.service.spec.ts +++ b/src/app/op-log/backup/state-snapshot.service.spec.ts @@ -20,6 +20,7 @@ import { selectPluginMetadataFeatureState } from '../../plugins/store/plugin-met import { selectReminderFeatureState } from '../../features/reminder/store/reminder.reducer'; import { ArchiveModel } from '../../features/time-tracking/time-tracking.model'; import { initialTimeTrackingState } from '../../features/time-tracking/store/time-tracking.reducer'; +import { TaskState } from '../../features/tasks/task.model'; describe('StateSnapshotService', () => { let service: StateSnapshotService; @@ -225,6 +226,32 @@ describe('StateSnapshotService', () => { expect(archiveDbAdapterSpy.loadArchiveYoung).toHaveBeenCalledTimes(1); expect(archiveDbAdapterSpy.loadArchiveOld).toHaveBeenCalledTimes(1); }); + + it('should capture NgRx state before awaiting archive I/O', async () => { + let releaseArchives!: () => void; + const archiveGate = new Promise((resolve) => { + releaseArchives = resolve; + }); + archiveDbAdapterSpy.loadArchiveYoung.and.callFake(async () => { + await archiveGate; + return mockArchiveYoung; + }); + archiveDbAdapterSpy.loadArchiveOld.and.callFake(async () => { + await archiveGate; + return mockArchiveOld; + }); + + const snapshotPromise = service.getStateSnapshotAsync(); + store.overrideSelector(selectTaskFeatureState, { + ...mockTaskState, + ids: ['later-task'], + } as unknown as TaskState); + store.refreshState(); + releaseArchives(); + + const snapshot = await snapshotPromise; + expect((snapshot.task as TaskState).ids).toEqual(['task1']); + }); }); describe('backward compatibility aliases', () => { diff --git a/src/app/op-log/backup/state-snapshot.service.ts b/src/app/op-log/backup/state-snapshot.service.ts index 119b92232d..ce3f125622 100644 --- a/src/app/op-log/backup/state-snapshot.service.ts +++ b/src/app/op-log/backup/state-snapshot.service.ts @@ -1,6 +1,5 @@ import { inject, Injectable } from '@angular/core'; import { Selector, Store } from '@ngrx/store'; -import { combineLatest, firstValueFrom } from 'rxjs'; import { first } from 'rxjs/operators'; import { selectBoardsState } from '../../features/boards/store/boards.selectors'; @@ -135,24 +134,18 @@ export class StateSnapshotService { * ⚠️ Returns live store references — never mutate (clone first). See class doc. */ async getStateSnapshotAsync(): Promise { + // Capture NgRx synchronously before the first await. Callers that hold the + // operation-log barrier can now establish an exact reducer-state cutoff: + // actions dispatched while archive I/O is pending are not half-included in + // the snapshot and will be persisted after it. + const ngRxData = this._getNgRxDataSync(); const [archiveYoung, archiveOld] = await Promise.all([ this._loadArchive('archiveYoung'), this._loadArchive('archiveOld'), ]); - const ngRxValues = await firstValueFrom( - combineLatest(SNAPSHOT_SELECTORS.map((s) => this._store.select(s.selector))).pipe( - first(), - ), - ); - - const result: Partial> = {}; - for (let i = 0; i < SNAPSHOT_SELECTORS.length; i++) { - result[SNAPSHOT_SELECTORS[i].key] = ngRxValues[i]; - } - return { - ...this._normalizeNgRxData(result), + ...ngRxData, archiveYoung, archiveOld, }; diff --git a/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts b/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts index 21d9b4d1ea..44a57dc990 100644 --- a/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts +++ b/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts @@ -249,7 +249,7 @@ describe('operationCaptureMetaReducer', () => { expect(buffered).toEqual([]); }); - it('should clear buffer after returning actions', () => { + it('should return a non-destructive snapshot until actions are acknowledged', () => { const action = createMockAction(); bufferDeferredAction(action); @@ -257,7 +257,7 @@ describe('operationCaptureMetaReducer', () => { const secondCall = getDeferredActions(); expect(firstCall).toEqual([action]); - expect(secondCall).toEqual([]); + expect(secondCall).toEqual([action]); }); }); @@ -329,7 +329,7 @@ describe('operationCaptureMetaReducer', () => { expect(getDeferredActions()).toEqual(actions); }); - it('should drop the oldest action only past the pathological hard cap', () => { + it('should retain every accepted action past the pathological warning cap', () => { const confirmSpy = spyNativeDialogs(); const actions = createManyActions(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP); const dropErrorCalls = (): unknown[][] => @@ -345,9 +345,8 @@ describe('operationCaptureMetaReducer', () => { expect(dropErrorCalls().length).toBe(1); const buffered = getDeferredActions(); - expect(buffered.length).toBe(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP); - // Oldest action was dropped, remaining order intact - expect(buffered[0]).toBe(actions[1]); + expect(buffered.length).toBe(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP + 1); + expect(buffered[0]).toBe(actions[0]); expect(buffered[buffered.length - 1]).toBe(extraAction); }); }); diff --git a/src/app/op-log/capture/operation-capture.meta-reducer.ts b/src/app/op-log/capture/operation-capture.meta-reducer.ts index 31a49d714c..eefd5476ee 100644 --- a/src/app/op-log/capture/operation-capture.meta-reducer.ts +++ b/src/app/op-log/capture/operation-capture.meta-reducer.ts @@ -148,13 +148,12 @@ const DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD = 10; export const DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD = 100; /** - * Pathological hard cap for the deferred actions buffer — purely a memory - * backstop against a runaway dispatch loop. ~100 buffered actions are + * Pathological high-water mark for the deferred actions buffer. ~100 buffered actions are * plausibly reachable by a real user during a stuck/very long remote-apply * window, so we must never drop there (see reload-warning threshold above, - * which tells the user to reload long before this cap can be hit through - * real interaction). Only past this cap is drop-oldest the lesser evil vs - * unbounded memory growth. + * which tells the user to reload long before this mark can be hit through + * real interaction). Crossing it logs loudly but still cannot discard an + * action whose reducer change has already been accepted. * Exported for tests. */ export const DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP = 5000; @@ -164,19 +163,15 @@ export const DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP = 5000; * Called by the meta-reducer when a persistent action arrives during sync. */ export const bufferDeferredAction = (action: PersistentAction): void => { - // Pathological cap only: dropping an accepted persistent action means its - // state mutation survives in NgRx while no operation will ever represent - // it → permanent, unsyncable divergence. So this drop exists solely as a - // memory backstop against a runaway dispatch loop, never as flow control. - // NOTE: The shifted action remains in deferredActionSet (WeakSet has no - // delete-by-value). The effect filters it via isDeferredAction(), and - // getDeferredActions() won't return it, so it is silently lost. + // Never drop an accepted persistent action: its reducer mutation already + // exists in NgRx, so removal here would create permanent unsyncable state. + // The cap remains a loud runaway-loop diagnostic, not a lossy flow-control + // mechanism. if (deferredActions.length >= DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP) { devError( `[operationCaptureMetaReducer] Deferred actions buffer exceeded pathological cap of ${DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP} items. ` + - `Dropping oldest action - this local change will NOT sync. Likely a runaway dispatch loop; reload the app.`, + 'Retaining all actions to preserve sync correctness. Likely a runaway dispatch loop; reload the app.', ); - deferredActions.shift(); } deferredActions.push(action); @@ -195,25 +190,32 @@ export const bufferDeferredAction = (action: PersistentAction): void => { }; /** - * Gets and clears the deferred actions buffer. - * Called after sync completes to process buffered actions. + * Returns a stable snapshot of the deferred actions buffer. Entries remain queued + * until their operation is durably written and explicitly acknowledged. */ export const getDeferredActions = (): PersistentAction[] => { - const actions = deferredActions; - deferredActions = []; - return actions; + return [...deferredActions]; +}; + +export const acknowledgeDeferredAction = (action: PersistentAction): void => { + const index = deferredActions.indexOf(action); + if (index === -1) { + return; + } + deferredActions.splice(index, 1); + deferredActionSet.delete(action); }; /** * Clears the deferred actions buffer without processing. * Used for cleanup during testing or error recovery. * - * Note: deferredActionSet (WeakSet) is not cleared here because WeakSet has - * no .clear() method. Entries are garbage-collected when action references are - * released. In practice, cleared actions are not reused by reference, so stale - * entries in the WeakSet are harmless. + * WeakSet has no clear(), so delete the known buffered references individually. */ export const clearDeferredActions = (): void => { + for (const action of deferredActions) { + deferredActionSet.delete(action); + } deferredActions = []; }; diff --git a/src/app/op-log/capture/operation-log.effects.spec.ts b/src/app/op-log/capture/operation-log.effects.spec.ts index 59a88e40c9..b8881bb659 100644 --- a/src/app/op-log/capture/operation-log.effects.spec.ts +++ b/src/app/op-log/capture/operation-log.effects.spec.ts @@ -16,6 +16,7 @@ import { COMPACTION_THRESHOLD } from '../core/operation-log.const'; import { bufferDeferredAction, clearDeferredActions, + getDeferredActions, } from './operation-capture.meta-reducer'; import { ClientIdService } from '../../core/util/client-id.service'; import { OperationCaptureService } from './operation-capture.service'; @@ -876,8 +877,31 @@ describe('OperationLogEffects', () => { // Should not throw - errors are logged but don't stop processing await expectAsync(effects.processDeferredActions()).toBeResolved(); - // Both actions should have been attempted - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(2); + // The failed write is retried once, then processing continues to action 2. + expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(3); + }); + + it('should keep an exhausted deferred write queued while acknowledging later successes', async () => { + const failedAction = createPersistentAction(ActionType.TASK_SHARED_ADD); + const successfulAction = createPersistentAction(ActionType.TASK_SHARED_UPDATE); + bufferDeferredAction(failedAction); + bufferDeferredAction(successfulAction); + let attempts = 0; + mockOpLogStore.appendWithVectorClockUpdate.and.callFake(() => { + attempts++; + return attempts <= 3 + ? Promise.reject(new Error('persistent failure')) + : Promise.resolve(1); + }); + + await effects.processDeferredActions(); + + expect(getDeferredActions()).toEqual([failedAction]); + + mockOpLogStore.appendWithVectorClockUpdate.and.resolveTo(2); + await effects.processDeferredActions(); + + expect(getDeferredActions()).toEqual([]); }); it('should use fresh vector clock for deferred actions', async () => { diff --git a/src/app/op-log/capture/operation-log.effects.ts b/src/app/op-log/capture/operation-log.effects.ts index a7097573fa..8b68de3511 100644 --- a/src/app/op-log/capture/operation-log.effects.ts +++ b/src/app/op-log/capture/operation-log.effects.ts @@ -28,7 +28,11 @@ import { import { CURRENT_SCHEMA_VERSION } from '../persistence/schema-migration.service'; import { OperationCaptureService } from './operation-capture.service'; import { ImmediateUploadService } from '../sync/immediate-upload.service'; -import { getDeferredActions, isDeferredAction } from './operation-capture.meta-reducer'; +import { + acknowledgeDeferredAction, + getDeferredActions, + isDeferredAction, +} from './operation-capture.meta-reducer'; import { ClientIdService } from '../../core/util/client-id.service'; import { SuperSyncStatusService } from '../sync/super-sync-status.service'; @@ -181,6 +185,9 @@ export class OperationLogEffects implements DeferredLocalActionsPort { devError( `[OperationLogEffects] Action ${action.type} has invalid entityId/entityIds (${action.meta.entityId}) - skipping persistence`, ); + if (isDeferredWrite) { + throw new Error(`Deferred action ${action.type} has invalid entity identifiers.`); + } return; } @@ -292,6 +299,9 @@ export class OperationLogEffects implements DeferredLocalActionsPort { window.location.reload(); }, }); + if (isDeferredWrite) { + throw new Error(`Deferred action ${action.type} has an invalid payload.`); + } return; // Skip persisting invalid operation } @@ -378,10 +388,12 @@ export class OperationLogEffects implements DeferredLocalActionsPort { this.notifyUserAndTriggerRollback(); } else { await this.handleQuotaExceeded(action, isDeferredWrite, options); + return; } } else { this.notifyUserAndTriggerRollback(); } + throw e; } } @@ -512,6 +524,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // Use lock for cross-tab coordination - only one tab handles quota at a time let bailReason: Error | null = null; + let recovered = false; await this.lockService.request('sp_quota_exceeded', async () => { if (options.callerHoldsOperationLogLock) { OpLog.err( @@ -541,6 +554,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // extractEntityChanges() inside writeOperation, so the retry simply // re-extracts the same changes. Pass isDeferredWrite through unchanged. await this.writeOperation(action, isDeferredWrite, options); + recovered = true; this.snackService.open({ type: 'SUCCESS', msg: T.F.SYNC.S.STORAGE_RECOVERED_AFTER_COMPACTION, @@ -562,6 +576,10 @@ export class OperationLogEffects implements DeferredLocalActionsPort { if (bailReason !== null) { throw bailReason; } + if (recovered) { + return; + } + throw new Error('Storage quota recovery failed; operation was not persisted.'); } private showStorageQuotaExceededError(): void { @@ -646,8 +664,13 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // Log error after all retries exhausted, continue processing remaining actions OpLog.err( `OperationLogEffects: Failed to process deferred action after ${MAX_RETRIES} retries`, - { actionType: action.type, error: lastError }, + { + actionType: action.type, + errorName: (lastError as Error | undefined)?.name, + }, ); + } else { + acknowledgeDeferredAction(action); } } diff --git a/src/app/op-log/core/types/apply.types.ts b/src/app/op-log/core/types/apply.types.ts index 7c2e72f24b..d27fc3cf2c 100644 --- a/src/app/op-log/core/types/apply.types.ts +++ b/src/app/op-log/core/types/apply.types.ts @@ -41,4 +41,11 @@ export interface ApplyOperationsOptions { * increaseSimpleCounterCounterToday. */ skipReducerDispatch?: boolean; + + /** + * Called after the bulk reducer dispatch commits and before archive side effects. + * Remote apply uses this to persist its reducer/archive checkpoint and merge the + * causal frontier before deferred local actions can be written. + */ + onReducersCommitted?: (ops: Operation[]) => Promise; } diff --git a/src/app/op-log/core/types/sync-results.types.ts b/src/app/op-log/core/types/sync-results.types.ts index f684fe756b..1ef8e4ea7b 100644 --- a/src/app/op-log/core/types/sync-results.types.ts +++ b/src/app/op-log/core/types/sync-results.types.ts @@ -1,4 +1,4 @@ -import { Operation, VectorClock } from '../operation.types'; +import { Operation, OperationLogEntry, VectorClock } from '../operation.types'; import { OperationSyncProviderMode } from '../../sync-providers/provider.interface'; /** @@ -106,6 +106,13 @@ export interface UploadResult { piggybackedOps: Operation[]; rejectedCount: number; rejectedOps: RejectedOpInfo[]; + /** Exact in-lock pending set considered by this upload round. */ + selectedPendingOps?: OperationLogEntry[]; + /** + * Accepted/local-only operation sequences whose acknowledgement was deliberately + * deferred until the caller has resolved and applied piggybacked operations. + */ + pendingAcknowledgementSeqs?: number[]; /** * Number of local-win update ops created during LWW conflict resolution. * These ops need to be uploaded to propagate local state to other clients. @@ -171,6 +178,13 @@ export interface UploadOptions { */ preUploadCallback?: () => Promise; + /** + * Return accepted sequence numbers to the caller instead of marking them synced + * immediately. Required when piggybacked full-state operations may need user + * resolution before the upload round is considered committed locally. + */ + deferAcknowledgement?: boolean; + /** * If true, instructs server to delete all existing user data before accepting uploaded operations. * Used for clean slate operations like encryption password changes or full imports. @@ -254,6 +268,10 @@ export type DownloadOutcome = | { /** User cancelled a SYNC_IMPORT conflict dialog. */ kind: 'cancelled'; + } + | { + /** Processing stopped at an op this app version cannot interpret safely. */ + kind: 'blocked_incompatible'; }; /** @@ -284,4 +302,8 @@ export type UploadOutcome = | { /** User cancelled a piggybacked SYNC_IMPORT conflict dialog. */ kind: 'cancelled'; + } + | { + /** Piggyback processing stopped at an incompatible operation. */ + kind: 'blocked_incompatible'; }; diff --git a/src/app/op-log/persistence/compact/compact-operation.types.ts b/src/app/op-log/persistence/compact/compact-operation.types.ts index d51976ced4..52ee6eba6a 100644 --- a/src/app/op-log/persistence/compact/compact-operation.types.ts +++ b/src/app/op-log/persistence/compact/compact-operation.types.ts @@ -65,6 +65,6 @@ export interface CompactOperationLogEntry { source: 'local' | 'remote'; syncedAt?: number; rejectedAt?: number; - applicationStatus?: 'pending' | 'applied' | 'failed'; + applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed'; retryCount?: number; } diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts index 6c77d85278..308be9126e 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts @@ -1450,7 +1450,7 @@ describe('OperationLogHydratorService', () => { expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([40, 41, 42]); }); - it('should mark the failed op and every op after it (in seq order) as still-failing', async () => { + it('should charge retry budget only to the attempted archive failure', async () => { const entries = [ failedEntry(40, 'op-a'), failedEntry(41, 'op-b'), @@ -1469,10 +1469,9 @@ describe('OperationLogHydratorService', () => { await service.retryFailedRemoteOps(); expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([40]); - // op-b (failed) and op-c (after it) are both marked failed, with the - // retry-cap so a permanently-failing op is eventually rejected. + // op-c remains archive-pending and has not consumed a retry attempt. expect(mockOpLogStore.markFailed).toHaveBeenCalledWith( - ['op-b', 'op-c'], + ['op-b'], MAX_CONFLICT_RETRY_ATTEMPTS, ); }); diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.ts b/src/app/op-log/persistence/operation-log-hydrator.service.ts index e35345841e..e972905af4 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.ts @@ -22,7 +22,6 @@ import { ValidateStateService } from '../validation/validate-state.service'; import { OperationApplierService } from '../apply/operation-applier.service'; import { HydrationStateService } from '../apply/hydration-state.service'; import { bulkApplyOperations } from '../apply/bulk-hydration.action'; -import { getFailedOpIdsFromBatch } from '../apply/failed-op-ids.util'; import { VectorClockService } from '../sync/vector-clock.service'; import { MAX_CONFLICT_RETRY_ATTEMPTS } from '../core/operation-log.const'; import { AppDataComplete } from '../model/model-config'; @@ -629,22 +628,20 @@ export class OperationLogHydratorService { ); } - // On a partial failure the batch applier stops at the first op whose archive - // side effect throws and returns it; that op and every op after it in seq - // order stay unapplied (slice-from-failure, shared with the primary path). - // markFailed bumps the retry count (rejecting ops past - // MAX_CONFLICT_RETRY_ATTEMPTS), so a permanently-failing op can't be retried - // forever. + // On a partial failure the batch applier stops at the first archive error. + // Charge only that attempted operation: successors remain archive-pending + // without consuming retry budget and will run after the blocker succeeds or + // becomes terminally rejected. if (result.failedOp) { - const stillFailedOpIds = getFailedOpIdsFromBatch(opsToApply, result.failedOp.op); + const failedOpIds = [result.failedOp.op.id]; OpLog.warn( `OperationLogHydratorService: Failed to retry op ${result.failedOp.op.id}`, result.failedOp.error, ); - await this.opLogStore.markFailed(stillFailedOpIds, MAX_CONFLICT_RETRY_ATTEMPTS); + await this.opLogStore.markFailed(failedOpIds, MAX_CONFLICT_RETRY_ATTEMPTS); OpLog.warn( - `OperationLogHydratorService: ${stillFailedOpIds.length} ops still failing after retry`, + 'OperationLogHydratorService: Archive operation still failing after retry', ); } } diff --git a/src/app/op-log/persistence/operation-log-recovery.service.spec.ts b/src/app/op-log/persistence/operation-log-recovery.service.spec.ts index 61be77bb5f..76a9bc50fa 100644 --- a/src/app/op-log/persistence/operation-log-recovery.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-recovery.service.spec.ts @@ -26,6 +26,7 @@ describe('OperationLogRecoveryService', () => { 'getPendingRemoteOps', 'markRejected', 'markApplied', + 'markArchivePending', 'getUnsynced', ]); mockOpLogStore.setVectorClock.and.resolveTo(undefined); @@ -183,22 +184,23 @@ describe('OperationLogRecoveryService', () => { await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); + expect(mockOpLogStore.markArchivePending).not.toHaveBeenCalled(); expect(mockOpLogStore.markRejected).not.toHaveBeenCalled(); }); - it('should mark valid pending ops as applied', async () => { + it('should mark valid crash-interrupted ops as archive-pending', async () => { const now = Date.now(); const pendingOps = [ { seq: 1, op: { id: 'op1' }, appliedAt: now - 1000, source: 'remote' }, { seq: 2, op: { id: 'op2' }, appliedAt: now - 2000, source: 'remote' }, ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markApplied.and.resolveTo(undefined); + mockOpLogStore.markArchivePending.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([1, 2]); + expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1, 2]); + expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); }); it('should reject ops that exceed PENDING_OPERATION_EXPIRY_MS', async () => { @@ -213,12 +215,12 @@ describe('OperationLogRecoveryService', () => { }, // Expired ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markApplied.and.resolveTo(undefined); + mockOpLogStore.markArchivePending.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([1]); + expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1]); expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['expired']); }); @@ -235,7 +237,7 @@ describe('OperationLogRecoveryService', () => { await service.recoverPendingRemoteOps(); expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['old1', 'old2']); - expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); + expect(mockOpLogStore.markArchivePending).not.toHaveBeenCalled(); }); it('should handle mixed valid and expired ops correctly', async () => { @@ -257,12 +259,12 @@ describe('OperationLogRecoveryService', () => { }, ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markApplied.and.resolveTo(undefined); + mockOpLogStore.markArchivePending.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([1, 3]); + expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1, 3]); expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['expired1', 'expired2']); }); }); diff --git a/src/app/op-log/persistence/operation-log-recovery.service.ts b/src/app/op-log/persistence/operation-log-recovery.service.ts index 0c66bc0daf..47d40c8020 100644 --- a/src/app/op-log/persistence/operation-log-recovery.service.ts +++ b/src/app/op-log/persistence/operation-log-recovery.service.ts @@ -136,8 +136,10 @@ export class OperationLogRecoveryService { /** * Recovers from pending remote ops that were stored but not applied (crash recovery). - * These ops are in the log and will be replayed during normal hydration, so we just - * need to mark them as applied to prevent them appearing as orphaned. + * These ops are replayed through reducers during normal hydration, but a crash may + * have happened before their archive side effects completed. Move them to the + * archive-pending checkpoint so hydration retries archive work without double-applying + * reducers. * * Operations pending for longer than PENDING_OPERATION_EXPIRY_MS are considered * superseded (likely due to data corruption or repeated failures) and are rejected @@ -169,13 +171,13 @@ export class OperationLogRecoveryService { ); } - // Mark valid ops as applied - they'll be replayed during normal hydration + // Reducers are replayed status-blind during hydration; archive work is retried after. if (validOps.length > 0) { const seqs = validOps.map((e) => e.seq); - await this.opLogStore.markApplied(seqs); + await this.opLogStore.markArchivePending(seqs); OpLog.warn( `OperationLogRecoveryService: Found ${validOps.length} pending remote ops from previous crash. ` + - `Marking as applied (they will be replayed during hydration).`, + 'Marking archive work pending (reducers will replay during hydration).', ); } diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index b505812618..2f1da31953 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -30,6 +30,7 @@ import { } from '../core/operation-log.const'; import { IndexedDBOpenError } from '../core/errors/indexed-db-open.error'; import { FULL_STATE_OPS_META_KEY, SINGLETON_KEY, STORE_NAMES } from './db-keys.const'; +import { ArchiveModel } from '../../features/time-tracking/time-tracking.model'; describe('OperationLogStoreService', () => { let service: OperationLogStoreService; @@ -1423,6 +1424,20 @@ describe('OperationLogStoreService', () => { }); describe('markApplied', () => { + it('should checkpoint reducer-committed operations as archive-pending', async () => { + const op = createTestOperation(); + const seq = await service.append(op, 'remote', { pendingApply: true }); + + await service.markArchivePending([seq]); + + const [stored] = await service.getOpsAfterSeq(0); + expect(stored.applicationStatus).toBe('archive_pending'); + expect((await service.getPendingRemoteOps()).length).toBe(0); + expect((await service.getFailedRemoteOps()).map((entry) => entry.op.id)).toEqual([ + op.id, + ]); + }); + it('should update applicationStatus from pending to applied', async () => { const op = createTestOperation(); const seq = await service.append(op, 'remote', { pendingApply: true }); @@ -1478,6 +1493,17 @@ describe('OperationLogStoreService', () => { expect(afterMarkApplied[0].applicationStatus).toBe('applied'); }); + it('should update applicationStatus from archive-pending to applied', async () => { + const op = createTestOperation(); + const seq = await service.append(op, 'remote', { pendingApply: true }); + await service.markArchivePending([seq]); + + await service.markApplied([seq]); + + const [stored] = await service.getOpsAfterSeq(0); + expect(stored.applicationStatus).toBe('applied'); + }); + it('should remove failed ops from getFailedRemoteOps after markApplied is called', async () => { // Create an op and mark it as pending const op = createTestOperation(); @@ -2301,6 +2327,139 @@ describe('OperationLogStoreService', () => { }); }); + describe('runRemoteStateReplacement', () => { + const createArchive = (taskId: string): ArchiveModel => + ({ + task: { + ids: [taskId], + entities: { [taskId]: { id: taskId, title: taskId } }, + }, + timeTracking: { project: {}, tag: {} }, + lastTimeTrackingFlush: 0, + }) as unknown as ArchiveModel; + + it('atomically replaces ops, cache, clock, metadata, and both archives', async () => { + await service.append( + createTestOperation({ + opType: OpType.SyncImport, + entityType: 'ALL' as EntityType, + }), + ); + const baselineState = { task: { ids: [], entities: {} } }; + const archiveYoung = createArchive('remote-young'); + const archiveOld = createArchive('remote-old'); + + await service.runRemoteStateReplacement({ + baselineState, + vectorClock: { remote: 4 }, + schemaVersion: 4, + snapshotEntityKeys: ['TASK:remote-task'], + archiveYoung, + archiveOld, + }); + + expect(await service.getOpsAfterSeq(0)).toEqual([]); + expect(await service.getLatestFullStateOpEntry()).toBeUndefined(); + expect(await service.loadStateCache()).toEqual( + jasmine.objectContaining({ + state: baselineState, + lastAppliedOpSeq: 0, + vectorClock: { remote: 4 }, + schemaVersion: 4, + snapshotEntityKeys: ['TASK:remote-task'], + }), + ); + expect(await service.getVectorClock()).toEqual({ remote: 4 }); + + const db = ( + service as unknown as { + db: IDBPDatabase; + } + ).db; + expect((await db.get(STORE_NAMES.ARCHIVE_YOUNG, SINGLETON_KEY)).data).toEqual( + archiveYoung, + ); + expect((await db.get(STORE_NAMES.ARCHIVE_OLD, SINGLETON_KEY)).data).toEqual( + archiveOld, + ); + }); + + it('rolls back every store if one archive write fails', async () => { + const priorOp = createTestOperation({ entityId: 'prior-task' }); + const priorState = { sentinel: 'prior-state' }; + const priorYoung = createArchive('prior-young'); + const priorOld = createArchive('prior-old'); + await service.append(priorOp); + await service.saveStateCache({ + state: priorState, + lastAppliedOpSeq: 1, + vectorClock: { testClient: 1 }, + compactedAt: Date.now(), + }); + await service.setVectorClock({ testClient: 1 }); + + const db = ( + service as unknown as { + db: IDBPDatabase; + } + ).db; + await db.put(STORE_NAMES.ARCHIVE_YOUNG, { + id: SINGLETON_KEY, + data: priorYoung, + lastModified: 1, + }); + await db.put(STORE_NAMES.ARCHIVE_OLD, { + id: SINGLETON_KEY, + data: priorOld, + lastModified: 1, + }); + + const realTransaction = db.transaction.bind(db); + spyOn(db, 'transaction').and.callFake((( + stores: Parameters[0], + mode: Parameters[1], + ) => { + const tx = realTransaction(stores, mode); + if (Array.isArray(stores) && stores.includes(STORE_NAMES.ARCHIVE_OLD)) { + const realObjectStore = tx.objectStore.bind(tx); + tx.objectStore = ((storeName: string) => { + const store = realObjectStore(storeName); + if (storeName === STORE_NAMES.ARCHIVE_OLD) { + store.put = async () => { + throw new Error('Simulated archive write failure'); + }; + } + return store; + }) as typeof tx.objectStore; + } + return tx; + }) as typeof db.transaction); + + await expectAsync( + service.runRemoteStateReplacement({ + baselineState: { sentinel: 'new-state' }, + vectorClock: { remote: 2 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('new-young'), + archiveOld: createArchive('new-old'), + }), + ).toBeRejected(); + + expect((await service.getOpsAfterSeq(0)).map((entry) => entry.op.id)).toEqual([ + priorOp.id, + ]); + expect((await service.loadStateCache())!.state).toEqual(priorState); + expect(await service.getVectorClock()).toEqual({ testClient: 1 }); + expect((await db.get(STORE_NAMES.ARCHIVE_YOUNG, SINGLETON_KEY)).data).toEqual( + priorYoung, + ); + expect((await db.get(STORE_NAMES.ARCHIVE_OLD, SINGLETON_KEY)).data).toEqual( + priorOld, + ); + }); + }); + describe('appendWithVectorClockUpdate', () => { it('should append operation and update vector clock atomically for local ops', async () => { const op = createTestOperation({ diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index 3e88d9e685..0db53f6a65 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -90,7 +90,7 @@ interface StoredOperationLogEntry { source: 'local' | 'remote'; syncedAt?: number; rejectedAt?: number; - applicationStatus?: 'pending' | 'applied' | 'failed'; + applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed'; retryCount?: number; } @@ -679,6 +679,24 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + await this._ensureInit(); + await this._adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { + for (const seq of seqs) { + const entry = await tx.get(STORE_NAMES.OPS, seq); + if (entry?.applicationStatus === 'pending') { + entry.applicationStatus = 'archive_pending'; + await tx.put(STORE_NAMES.OPS, entry); + } + } + }); + } + /** * Marks operations as successfully applied. * Called after remote operations have been dispatched to NgRx. @@ -689,11 +707,12 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { for (const seq of seqs) { const entry = await tx.get(STORE_NAMES.OPS, seq); - // Allow transitioning from 'pending' or 'failed' to 'applied' - // 'failed' ops can be retried and need to be cleared when successful + // Failed/archive-pending ops can be retried and cleared when successful. if ( entry && - (entry.applicationStatus === 'pending' || entry.applicationStatus === 'failed') + (entry.applicationStatus === 'pending' || + entry.applicationStatus === 'archive_pending' || + entry.applicationStatus === 'failed') ) { entry.applicationStatus = 'applied'; await tx.put(STORE_NAMES.OPS, entry); @@ -1100,20 +1119,31 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { await this._ensureInit(); let storedEntries: StoredOperationLogEntry[]; try { - // Exact compound-key match expressed as a degenerate [k, k] range. - storedEntries = await this._adapter.getAllFromIndex( - STORE_NAMES.OPS, - OPS_INDEXES.BY_SOURCE_AND_STATUS, - { lower: ['remote', 'failed'], upper: ['remote', 'failed'] }, - ); + const [archivePendingEntries, failedEntries] = await Promise.all([ + this._adapter.getAllFromIndex( + STORE_NAMES.OPS, + OPS_INDEXES.BY_SOURCE_AND_STATUS, + { + lower: ['remote', 'archive_pending'], + upper: ['remote', 'archive_pending'], + }, + ), + this._adapter.getAllFromIndex( + STORE_NAMES.OPS, + OPS_INDEXES.BY_SOURCE_AND_STATUS, + { lower: ['remote', 'failed'], upper: ['remote', 'failed'] }, + ), + ]); + storedEntries = [...archivePendingEntries, ...failedEntries]; } catch (e) { // Fallback for databases created before version 3 index migration Log.warn( @@ -1121,7 +1151,10 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort(STORE_NAMES.OPS); storedEntries = allOps.filter( - (entry) => entry.source === 'remote' && entry.applicationStatus === 'failed', + (entry) => + entry.source === 'remote' && + (entry.applicationStatus === 'archive_pending' || + entry.applicationStatus === 'failed'), ); } // Decode and filter out rejected ops @@ -1504,6 +1537,83 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + await this._ensureInit(); + + const now = Date.now(); + try { + await this._adapter.transaction( + [ + STORE_NAMES.OPS, + STORE_NAMES.META, + STORE_NAMES.STATE_CACHE, + STORE_NAMES.VECTOR_CLOCK, + STORE_NAMES.ARCHIVE_YOUNG, + STORE_NAMES.ARCHIVE_OLD, + ], + 'readwrite', + async (tx) => { + await tx.clear(STORE_NAMES.OPS); + await tx.put( + STORE_NAMES.META, + buildFullStateOpsMeta([]), + FULL_STATE_OPS_META_KEY, + ); + await tx.put(STORE_NAMES.STATE_CACHE, { + id: SINGLETON_KEY, + state: opts.baselineState, + lastAppliedOpSeq: 0, + vectorClock: opts.vectorClock, + compactedAt: now, + schemaVersion: opts.schemaVersion, + snapshotEntityKeys: opts.snapshotEntityKeys, + }); + await tx.put( + STORE_NAMES.VECTOR_CLOCK, + { clock: opts.vectorClock, lastUpdate: now }, + SINGLETON_KEY, + ); + await tx.put(STORE_NAMES.ARCHIVE_YOUNG, { + id: SINGLETON_KEY, + data: opts.archiveYoung, + lastModified: now, + }); + await tx.put(STORE_NAMES.ARCHIVE_OLD, { + id: SINGLETON_KEY, + data: opts.archiveOld, + lastModified: now, + }); + }, + ); + + this._invalidateAppliedAndUnsyncedCaches(); + this._vectorClockCache = { ...opts.vectorClock }; + } catch (e) { + if (e instanceof DOMException && e.name === 'QuotaExceededError') { + throw new StorageQuotaExceededError(); + } + throw e; + } + } + // ============================================================ // Vector Clock Management (Performance Optimization) // ============================================================ diff --git a/src/app/op-log/persistence/schema-migration.service.spec.ts b/src/app/op-log/persistence/schema-migration.service.spec.ts index 409998fd9c..bc722b9abb 100644 --- a/src/app/op-log/persistence/schema-migration.service.spec.ts +++ b/src/app/op-log/persistence/schema-migration.service.spec.ts @@ -147,6 +147,13 @@ describe('SchemaMigrationService', () => { // Should return the operation (with undefined treated as version 1) expect(result).not.toBeNull(); }); + + it('should reject a present non-integer schema version at the migration boundary', () => { + const op = createMockOperation('malformed-op'); + op.schemaVersion = '2' as unknown as number; + + expect(() => service.migrateOperation(op)).toThrowError(/schemaVersion/); + }); }); describe('migrateOperations', () => { @@ -181,6 +188,33 @@ describe('SchemaMigrationService', () => { expect(result.map((op) => op.id)).toEqual(['op-a', 'op-b', 'op-c']); }); + + it('should preserve unique migrated identities when a v1 config operation splits', () => { + const op = createMockOperation('config-op', 1); + op.actionType = ActionType.GLOBAL_CONFIG_UPDATE_SECTION; + op.entityType = 'GLOBAL_CONFIG'; + op.entityId = 'misc'; + op.entityIds = ['misc']; + op.payload = { + actionPayload: { + sectionKey: 'misc', + sectionCfg: { + isConfirmBeforeTaskDelete: true, + unrelatedMiscSetting: 'keep-me', + }, + }, + entityChanges: [], + }; + + const result = service.migrateOperations([op]); + + expect(result.map((migrated) => migrated.id)).toEqual([ + 'config-op_misc', + 'config-op_tasks', + ]); + expect(result.map((migrated) => migrated.entityId)).toEqual(['misc', 'tasks']); + expect(result.map((migrated) => migrated.entityIds)).toEqual([['misc'], ['tasks']]); + }); }); describe('migrateIfNeeded (deprecated)', () => { diff --git a/src/app/op-log/persistence/schema-migration.service.ts b/src/app/op-log/persistence/schema-migration.service.ts index f3a1ebdcbd..4fbaaafc5a 100644 --- a/src/app/op-log/persistence/schema-migration.service.ts +++ b/src/app/op-log/persistence/schema-migration.service.ts @@ -23,6 +23,22 @@ export const MIN_SUPPORTED_SCHEMA_VERSION = SHARED_MIN_SUPPORTED_SCHEMA_VERSION; // Re-export types export type { SchemaMigration }; +export const getOperationSchemaVersion = (op: { schemaVersion?: unknown }): number => { + if (op.schemaVersion === undefined) { + return 1; + } + if ( + typeof op.schemaVersion !== 'number' || + !Number.isInteger(op.schemaVersion) || + op.schemaVersion < 0 + ) { + throw new Error( + 'Operation schemaVersion must be a non-negative integer when present.', + ); + } + return op.schemaVersion; +}; + /** * Interface for state cache that may need migration. */ @@ -133,7 +149,7 @@ export class SchemaMigrationService { * @returns The migrated operation(s), or null if it should be dropped */ migrateOperation(op: Operation): Operation | Operation[] | null { - const opVersion = op.schemaVersion ?? 1; + const opVersion = getOperationSchemaVersion(op); if (opVersion >= CURRENT_SCHEMA_VERSION) { return op; @@ -164,6 +180,7 @@ export class SchemaMigrationService { if (Array.isArray(result.data)) { return result.data.map((migratedOpLike) => ({ ...op, + id: migratedOpLike.id, opType: migratedOpLike.opType as Operation['opType'], entityType: migratedOpLike.entityType as Operation['entityType'], entityId: migratedOpLike.entityId, @@ -176,9 +193,11 @@ export class SchemaMigrationService { // Merge migrated fields back into the original operation return { ...op, + id: result.data.id, opType: result.data.opType as Operation['opType'], entityType: result.data.entityType as Operation['entityType'], entityId: result.data.entityId, + entityIds: result.data.entityIds, payload: result.data.payload, schemaVersion: result.data.schemaVersion, }; @@ -225,13 +244,14 @@ export class SchemaMigrationService { * Returns true if the operation needs migration. */ operationNeedsMigration(op: Operation): boolean { + const schemaVersion = getOperationSchemaVersion(op); return sharedOperationNeedsMigration( { id: op.id, opType: op.opType, entityType: op.entityType, payload: op.payload, - schemaVersion: op.schemaVersion ?? 1, + schemaVersion, }, CURRENT_SCHEMA_VERSION, ); diff --git a/src/app/op-log/sync/conflict-resolution.service.spec.ts b/src/app/op-log/sync/conflict-resolution.service.spec.ts index 669b6dd0af..64690bf83d 100644 --- a/src/app/op-log/sync/conflict-resolution.service.spec.ts +++ b/src/app/op-log/sync/conflict-resolution.service.spec.ts @@ -55,6 +55,7 @@ describe('ConflictResolutionService', () => { 'append', 'appendBatchSkipDuplicates', 'appendWithVectorClockUpdate', + 'markArchivePending', 'markApplied', 'markRejected', 'markFailed', @@ -1787,8 +1788,9 @@ describe('ConflictResolutionService', () => { mockOpLogStore.hasOp.and.resolveTo(false); mockOpLogStore.append.and.resolveTo(1); - mockOperationApplier.applyOperations.and.resolveTo({ - appliedOps: [remoteOp], + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: [remoteOp] }; }); mockOpLogStore.markApplied.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); @@ -1812,10 +1814,14 @@ describe('ConflictResolutionService', () => { const callOrder: string[] = []; mockOpLogStore.hasOp.and.resolveTo(false); - mockOperationApplier.applyOperations.and.callFake(async () => { + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { callOrder.push('applyOperations'); + await options?.onReducersCommitted?.(ops); return { appliedOps: [remoteOp] }; }); + mockOpLogStore.markArchivePending.and.callFake(async () => { + callOrder.push('markArchivePending'); + }); mockOpLogStore.markApplied.and.callFake(async () => { callOrder.push('markApplied'); }); @@ -1831,16 +1837,21 @@ describe('ConflictResolutionService', () => { callerHoldsOperationLogLock: true, }); - expect(mockOperationApplier.applyOperations).toHaveBeenCalledWith([remoteOp], { - skipDeferredLocalActions: true, - }); + expect(mockOperationApplier.applyOperations).toHaveBeenCalledWith( + [remoteOp], + jasmine.objectContaining({ + skipDeferredLocalActions: true, + onReducersCommitted: jasmine.any(Function), + }), + ); expect(mockOperationLogEffects.processDeferredActions).toHaveBeenCalledWith({ callerHoldsOperationLogLock: true, }); expect(callOrder).toEqual([ 'applyOperations', - 'markApplied', + 'markArchivePending', 'mergeRemoteOpClocks', + 'markApplied', 'processDeferredActions', ]); }); @@ -1865,8 +1876,9 @@ describe('ConflictResolutionService', () => { mockOpLogStore.hasOp.and.resolveTo(false); mockOpLogStore.append.and.resolveTo(1); - mockOperationApplier.applyOperations.and.resolveTo({ - appliedOps: [conflictRemoteOp, nonConflictingOp], + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: [conflictRemoteOp, nonConflictingOp] }; }); mockOpLogStore.markApplied.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index 4603755a33..1d7e93a747 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -35,7 +35,6 @@ import { } from '../core/operation.types'; import { toLwwUpdateActionType } from '../core/lww-update-action-types'; import { OperationApplierService } from '../apply/operation-applier.service'; -import { getFailedOpIdsFromBatch } from '../apply/failed-op-ids.util'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { OpLog } from '../../core/log'; import { toEntityKey } from '../util/entity-key.util'; @@ -430,6 +429,18 @@ export class ConflictResolutionService { const opIdToSeq = new Map(allStoredOps.map((o) => [o.id, o.seq])); const applyResult = await this.operationApplier.applyOperations(allOpsToApply, { skipDeferredLocalActions: true, + onReducersCommitted: async (reducerCommittedOps) => { + const reducerCommittedSeqs = reducerCommittedOps + .map((op) => opIdToSeq.get(op.id)) + .filter((seq): seq is number => seq !== undefined); + if (reducerCommittedSeqs.length !== reducerCommittedOps.length) { + throw new Error( + 'ConflictResolutionService: reducer commit contained an unknown operation.', + ); + } + await this.opLogStore.markArchivePending(reducerCommittedSeqs); + await this.opLogStore.mergeRemoteOpClocks(reducerCommittedOps); + }, }); const appliedSeqs = applyResult.appliedOps @@ -439,28 +450,17 @@ export class ConflictResolutionService { if (appliedSeqs.length > 0) { await this.opLogStore.markApplied(appliedSeqs); - // CRITICAL: Merge remote ops' vector clocks into local clock. - // This ensures subsequent local operations have clocks that "dominate" - // the applied remote ops (GREATER_THAN instead of CONCURRENT). - // Without this, ops created after conflict resolution would have clocks - // that are CONCURRENT with the applied ops, causing them to be incorrectly - // filtered by SyncImportFilterService or rejected as conflicts on next sync. - await this.opLogStore.mergeRemoteOpClocks(applyResult.appliedOps); - OpLog.normal( `ConflictResolutionService: Successfully applied ${appliedSeqs.length} ops`, ); } if (applyResult.failedOp) { - const failedOpIds = getFailedOpIdsFromBatch( - allOpsToApply, - applyResult.failedOp.op, - ); + const failedOpIds = [applyResult.failedOp.op.id]; OpLog.err( `ConflictResolutionService: ${applyResult.appliedOps.length} ops applied before failure. ` + - `Marking ${failedOpIds.length} ops as failed.`, + 'Marking the attempted archive operation as failed.', applyResult.failedOp.error, ); await this.opLogStore.markFailed(failedOpIds, MAX_CONFLICT_RETRY_ATTEMPTS); diff --git a/src/app/op-log/sync/immediate-upload.service.spec.ts b/src/app/op-log/sync/immediate-upload.service.spec.ts index 62cf2a5cff..6d18e1ce4a 100644 --- a/src/app/op-log/sync/immediate-upload.service.spec.ts +++ b/src/app/op-log/sync/immediate-upload.service.spec.ts @@ -179,6 +179,19 @@ describe('ImmediateUploadService', () => { expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalled(); }); + it('should report ERROR when piggyback processing is blocked by an incompatible op', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.resolveTo({ + kind: 'blocked_incompatible', + }); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + })); + it('should NOT show checkmark when piggybacked ops exist (multiple)', fakeAsync(() => { mockSyncService.uploadPendingOps.and.returnValue( Promise.resolve(completedResult({ uploadedCount: 5, piggybackedOpsCount: 3 })), diff --git a/src/app/op-log/sync/immediate-upload.service.ts b/src/app/op-log/sync/immediate-upload.service.ts index 9f3cd1d2de..22c0fc0c9b 100644 --- a/src/app/op-log/sync/immediate-upload.service.ts +++ b/src/app/op-log/sync/immediate-upload.service.ts @@ -237,6 +237,14 @@ export class ImmediateUploadService implements OnDestroy { return; } + if (result.kind === 'blocked_incompatible') { + OpLog.warn( + 'ImmediateUploadService: Piggyback processing blocked by an incompatible operation', + ); + this._providerManager.setSyncStatus('ERROR'); + return; + } + // result.kind === 'completed' from here // If LWW local-wins created new update ops from piggybacked ops, diff --git a/src/app/op-log/sync/operation-log-download.service.spec.ts b/src/app/op-log/sync/operation-log-download.service.spec.ts index 1572a1d60e..23e78e857b 100644 --- a/src/app/op-log/sync/operation-log-download.service.spec.ts +++ b/src/app/op-log/sync/operation-log-download.service.spec.ts @@ -718,6 +718,51 @@ describe('OperationLogDownloadService', () => { expect(mockApiProvider.downloadOps).toHaveBeenCalledTimes(2); }); + it('should reject an empty page that claims more data', async () => { + mockApiProvider.downloadOps.and.resolveTo({ + ops: [], + hasMore: true, + latestSeq: 10, + }); + + const result = await service.downloadRemoteOps(mockApiProvider); + + expect(result.success).toBeFalse(); + expect(result.newOps).toEqual([]); + expect(mockSuperSyncStatusService.markRemoteChecked).not.toHaveBeenCalled(); + }); + + it('should reject a page that does not advance its cursor while claiming more data', async () => { + mockApiProvider.getLastServerSeq.and.resolveTo(5); + mockApiProvider.downloadOps.and.resolveTo({ + ops: [ + { + serverSeq: 5, + receivedAt: Date.now(), + op: { + id: 'op-stuck', + clientId: 'c1', + actionType: '[Task] Update' as ActionType, + opType: OpType.Update, + entityType: 'TASK', + payload: {}, + vectorClock: {}, + timestamp: Date.now(), + schemaVersion: 1, + }, + }, + ], + hasMore: true, + latestSeq: 10, + }); + + const result = await service.downloadRemoteOps(mockApiProvider); + + expect(result.success).toBeFalse(); + expect(result.newOps).toEqual([]); + expect(mockApiProvider.downloadOps).toHaveBeenCalledTimes(1); + }); + it('should filter already applied operations', async () => { mockOpLogStore.getAppliedOpIds.and.returnValue( Promise.resolve(new Set(['op-1'])), diff --git a/src/app/op-log/sync/operation-log-download.service.ts b/src/app/op-log/sync/operation-log-download.service.ts index daf055fb21..0034b2d8aa 100644 --- a/src/app/op-log/sync/operation-log-download.service.ts +++ b/src/app/op-log/sync/operation-log-download.service.ts @@ -229,6 +229,12 @@ export class OperationLogDownloadService implements OnDestroy { } if (response.ops.length === 0) { + if (response.hasMore) { + OpLog.error( + 'OperationLogDownloadService: Server returned an empty page with hasMore=true. Aborting to avoid accepting a partial download.', + ); + downloadFailed = true; + } // No ops to download - caller will persist latestServerSeq after this method returns break; } @@ -316,8 +322,18 @@ export class OperationLogDownloadService implements OnDestroy { break; } - // Update cursors - sinceSeq = response.ops[response.ops.length - 1].serverSeq; + // Update cursors. A page that claims more data must advance the cursor; + // otherwise accepting the accumulated prefix would silently skip the + // unseen suffix (or spin until the iteration cap). + const nextSinceSeq = response.ops[response.ops.length - 1].serverSeq; + if (response.hasMore && nextSinceSeq <= sinceSeq) { + OpLog.error( + `OperationLogDownloadService: Non-progressing page cursor (${nextSinceSeq} <= ${sinceSeq}) with hasMore=true. Aborting partial download.`, + ); + downloadFailed = true; + break; + } + sinceSeq = nextSinceSeq; hasMore = response.hasMore; // Monotonicity check: warn if server seq decreased (indicates potential server bug) diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index f0cc485793..30ea0d7307 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -37,6 +37,7 @@ import { BackupService } from '../backup/backup.service'; import { T } from '../../t.const'; import { INBOX_PROJECT } from '../../features/project/project.const'; import { TODAY_TAG, SYSTEM_TAG_IDS } from '../../features/tag/tag.const'; +import { OperationSyncCapable } from '../sync-providers/provider.interface'; describe('OperationLogSyncService', () => { let service: OperationLogSyncService; @@ -50,6 +51,9 @@ describe('OperationLogSyncService', () => { let stateSnapshotServiceSpy: jasmine.SpyObj; let backupServiceSpy: jasmine.SpyObj; let syncImportConflictDialogServiceSpy: jasmine.SpyObj; + let schemaMigrationServiceSpy: jasmine.SpyObj; + let validateStateServiceSpy: jasmine.SpyObj; + let lockServiceSpy: jasmine.SpyObj; beforeEach(() => { snackServiceSpy = jasmine.createSpyObj('SnackService', ['open']); @@ -58,14 +62,17 @@ describe('OperationLogSyncService', () => { 'loadStateCache', 'getLastSeq', 'getOpById', + 'markSynced', 'markRejected', 'setVectorClock', 'clearFullStateOps', 'getVectorClock', 'appendBatchSkipDuplicates', 'hasSyncedOps', + 'runRemoteStateReplacement', ]); opLogStoreSpy.hasSyncedOps.and.resolveTo(true); + opLogStoreSpy.markSynced.and.resolveTo(); opLogStoreSpy.setVectorClock.and.resolveTo(); opLogStoreSpy.clearFullStateOps.and.resolveTo(); opLogStoreSpy.getVectorClock.and.resolveTo(null); @@ -74,6 +81,26 @@ describe('OperationLogSyncService', () => { writtenOps: [], skippedCount: 0, }); + opLogStoreSpy.runRemoteStateReplacement.and.resolveTo(); + + schemaMigrationServiceSpy = jasmine.createSpyObj('SchemaMigrationService', [ + 'getCurrentVersion', + 'migrateOperation', + 'migrateOperations', + ]); + schemaMigrationServiceSpy.migrateOperations.and.callFake((ops) => ops); + + validateStateServiceSpy = jasmine.createSpyObj('ValidateStateService', [ + 'validateAndRepair', + 'validateAndRepairCurrentState', + ]); + validateStateServiceSpy.validateAndRepair.and.resolveTo({ + isValid: true, + wasRepaired: false, + }); + + lockServiceSpy = jasmine.createSpyObj('LockService', ['request']); + lockServiceSpy.request.and.callFake(async (_name, callback) => callback()); serverMigrationServiceSpy = jasmine.createSpyObj('ServerMigrationService', [ 'checkAndHandleMigration', 'handleServerMigration', @@ -137,13 +164,7 @@ describe('OperationLogSyncService', () => { providers: [ OperationLogSyncService, provideMockStore(), - { - provide: SchemaMigrationService, - useValue: jasmine.createSpyObj('SchemaMigrationService', [ - 'getCurrentVersion', - 'migrateOperation', - ]), - }, + { provide: SchemaMigrationService, useValue: schemaMigrationServiceSpy }, { provide: SnackService, useValue: snackServiceSpy }, { provide: OperationLogStoreService, useValue: opLogStoreSpy }, { @@ -166,13 +187,7 @@ describe('OperationLogSyncService', () => { 'checkOpForConflicts', ]), }, - { - provide: ValidateStateService, - useValue: jasmine.createSpyObj('ValidateStateService', [ - 'validateAndRepair', - 'validateAndRepairCurrentState', - ]), - }, + { provide: ValidateStateService, useValue: validateStateServiceSpy }, { provide: RepairOperationService, useValue: jasmine.createSpyObj('RepairOperationService', [ @@ -191,10 +206,7 @@ describe('OperationLogSyncService', () => { 'downloadRemoteOps', ]), }, - { - provide: LockService, - useValue: jasmine.createSpyObj('LockService', ['request']), - }, + { provide: LockService, useValue: lockServiceSpy }, { provide: OperationLogCompactionService, useValue: jasmine.createSpyObj('OperationLogCompactionService', ['compact']), @@ -747,6 +759,48 @@ describe('OperationLogSyncService', () => { expect(rejectedOpsHandlerServiceSpy.handleRejectedOps).not.toHaveBeenCalled(); }); + it('should return a terminal outcome and keep acknowledgements pending when piggyback processing is incompatible', async () => { + const piggybackedOp = { + id: 'future-op', + clientId: 'client-B', + actionType: 'test' as ActionType, + opType: OpType.Update, + entityType: 'TASK' as const, + entityId: 'task-1', + payload: {}, + vectorClock: { clientB: 1 }, + timestamp: Date.now(), + schemaVersion: 99, + }; + uploadServiceSpy.uploadPendingOps.and.resolveTo({ + uploadedCount: 1, + piggybackedOps: [piggybackedOp], + rejectedCount: 0, + rejectedOps: [], + pendingAcknowledgementSeqs: [1], + lastServerSeqToPersist: 9, + }); + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + }); + const setLastServerSeq = jasmine.createSpy('setLastServerSeq').and.resolveTo(); + const provider = { + ...mockProvider, + setLastServerSeq, + } as unknown as OperationSyncCapable; + + const result = await service.uploadPendingOps(provider); + + expect(result.kind).toBe('blocked_incompatible'); + expect(opLogStoreSpy.markSynced).not.toHaveBeenCalled(); + expect(setLastServerSeq).not.toHaveBeenCalled(); + expect(rejectedOpsHandlerServiceSpy.handleRejectedOps).not.toHaveBeenCalled(); + }); + it('should not call handleRejectedOps when there are no rejected ops', async () => { uploadServiceSpy.uploadPendingOps.and.returnValue( Promise.resolve({ @@ -942,7 +996,7 @@ describe('OperationLogSyncService', () => { // Cursor stays behind the blocked op so it is re-downloaded and retried // after an app update instead of skipped forever. - expect(result.kind).toBe('ops_processed'); + expect(result.kind).toBe('blocked_incompatible'); expect(setLastServerSeqSpy).not.toHaveBeenCalled(); }); @@ -1190,10 +1244,7 @@ describe('OperationLogSyncService', () => { ]); }); - it('should NOT throw LocalDataConflictError for clients with only system/config ops (no user data)', async () => { - // Clients with only system/config ops (no tasks/projects/tags) should NOT see conflict dialog. - // They should just download the remote data. - + it('should protect unsynced user config from file-snapshot replacement', async () => { const unsyncedEntry: OperationLogEntry = { seq: 1, op: { @@ -1239,9 +1290,10 @@ describe('OperationLogSyncService', () => { setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), } as any; - // Should NOT throw - fresh client should proceed with download - await expectAsync(service.downloadRemoteOps(mockProvider)).toBeResolved(); - expect(syncHydrationServiceSpy.hydrateFromRemoteSync).toHaveBeenCalled(); + await expectAsync( + service.downloadRemoteOps(mockProvider), + ).toBeRejectedWithError(LocalDataConflictError); + expect(syncHydrationServiceSpy.hydrateFromRemoteSync).not.toHaveBeenCalled(); }); it('should throw LocalDataConflictError when only config ops but store has meaningful data (provider switch)', async () => { @@ -2289,7 +2341,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), - } as any; + } as unknown as OperationSyncCapable; await service.forceUploadLocalState(mockProvider); @@ -2318,7 +2370,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), - } as any; + } as unknown as OperationSyncCapable; await service.forceUploadLocalState(mockProvider); @@ -2331,7 +2383,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { supportsOperationSync: true, - } as any; + } as unknown as OperationSyncCapable; await expectAsync(service.forceUploadLocalState(mockProvider)).toBeRejectedWith( error, @@ -2345,7 +2397,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), - } as any; + } as unknown as OperationSyncCapable; await expectAsync(service.forceUploadLocalState(mockProvider)).toBeRejectedWith( error, @@ -2391,16 +2443,13 @@ describe('OperationLogSyncService', () => { OperationLogDownloadService, ) as jasmine.SpyObj; - opLogStoreSpy.clearAllOperations = jasmine - .createSpy('clearAllOperations') - .and.resolveTo(); - opLogStoreSpy.saveStateCache = jasmine.createSpy('saveStateCache').and.resolveTo(); + opLogStoreSpy.runRemoteStateReplacement.calls.reset(); }); it('should download BEFORE any destructive local mutation', async () => { const callOrder: string[] = []; - opLogStoreSpy.clearAllOperations.and.callFake(async () => { - callOrder.push('clearAllOperations'); + opLogStoreSpy.runRemoteStateReplacement.and.callFake(async () => { + callOrder.push('runRemoteStateReplacement'); }); downloadServiceSpy.downloadRemoteOps.and.callFake(async () => { callOrder.push('downloadRemoteOps'); @@ -2421,25 +2470,31 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); - expect(callOrder).toEqual(['downloadRemoteOps', 'clearAllOperations']); + expect(callOrder).toEqual(['downloadRemoteOps', 'runRemoteStateReplacement']); }); - it('should capture a safety backup BEFORE clearing local data (#8107)', async () => { + it('should capture a safety backup after download but before replacement (#8107)', async () => { const callOrder: string[] = []; + downloadServiceSpy.downloadRemoteOps.and.callFake(async () => { + callOrder.push('downloadRemoteOps'); + return { + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }; + }); backupServiceSpy.captureImportBackup.and.callFake(async () => { callOrder.push('captureImportBackup'); return 1; }); - opLogStoreSpy.clearAllOperations.and.callFake(async () => { - callOrder.push('clearAllOperations'); + writeFlushServiceSpy.flushPendingWrites.and.callFake(async () => { + callOrder.push('flushPendingWrites'); }); - downloadServiceSpy.downloadRemoteOps.and.resolveTo({ - newOps: [makeRemoteOp()], - needsFullStateUpload: false, - success: true, - providerMode: 'superSyncOps', - failedFileCount: 0, - latestServerSeq: 1, + opLogStoreSpy.runRemoteStateReplacement.and.callFake(async () => { + callOrder.push('runRemoteStateReplacement'); }); const mockProvider = { supportsOperationSync: true, @@ -2448,10 +2503,23 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); - expect(callOrder).toEqual(['captureImportBackup', 'clearAllOperations']); + expect(callOrder).toEqual([ + 'downloadRemoteOps', + 'flushPendingWrites', + 'captureImportBackup', + 'runRemoteStateReplacement', + ]); }); it('should ABORT without wiping local data if the safety backup fails (#8107)', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); backupServiceSpy.captureImportBackup.and.rejectWith(new Error('disk full')); const mockProvider = { supportsOperationSync: true, @@ -2460,8 +2528,9 @@ describe('OperationLogSyncService', () => { await expectAsync(service.forceDownloadRemoteState(mockProvider)).toBeRejected(); - expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); - expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); + expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); + expect(downloadServiceSpy.downloadRemoteOps).toHaveBeenCalled(); + expect(backupServiceSpy.captureImportBackup).toHaveBeenCalled(); }); it('should offer to restore the previous data after replacing (#8107)', async () => { @@ -2510,6 +2579,33 @@ describe('OperationLogSyncService', () => { expect(setLastServerSeqSpy).toHaveBeenCalledWith(0); }); + it('should reset the external cursor before committing the local replacement', async () => { + const callOrder: string[] = []; + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + opLogStoreSpy.runRemoteStateReplacement.and.callFake(async () => { + callOrder.push('replace'); + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine + .createSpy('setLastServerSeq') + .and.callFake(async (seq: number) => { + if (seq === 0) callOrder.push('cursor-zero'); + }), + } as unknown as OperationSyncCapable; + + await service.forceDownloadRemoteState(mockProvider); + + expect(callOrder.slice(0, 2)).toEqual(['cursor-zero', 'replace']); + }); + it('should download raw history: forceFromSeq0 AND includeOwnAndAppliedOps', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], @@ -2551,7 +2647,7 @@ describe('OperationLogSyncService', () => { service.forceDownloadRemoteState(mockProvider), ).toBeRejectedWithError(/Download failed/); expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); - expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); + expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); expect(setLastServerSeqSpy).not.toHaveBeenCalled(); }); @@ -2573,10 +2669,95 @@ describe('OperationLogSyncService', () => { await expectAsync( service.forceDownloadRemoteState(mockProvider), ).toBeRejectedWithError(/newer schema version/); - expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); + expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); }); + it('should run all operation migrations before backup or replacement', async () => { + const remoteOp = { ...makeRemoteOp(), schemaVersion: 1 }; + const migratedOp = { ...remoteOp, schemaVersion: 4 }; + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [remoteOp], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + schemaMigrationServiceSpy.migrateOperations.and.returnValue([migratedOp]); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await service.forceDownloadRemoteState(mockProvider); + + expect(schemaMigrationServiceSpy.migrateOperations).toHaveBeenCalledOnceWith([ + remoteOp, + ]); + expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith( + [migratedOp], + { + skipConflictDetection: true, + callerHoldsOperationLogLock: true, + }, + ); + }); + + it('should abort before backup and replacement when operation migration fails', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + schemaMigrationServiceSpy.migrateOperations.and.throwError('bad migration'); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/migration failed/); + + expect(backupServiceSpy.captureImportBackup).not.toHaveBeenCalled(); + expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); + }); + + it('should validate a file snapshot before backup and replacement', async () => { + const snapshotState = { task: { ids: ['remote-task'] } }; + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [], + needsFullStateUpload: false, + success: true, + providerMode: 'fileSnapshotOps', + failedFileCount: 0, + snapshotState, + latestServerSeq: 1, + }); + validateStateServiceSpy.validateAndRepair.and.resolveTo({ + isValid: false, + wasRepaired: false, + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/snapshot is invalid/); + + expect(validateStateServiceSpy.validateAndRepair).toHaveBeenCalledOnceWith( + snapshotState, + ); + expect(backupServiceSpy.captureImportBackup).not.toHaveBeenCalled(); + expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); + }); + it('should NOT advance the cursor past 0 when replay blocks on a migration failure', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], @@ -2641,7 +2822,10 @@ describe('OperationLogSyncService', () => { expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith( mockOps, - { skipConflictDetection: true }, + { + skipConflictDetection: true, + callerHoldsOperationLogLock: true, + }, ); }); @@ -2703,7 +2887,7 @@ describe('OperationLogSyncService', () => { service.forceDownloadRemoteState(mockProvider), ).toBeRejectedWithError(/no data to rebuild from/); expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); - expect(opLogStoreSpy.clearAllOperations).not.toHaveBeenCalled(); + expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); expect(setLastServerSeqSpy).not.toHaveBeenCalled(); }); @@ -2753,9 +2937,9 @@ describe('OperationLogSyncService', () => { expect(setLastServerSeqSpy).toHaveBeenCalledWith(1); }); - it('should propagate errors from clearAllOperations', async () => { + it('should propagate errors from the atomic replacement', async () => { const error = new Error('Failed to clear ops'); - opLogStoreSpy.clearAllOperations.and.rejectWith(error); + opLogStoreSpy.runRemoteStateReplacement.and.rejectWith(error); downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], needsFullStateUpload: false, @@ -2816,7 +3000,10 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith( mockOps, - { skipConflictDetection: true }, + { + skipConflictDetection: true, + callerHoldsOperationLogLock: true, + }, ); }); }); @@ -3504,7 +3691,7 @@ describe('OperationLogSyncService', () => { expect(result.kind).toBe('ops_processed'); }); - it('should process incoming SYNC_IMPORT when pending ops are config-only', async () => { + it('should prompt before replacing pending user config with an incoming SYNC_IMPORT', async () => { const incomingSyncImport = createIncomingSyncImport(); downloadServiceSpy.downloadRemoteOps.and.resolveTo({ @@ -3542,19 +3729,14 @@ describe('OperationLogSyncService', () => { const result = await service.downloadRemoteOps(mockProvider); - expect( - syncImportConflictDialogServiceSpy.showConflictDialog, - ).not.toHaveBeenCalled(); - // No example-task ops pending -> nothing is rejected (empty-array guard). + expect(syncImportConflictDialogServiceSpy.showConflictDialog).toHaveBeenCalled(); expect(opLogStoreSpy.markRejected).not.toHaveBeenCalled(); - expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith([ - incomingSyncImport, - ]); - expect(mockProvider.setLastServerSeq).toHaveBeenCalledWith(42); - expect(result.kind).toBe('ops_processed'); + expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); + expect(mockProvider.setLastServerSeq).not.toHaveBeenCalled(); + expect(result.kind).toBe('cancelled'); }); - it('should process incoming SYNC_IMPORT when pending ops are only config and startup example tasks', async () => { + it('should prompt when pending user config exists alongside startup example tasks', async () => { const incomingSyncImport = createIncomingSyncImport(); downloadServiceSpy.downloadRemoteOps.and.resolveTo({ @@ -3619,20 +3801,11 @@ describe('OperationLogSyncService', () => { const result = await service.downloadRemoteOps(mockProvider); - expect( - syncImportConflictDialogServiceSpy.showConflictDialog, - ).not.toHaveBeenCalled(); - expect(opLogStoreSpy.markRejected).toHaveBeenCalledWith([ - 'local-example-task-op-1', - 'local-example-task-op-2', - 'local-example-task-op-3', - 'local-example-task-op-4', - ]); - expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith([ - incomingSyncImport, - ]); - expect(mockProvider.setLastServerSeq).toHaveBeenCalledWith(42); - expect(result.kind).toBe('ops_processed'); + expect(syncImportConflictDialogServiceSpy.showConflictDialog).toHaveBeenCalled(); + expect(opLogStoreSpy.markRejected).not.toHaveBeenCalled(); + expect(remoteOpsProcessingServiceSpy.processRemoteOps).not.toHaveBeenCalled(); + expect(mockProvider.setLastServerSeq).not.toHaveBeenCalled(); + expect(result.kind).toBe('cancelled'); }); }); @@ -3673,6 +3846,7 @@ describe('OperationLogSyncService', () => { piggybackedOps: [piggybackedSyncImport], rejectedCount: 0, rejectedOps: [], + pendingAcknowledgementSeqs: [1], }); // Client has pending ops @@ -3709,6 +3883,7 @@ describe('OperationLogSyncService', () => { syncImportReason: 'SERVER_MIGRATION', }), ); + expect(opLogStoreSpy.markSynced).not.toHaveBeenCalled(); expect(result.kind).toBe('cancelled'); }); @@ -3725,13 +3900,6 @@ describe('OperationLogSyncService', () => { schemaVersion: 1, }; - uploadServiceSpy.uploadPendingOps.and.resolveTo({ - uploadedCount: 1, - piggybackedOps: [piggybackedSyncImport], - rejectedCount: 0, - rejectedOps: [], - }); - const pendingEntry: OperationLogEntry = { seq: 1, op: { @@ -3749,14 +3917,17 @@ describe('OperationLogSyncService', () => { appliedAt: Date.now(), source: 'local', }; - // The op is pending BEFORE the upload but marked synced DURING it (server - // accepted it in the same round that piggybacked the import) — a live - // post-upload read no longer sees it. - let getUnsyncedCalls = 0; - opLogStoreSpy.getUnsynced.and.callFake(async () => { - getUnsyncedCalls++; - return getUnsyncedCalls === 1 ? [pendingEntry] : []; + uploadServiceSpy.uploadPendingOps.and.resolveTo({ + uploadedCount: 1, + piggybackedOps: [piggybackedSyncImport], + rejectedCount: 0, + rejectedOps: [], + selectedPendingOps: [pendingEntry], + pendingAcknowledgementSeqs: [pendingEntry.seq], }); + // The accepted operation is represented by the exact in-lock upload snapshot; + // it no longer needs to remain live-unsynced for the gate to protect it. + opLogStoreSpy.getUnsynced.and.resolveTo([]); syncImportConflictDialogServiceSpy.showConflictDialog.and.resolveTo('CANCEL'); @@ -3771,6 +3942,7 @@ describe('OperationLogSyncService', () => { expect(syncImportConflictDialogServiceSpy.showConflictDialog).toHaveBeenCalledWith( jasmine.objectContaining({ scenario: 'INCOMING_IMPORT' }), ); + expect(opLogStoreSpy.markSynced).not.toHaveBeenCalled(); expect(result.kind).toBe('cancelled'); }); @@ -3819,7 +3991,7 @@ describe('OperationLogSyncService', () => { expect(result.kind).toBe('completed'); }); - it('should not flush again before checking piggybacked SYNC_IMPORT conflicts', async () => { + it('should flush again before checking piggybacked SYNC_IMPORT conflicts', async () => { const piggybackedSyncImport: Operation = { id: 'import-1', clientId: 'client-B', @@ -3846,7 +4018,7 @@ describe('OperationLogSyncService', () => { const result = await service.uploadPendingOps(mockProvider); - expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalledTimes(1); + expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalledTimes(2); expect(opLogStoreSpy.getUnsynced).toHaveBeenCalled(); expect(result.kind).toBe('completed'); }); @@ -3898,6 +4070,7 @@ describe('OperationLogSyncService', () => { }); it('should process piggybacked ops normally when no SYNC_IMPORT present', async () => { + const events: string[] = []; const piggybackedOp: Operation = { id: 'op-1', clientId: 'client-B', @@ -3916,9 +4089,23 @@ describe('OperationLogSyncService', () => { piggybackedOps: [piggybackedOp], rejectedCount: 0, rejectedOps: [], + pendingAcknowledgementSeqs: [1], }); opLogStoreSpy.getUnsynced.and.resolveTo([]); + remoteOpsProcessingServiceSpy.processRemoteOps.and.callFake(async () => { + events.push('processRemoteOps'); + return { + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, + }; + }); + opLogStoreSpy.markSynced.and.callFake(async () => { + events.push('markSynced'); + }); const mockProvider = { isReady: () => Promise.resolve(true), @@ -3934,6 +4121,8 @@ describe('OperationLogSyncService', () => { expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith([ piggybackedOp, ]); + expect(opLogStoreSpy.markSynced).toHaveBeenCalledOnceWith([1]); + expect(events).toEqual(['processRemoteOps', 'markSynced']); expect(result.kind).not.toBe('cancelled'); }); diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 7fd655e5f8..6d97e0d7ce 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -38,13 +38,23 @@ import { SyncImportConflictResolution, } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { SyncImportConflictGateService } from './sync-import-conflict-gate.service'; -import { CURRENT_SCHEMA_VERSION } from '../persistence/schema-migration.service'; +import { + CURRENT_SCHEMA_VERSION, + MIN_SUPPORTED_SCHEMA_VERSION, + SchemaMigrationService, + getOperationSchemaVersion, +} from '../persistence/schema-migration.service'; import { SyncProviderManager } from '../sync-providers/provider-manager.service'; -import { getDefaultMainModelData } from '../model/model-config'; +import { getDefaultMainModelData, MODEL_CONFIGS } from '../model/model-config'; import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { SyncLocalStateService } from './sync-local-state.service'; import { SyncImportConflictCoordinatorService } from './sync-import-conflict-coordinator.service'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; +import { Operation } from '../core/operation.types'; +import { LockService } from './lock.service'; +import { LOCK_NAMES } from '../core/operation-log.const'; +import { ValidateStateService } from '../validation/validate-state.service'; +import { extractEntityKeysFromState } from '../persistence/extract-entity-keys'; /** * Orchestrates synchronization of the Operation Log with remote storage. @@ -122,6 +132,9 @@ export class OperationLogSyncService { private superSyncStatusService = inject(SuperSyncStatusService); private serverMigrationService = inject(ServerMigrationService); private writeFlushService = inject(OperationWriteFlushService); + private lockService = inject(LockService); + private schemaMigrationService = inject(SchemaMigrationService); + private validateStateService = inject(ValidateStateService); // Extracted services private remoteOpsProcessingService = inject(RemoteOpsProcessingService); @@ -189,11 +202,9 @@ export class OperationLogSyncService { // we would upload an incomplete set. This flush waits for all queued writes. await this.writeFlushService.flushPendingWrites(); - // Capture never-synced status BEFORE the upload runs: uploadService.uploadPendingOps - // marks accepted ops synced, which flips hasSyncedOps() and would defeat the piggyback - // conflict gate's never-synced guard if it read live state afterwards. The orchestrator - // passes a value captured even earlier (pre-download, since download also persists - // synced ops); fall back to a local read for standalone upload callers. + // Capture never-synced status before the upload runs. The orchestrator passes a + // value captured even earlier (pre-download, since download persists synced ops); + // fall back to a local read for standalone upload callers. const isNeverSyncedAtSyncStart = options?.isNeverSynced ?? !(await this.opLogStore.hasSyncedOps()); @@ -209,14 +220,6 @@ export class OperationLogSyncService { return { kind: 'blocked_fresh_client' }; } - // Capture pending ops BEFORE the upload round (the flush at the top of this - // method already drained in-flight writes). The piggyback conflict gate below - // must judge "would this import discard local work?" against this snapshot: - // ops accepted by the server during the upload are marked synced per chunk, - // so a post-upload getUnsynced() read no longer sees local work that a - // piggybacked import is about to replace. - const pendingOpsAtUploadStart = await this.opLogStore.getUnsynced(); - // SERVER MIGRATION CHECK: Passed as callback to execute INSIDE the upload lock. // This prevents race conditions where multiple tabs could both detect migration // and create duplicate SYNC_IMPORT operations. @@ -227,6 +230,9 @@ export class OperationLogSyncService { ? undefined : () => this.serverMigrationService.checkAndHandleMigration(syncProvider), skipPiggybackProcessing: options?.skipPiggybackProcessing, + // Keep accepted operations pending until piggyback processing commits. This + // preserves the conflict gate across cancellation and crash/retry boundaries. + deferAcknowledgement: true, }); // STEP 1: Process piggybacked ops FIRST @@ -258,7 +264,8 @@ export class OperationLogSyncService { result.piggybackedOps, { isNeverSynced: isNeverSyncedAtSyncStart, - preCapturedPendingOps: pendingOpsAtUploadStart, + flushPendingWrites: true, + preCapturedPendingOps: result.selectedPendingOps ?? [], }, ); if (piggybackedConflict.fullStateOp) { @@ -319,6 +326,10 @@ export class OperationLogSyncService { localWinOpsCreated = processResult.localWinOpsCreated; // Validation failure (if any) is on the session-validation latch. + if (processResult.blockedByIncompatibleOp) { + return { kind: 'blocked_incompatible' }; + } + // #8304: Persist lastServerSeq ONLY now that the piggybacked ops have been applied // above. The upload service deferred this (see UploadResult.lastServerSeqToPersist) // so that a crash — or a cancelled/USE_REMOTE/USE_LOCAL SYNC_IMPORT dialog, all of @@ -326,14 +337,16 @@ export class OperationLogSyncService { // that were never stored. Mirrors the download path's invariant. // A version/migration block keeps the cursor behind the blocked op so it is // re-downloaded and retried after an app update instead of skipped forever. - if ( - result.lastServerSeqToPersist !== undefined && - !processResult.blockedByIncompatibleOp - ) { + if (result.lastServerSeqToPersist !== undefined) { await syncProvider.setLastServerSeq(result.lastServerSeqToPersist); } } + const pendingAcknowledgementSeqs = result.pendingAcknowledgementSeqs ?? []; + if (pendingAcknowledgementSeqs.length > 0) { + await this.opLogStore.markSynced(pendingAcknowledgementSeqs); + } + // STEP 2: Handle server-rejected operations // handleRejectedOps may create merged ops for concurrent modifications. // These need to be uploaded, so we add them to localWinOpsCreated. @@ -367,6 +380,8 @@ export class OperationLogSyncService { case 'server_migration_handled': case 'cancelled': return { newOpsCount: 0 }; + case 'blocked_incompatible': + throw new Error('Nested download blocked by an incompatible remote operation.'); } }; try { @@ -878,6 +893,10 @@ export class OperationLogSyncService { result.newOps, ); + if (processResult.blockedByIncompatibleOp) { + return { kind: 'blocked_incompatible' }; + } + // ───────────────────────────────────────────────────────────────────────── // Handle SYNC_IMPORT conflict: all remote ops filtered by STORED local import. // This happens when user imports/restores data locally, and other devices @@ -933,7 +952,7 @@ export class OperationLogSyncService { // This is the correct behavior - better to re-download than to skip ops. // A version/migration block keeps the cursor behind the blocked op so it is // re-downloaded and retried after an app update instead of skipped forever. - if (result.latestServerSeq !== undefined && !processResult.blockedByIncompatibleOp) { + if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); } @@ -1076,10 +1095,14 @@ export class OperationLogSyncService { result.newOps, ); + if (processResult.blockedByIncompatibleOp) { + return { kind: 'blocked_incompatible' }; + } + // Persist the cursor only AFTER the ops are applied, matching the normal // incremental path's crash-safety ordering. A version/migration block keeps // the cursor behind the blocked op (retried after an app update). - if (result.latestServerSeq !== undefined && !processResult.blockedByIncompatibleOp) { + if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); } @@ -1156,24 +1179,6 @@ export class OperationLogSyncService { 'OperationLogSyncService: Force downloading remote state for a full rebuild.', ); - // Safety net (#8107): snapshot current local state before we destroy it, so an - // unintended "Use Server Data" — which can roll the user back to a stale server - // snapshot — is reversible via the Undo affordance shown on success below. - // Fail-safe: if the backup can't be written, abort rather than wipe without a - // recovery point (mirrors BackupService._persistImportToOperationLog). - let backupSavedAt: number; - try { - backupSavedAt = await this.backupService.captureImportBackup(); - } catch (e) { - OpLog.warn( - 'OperationLogSyncService: Pre-replace safety backup failed; aborting force download.', - { name: (e as Error | undefined)?.name }, - ); - throw new Error( - 'Pre-replace safety backup failed; aborting to preserve local state.', - ); - } - // ───────────────────────────────────────────────────────────────────────── // PHASE 1 — download and validate. Nothing local is mutated until the // complete server history is in memory: a network failure here must leave @@ -1207,23 +1212,19 @@ export class OperationLogSyncService { throw new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'); } - // Refuse BEFORE destroying anything if the remote history contains ops this - // client cannot interpret (newer schema) — replay would block midway and - // leave a partial rebuild. Migration exceptions can still surface during - // replay; those are handled after processRemoteOps below. - const tooNewOp = result.newOps.find( - (op) => (op.schemaVersion ?? 1) > CURRENT_SCHEMA_VERSION, - ); - if (tooNewOp) { - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.VERSION_TOO_OLD, - actionStr: T.PS.UPDATE_APP, - actionFn: () => window.open('https://super-productivity.com/download', '_blank'), - }); - throw new Error( - 'USE_REMOTE aborted: remote history contains ops from a newer schema version — update the app first.', - ); + const migratedRemoteOps = this._preflightRemoteOperations(result.newOps); + + let snapshotState = result.snapshotState as Record | undefined; + if (hasSnapshotState && snapshotState) { + const validation = await this.validateStateService.validateAndRepair(snapshotState); + if (!validation.isValid) { + throw new Error( + 'USE_REMOTE aborted: remote snapshot is invalid and could not be repaired.', + ); + } + if (validation.wasRepaired && validation.repairedState) { + snapshotState = validation.repairedState; + } } // ───────────────────────────────────────────────────────────────────────── @@ -1234,9 +1235,6 @@ export class OperationLogSyncService { // variant made the duplicate filter skip them, so a "rebuild" replayed only // an unseen suffix onto a defaults reset. // ───────────────────────────────────────────────────────────────────────── - await this.opLogStore.clearAllOperations(); - await syncProvider.setLastServerSeq(0); - // Reset the vector clock to the remote's causal knowledge (snapshot clock // merged with every downloaded op clock). This also drops entries from // rejected local ops that would otherwise pollute conflict detection. @@ -1244,113 +1242,202 @@ export class OperationLogSyncService { for (const opClock of result.allOpClocks ?? []) { rebuiltClock = mergeVectorClocks(rebuiltClock, opClock); } - await this.opLogStore.setVectorClock(rebuiltClock); - OpLog.normal('OperationLogSyncService: Reset vector clock to rebuilt remote clock.'); - - // FILE-BASED SYNC: Handle snapshot state from force download. - // When downloading from seq 0 on file-based providers, we may receive a - // snapshotState instead of incremental ops. This happens when the remote - // has a SYNC_IMPORT (full state snapshot) with empty recentOps. - // hydrateFromRemoteSync persists its own state cache + vector clock. - if (result.providerMode === 'fileSnapshotOps' && result.snapshotState) { - OpLog.normal( - 'OperationLogSyncService: Force download received snapshotState. Hydrating...', - ); - - // Hydrate from snapshot - DON'T create SYNC_IMPORT since we're - // accepting remote state, not uploading local state. - await this.syncHydrationService.hydrateFromRemoteSync( - result.snapshotState as Record, - result.snapshotVectorClock, - false, // Don't create SYNC_IMPORT - ); - - // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. - // Same rationale as downloadRemoteOps: file-based providers return ALL - // recentOps on every download and rely on getAppliedOpIds() to filter them. - if (result.newOps.length > 0) { - const appendResult = await this.opLogStore.appendBatchSkipDuplicates( - result.newOps, - 'remote', - ); - OpLog.normal( - `OperationLogSyncService: Wrote ${appendResult.writtenOps.length} snapshot ops to IndexedDB ` + - 'after force-download hydration.' + - (appendResult.skippedCount > 0 - ? ` Skipped ${appendResult.skippedCount} duplicate(s).` - : ''), - ); - } - - // Update lastServerSeq after hydration - if (result.latestServerSeq !== undefined) { - await syncProvider.setLastServerSeq(result.latestServerSeq); - } - - OpLog.normal( - 'OperationLogSyncService: Force download snapshot hydration complete.', - ); - this._showRestorePreviousDataSnack(backupSavedAt); - return; - } - - // Crash safety: persist a defaults baseline so an interrupted replay - // hydrates as defaults + appended prefix on next boot (a consistent prefix - // of server state, completed by the next sync from cursor 0) instead of - // layering the new history onto the stale pre-replace snapshot. - await this.opLogStore.saveStateCache({ - state: getDefaultMainModelData(), - lastAppliedOpSeq: 0, - vectorClock: rebuiltClock, - compactedAt: Date.now(), - snapshotEntityKeys: [], - }); - - // Reset live state to defaults, then replay the COMPLETE server history on - // top. A full-state op in the history replaces state again by its own - // semantics; a purely incremental history rebuilds from this baseline. const defaultData = getDefaultMainModelData(); - this.store.dispatch( - loadAllData({ - appDataComplete: defaultData as Parameters< - typeof loadAllData - >[0]['appDataComplete'], - }), - ); - // Brief yield to let NgRx process the state reset - await new Promise((resolve) => setTimeout(resolve, 0)); + const baselineState = snapshotState ?? defaultData; + const archiveYoung = + (snapshotState?.[ + 'archiveYoung' + ] as typeof MODEL_CONFIGS.archiveYoung.defaultData) ?? + MODEL_CONFIGS.archiveYoung.defaultData!; + const archiveOld = + (snapshotState?.['archiveOld'] as typeof MODEL_CONFIGS.archiveOld.defaultData) ?? + MODEL_CONFIGS.archiveOld.defaultData!; - // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). - // Skip conflict detection because the NgRx store was just reset to empty state, - // which causes all entities to appear missing and CONCURRENT ops to be discarded. - // Validation failure is surfaced via the session-validation latch. (#7330) - const processResult = await this.remoteOpsProcessingService.processRemoteOps( - result.newOps, - { skipConflictDetection: true }, - ); + let capturedBackupSavedAt: number | undefined; + let replacementCommitted = false; + let backupSavedAt: number; + try { + backupSavedAt = await this.lockService.request( + LOCK_NAMES.OPERATION_LOG, + async () => { + // Include actions dispatched while the network request and preflight were + // in flight in the reversible safety backup. + await this.writeFlushService.flushPendingWrites(); + let savedAt: number; + try { + savedAt = await this.backupService.captureImportBackup(); + capturedBackupSavedAt = savedAt; + } catch (e) { + OpLog.warn( + 'OperationLogSyncService: Pre-replace safety backup failed; aborting force download.', + { name: (e as Error | undefined)?.name }, + ); + throw new Error( + 'Pre-replace safety backup failed; aborting to preserve local state.', + ); + } - if (processResult.blockedByIncompatibleOp) { - // Version blocks were pre-checked above; only a migration exception lands - // here. The rebuild is partial: keep the cursor at 0 so the next sync - // retries the remainder, and surface the failure — the Undo snack still - // offers the pre-replace backup. - this._showRestorePreviousDataSnack(backupSavedAt); - throw new Error( - 'USE_REMOTE incomplete: an op failed schema migration during replay.', + // The provider cursor lives outside SUP_OPS, so it cannot join the IDB + // transaction. Reset it first: a crash/failure before the transaction + // merely causes a safe re-download onto intact local state, while a + // commit can never become visible with a stale cursor that skips the + // remote history required to rebuild the baseline. + await syncProvider.setLastServerSeq(0); + await this.opLogStore.runRemoteStateReplacement({ + baselineState, + vectorClock: rebuiltClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + snapshotEntityKeys: extractEntityKeysFromState( + baselineState as Parameters[0], + ), + archiveYoung, + archiveOld, + }); + replacementCommitted = true; + + OpLog.normal( + 'OperationLogSyncService: Replaced local persistence with remote baseline.', + ); + + // FILE-BASED SYNC: Handle snapshot state from force download. + // When downloading from seq 0 on file-based providers, we may receive a + // snapshotState instead of incremental ops. This happens when the remote + // has a SYNC_IMPORT (full state snapshot) with empty recentOps. + // hydrateFromRemoteSync persists its own state cache + vector clock. + if (result.providerMode === 'fileSnapshotOps' && snapshotState) { + OpLog.normal( + 'OperationLogSyncService: Force download received snapshotState. Hydrating...', + ); + + // Hydrate from snapshot - DON'T create SYNC_IMPORT since we're + // accepting remote state, not uploading local state. + await this.syncHydrationService.hydrateFromRemoteSync( + snapshotState, + result.snapshotVectorClock, + false, // Don't create SYNC_IMPORT + ); + + // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. + // Same rationale as downloadRemoteOps: file-based providers return ALL + // recentOps on every download and rely on getAppliedOpIds() to filter them. + if (migratedRemoteOps.length > 0) { + const appendResult = await this.opLogStore.appendBatchSkipDuplicates( + migratedRemoteOps, + 'remote', + ); + OpLog.normal( + `OperationLogSyncService: Wrote ${appendResult.writtenOps.length} snapshot ops to IndexedDB ` + + 'after force-download hydration.' + + (appendResult.skippedCount > 0 + ? ` Skipped ${appendResult.skippedCount} duplicate(s).` + : ''), + ); + } + + // Update lastServerSeq after hydration + if (result.latestServerSeq !== undefined) { + await syncProvider.setLastServerSeq(result.latestServerSeq); + } + + OpLog.normal( + 'OperationLogSyncService: Force download snapshot hydration complete.', + ); + return savedAt; + } + + // Reset live state to defaults, then replay the COMPLETE server history on + // top. A full-state op in the history replaces state again by its own + // semantics; a purely incremental history rebuilds from this baseline. + this.store.dispatch( + loadAllData({ + appDataComplete: defaultData as Parameters< + typeof loadAllData + >[0]['appDataComplete'], + }), + ); + // Brief yield to let NgRx process the state reset + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). + // Skip conflict detection because the NgRx store was just reset to empty state, + // which causes all entities to appear missing and CONCURRENT ops to be discarded. + // Validation failure is surfaced via the session-validation latch. (#7330) + const processResult = await this.remoteOpsProcessingService.processRemoteOps( + migratedRemoteOps, + { + skipConflictDetection: true, + callerHoldsOperationLogLock: true, + }, + ); + + if (processResult.blockedByIncompatibleOp) { + // Version blocks were pre-checked above; only a migration exception lands + // here. The rebuild is partial: keep the cursor at 0 so the next sync + // retries the remainder, and surface the failure — the Undo snack still + // offers the pre-replace backup. + throw new Error( + 'USE_REMOTE incomplete: an op failed schema migration during replay.', + ); + } + + // Update lastServerSeq + if (result.latestServerSeq !== undefined) { + await syncProvider.setLastServerSeq(result.latestServerSeq); + } + + OpLog.normal( + `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, + ); + return savedAt; + }, ); + } catch (e) { + if (replacementCommitted && capturedBackupSavedAt !== undefined) { + this._showRestorePreviousDataSnack(capturedBackupSavedAt); + } + throw e; } - // Update lastServerSeq - if (result.latestServerSeq !== undefined) { - await syncProvider.setLastServerSeq(result.latestServerSeq); - } - - OpLog.normal( - `OperationLogSyncService: Force download complete. Rebuilt from ${result.newOps.length} ops.`, - ); this._showRestorePreviousDataSnack(backupSavedAt); } + private _preflightRemoteOperations(remoteOps: Operation[]): Operation[] { + for (const op of remoteOps) { + let version: number; + try { + version = getOperationSchemaVersion(op as { schemaVersion?: unknown }); + } catch { + throw new Error( + 'USE_REMOTE aborted: remote history has an invalid schema version.', + ); + } + + if (version < MIN_SUPPORTED_SCHEMA_VERSION) { + throw new Error( + 'USE_REMOTE aborted: remote history contains an unsupported schema version.', + ); + } + if (version > CURRENT_SCHEMA_VERSION) { + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.VERSION_TOO_OLD, + actionStr: T.PS.UPDATE_APP, + actionFn: () => + window.open('https://super-productivity.com/download', '_blank'), + }); + throw new Error( + 'USE_REMOTE aborted: remote history contains ops from a newer schema version — update the app first.', + ); + } + } + + try { + return this.schemaMigrationService.migrateOperations(remoteOps); + } catch { + throw new Error('USE_REMOTE aborted: remote operation migration failed.'); + } + } + /** * Shows a non-blocking snack after a destructive "Use Server Data" replace, * offering to restore the local snapshot captured before the wipe — making the diff --git a/src/app/op-log/sync/operation-log-upload.service.spec.ts b/src/app/op-log/sync/operation-log-upload.service.spec.ts index 8a9a5fac18..ea4a810525 100644 --- a/src/app/op-log/sync/operation-log-upload.service.spec.ts +++ b/src/app/op-log/sync/operation-log-upload.service.spec.ts @@ -320,6 +320,30 @@ describe('OperationLogUploadService', () => { expect(mockOpLogStore.markSynced).toHaveBeenCalledWith([1, 2]); }); + it('should defer acknowledgements and return the exact selected batch for piggyback resolution', async () => { + const pendingOps = [ + createMockEntry(1, 'op-1', 'client-1'), + createMockEntry(2, 'op-2', 'client-1'), + ]; + mockOpLogStore.getUnsynced.and.resolveTo(pendingOps); + mockApiProvider.uploadOps.and.resolveTo({ + results: [ + { opId: 'op-1', accepted: true }, + { opId: 'op-2', accepted: true }, + ], + latestSeq: 10, + newOps: [], + }); + + const result = await service.uploadPendingOps(mockApiProvider, { + deferAcknowledgement: true, + }); + + expect(mockOpLogStore.markSynced).not.toHaveBeenCalled(); + expect(result.selectedPendingOps).toEqual(pendingOps); + expect(result.pendingAcknowledgementSeqs).toEqual([1, 2]); + }); + it('should mark accepted seqs correctly when server results are out of order', async () => { const pendingOps = [ createMockEntry(1, 'op-1', 'client-1'), diff --git a/src/app/op-log/sync/operation-log-upload.service.ts b/src/app/op-log/sync/operation-log-upload.service.ts index 93bef79ca4..29beba2f14 100644 --- a/src/app/op-log/sync/operation-log-upload.service.ts +++ b/src/app/op-log/sync/operation-log-upload.service.ts @@ -86,6 +86,24 @@ export class OperationLogUploadService { let uploadedCount = 0; let rejectedCount = 0; let hasMorePiggyback = false; + let selectedPendingOps: OperationLogEntry[] = []; + const pendingAcknowledgementSeqs: number[] = []; + const pendingAcknowledgementSeqSet = new Set(); + const acknowledge = async (seqs: number[]): Promise => { + if (seqs.length === 0) { + return; + } + if (!options?.deferAcknowledgement) { + await this.opLogStore.markSynced(seqs); + return; + } + for (const seq of seqs) { + if (!pendingAcknowledgementSeqSet.has(seq)) { + pendingAcknowledgementSeqSet.add(seq); + pendingAcknowledgementSeqs.push(seq); + } + } + }; // Track encryption state of piggybacked operations for detecting encryption config mismatch. // When another client disables encryption, all piggybacked ops will be unencrypted. // We track this BEFORE decryption to detect the server's actual encryption state. @@ -107,6 +125,7 @@ export class OperationLogUploadService { } const pendingOps = await this.opLogStore.getUnsynced(); + selectedPendingOps = pendingOps; if (pendingOps.length === 0) { OpLog.normal('OperationLogUploadService: No pending operations to upload.'); @@ -198,7 +217,7 @@ export class OperationLogUploadService { isCleanSlateForOp, ); if (result.accepted) { - await this.opLogStore.markSynced([entry.seq]); + await acknowledge([entry.seq]); uploadedCount++; if (result.serverSeq !== undefined) { await syncProvider.setLastServerSeq(result.serverSeq); @@ -258,7 +277,7 @@ export class OperationLogUploadService { if (opsIncludedInSnapshot.length > 0) { const seqs = opsIncludedInSnapshot.map((entry) => entry.seq); - await this.opLogStore.markSynced(seqs); + await acknowledge(seqs); uploadedCount += seqs.length; OpLog.normal( `OperationLogUploadService: Marked ${seqs.length} regular ops as synced ` + @@ -293,7 +312,7 @@ export class OperationLogUploadService { } } if (localOnlySeqs.length > 0) { - await this.opLogStore.markSynced(localOnlySeqs); + await acknowledge(localOnlySeqs); uploadedCount += localOnlySeqs.length; OpLog.normal( `OperationLogUploadService: Marked ${localOnlySeqs.length} local-only op(s) as synced without upload`, @@ -339,7 +358,7 @@ export class OperationLogUploadService { .filter((seq): seq is number => seq !== undefined); if (acceptedSeqs.length > 0) { - await this.opLogStore.markSynced(acceptedSeqs); + await acknowledge(acceptedSeqs); uploadedCount += acceptedSeqs.length; } @@ -468,6 +487,9 @@ export class OperationLogUploadService { ...(piggybackHasOnlyUnencryptedData ? { piggybackHasOnlyUnencryptedData } : {}), ...(lastServerSeqToPersist !== undefined ? { lastServerSeqToPersist } : {}), ...(encryptionRequiredKeyMissing ? { encryptionRequiredKeyMissing: true } : {}), + ...(options?.deferAcknowledgement + ? { selectedPendingOps, pendingAcknowledgementSeqs } + : {}), }; } diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index c59653fc0f..2c77b7527f 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -73,6 +73,7 @@ describe('RemoteOpsProcessingService', () => { 'append', 'appendBatchSkipDuplicates', 'appendWithVectorClockUpdate', + 'markArchivePending', 'markApplied', 'markFailed', 'mergeRemoteOpClocks', @@ -369,6 +370,57 @@ describe('RemoteOpsProcessingService', () => { ]); }); + it('should log conflict identities without logging operation payloads', async () => { + const localOp = { + id: 'local-op', + entityType: 'TASK', + entityId: 'task-1', + payload: { title: 'private local title' }, + } as Operation; + const remoteOp = { + id: 'remote-op', + entityType: 'TASK', + entityId: 'task-1', + payload: { title: 'private remote title' }, + schemaVersion: 1, + } as Operation; + spyOn(service, 'detectConflicts').and.resolveTo({ + nonConflicting: [], + conflicts: [ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ], + }); + conflictResolutionServiceSpy.autoResolveConflictsLWW.and.resolveTo({ + localWinOpsCreated: 0, + }); + vectorClockServiceSpy.getEntityFrontier.and.resolveTo(new Map()); + const warnSpy = spyOn(OpLog, 'warn'); + + await service.processRemoteOps([remoteOp]); + + const summary = warnSpy.calls + .allArgs() + .find(([message]) => String(message).includes('Detected 1 conflicts'))?.[1]; + expect(summary).toEqual({ + conflicts: [ + { + entityType: 'TASK', + entityId: 'task-1', + localOpIds: ['local-op'], + remoteOpIds: ['remote-op'], + suggestedResolution: 'manual', + }, + ], + }); + expect(JSON.stringify(summary)).not.toContain('private'); + }); + it('should drop operations if migrateOperation returns null', async () => { const remoteOps: Operation[] = [ { id: 'op1', schemaVersion: 1 } as Operation, @@ -516,6 +568,21 @@ describe('RemoteOpsProcessingService', () => { expect(result.blockedByIncompatibleOp).toBe(true); }); + for (const invalidVersion of [null, '2', 1.5, {}, Number.NaN]) { + it(`should block malformed schemaVersion ${String(invalidVersion)}`, async () => { + const remoteOp = { + id: 'malformed-version', + schemaVersion: invalidVersion, + } as unknown as Operation; + + const result = await service.processRemoteOps([remoteOp]); + + expect(result.blockedByIncompatibleOp).toBeTrue(); + expect(schemaMigrationServiceSpy.migrateOperation).not.toHaveBeenCalled(); + expect(opLogStoreSpy.appendBatchSkipDuplicates).not.toHaveBeenCalled(); + }); + } + it('should process the prefix before a too-new op but flag the block', async () => { const remoteOps: Operation[] = [ { id: 'op1', schemaVersion: 1 } as Operation, @@ -1337,9 +1404,10 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: remoteOps }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: remoteOps }; + }); await service.applyNonConflictingOps(remoteOps); @@ -1352,10 +1420,14 @@ describe('RemoteOpsProcessingService', () => { ]; const callOrder: string[] = []; - operationApplierServiceSpy.applyOperations.and.callFake(async () => { + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { callOrder.push('applyOperations'); + await options?.onReducersCommitted?.(ops); return { appliedOps: remoteOps }; }); + opLogStoreSpy.markArchivePending.and.callFake(async () => { + callOrder.push('markArchivePending'); + }); opLogStoreSpy.markApplied.and.callFake(async () => { callOrder.push('markApplied'); }); @@ -1368,16 +1440,21 @@ describe('RemoteOpsProcessingService', () => { await service.applyNonConflictingOps(remoteOps, true); - expect(operationApplierServiceSpy.applyOperations).toHaveBeenCalledWith(remoteOps, { - skipDeferredLocalActions: true, - }); + expect(operationApplierServiceSpy.applyOperations).toHaveBeenCalledWith( + remoteOps, + jasmine.objectContaining({ + skipDeferredLocalActions: true, + onReducersCommitted: jasmine.any(Function), + }), + ); expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ callerHoldsOperationLogLock: true, }); expect(callOrder).toEqual([ 'applyOperations', - 'markApplied', + 'markArchivePending', 'mergeRemoteOpClocks', + 'markApplied', 'processDeferredActions', ]); }); @@ -1397,7 +1474,7 @@ describe('RemoteOpsProcessingService', () => { expect(opLogStoreSpy.mergeRemoteOpClocks).not.toHaveBeenCalled(); }); - it('should mark failed ops and run validation on partial failure', async () => { + it('should charge only the attempted archive failure and run validation', async () => { const remoteOps: Operation[] = [ createFullOp({ id: 'op-1' }), createFullOp({ id: 'op-2' }), @@ -1407,17 +1484,19 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); opLogStoreSpy.markApplied.and.returnValue(Promise.resolve()); opLogStoreSpy.markFailed.and.returnValue(Promise.resolve()); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: [remoteOps[0]], failedOp: { op: remoteOps[1], error: new Error('Test error') }, - }), - ); + }; + }); await expectAsync(service.applyNonConflictingOps(remoteOps)).toBeRejected(); - // Should mark op-2 and op-3 as failed - expect(opLogStoreSpy.markFailed).toHaveBeenCalledWith(['op-2', 'op-3']); + expect(opLogStoreSpy.markArchivePending).toHaveBeenCalledWith([1, 2, 3]); + expect(opLogStoreSpy.mergeRemoteOpClocks).toHaveBeenCalledWith(remoteOps); + expect(opLogStoreSpy.markFailed).toHaveBeenCalledWith(['op-2']); // Should run validation after partial failure expect(validateStateServiceSpy.validateAndRepairCurrentState).toHaveBeenCalledWith( 'partial-apply-failure', @@ -1486,7 +1565,10 @@ describe('RemoteOpsProcessingService', () => { // Should apply only the non-duplicate op expect(operationApplierServiceSpy.applyOperations).toHaveBeenCalledWith( [remoteOps[1]], - { skipDeferredLocalActions: true }, + jasmine.objectContaining({ + skipDeferredLocalActions: true, + onReducersCommitted: jasmine.any(Function), + }), ); }); diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index d1e910f3c9..8d64f02ad2 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -21,6 +21,7 @@ import { VectorClockService } from './vector-clock.service'; import { MIN_SUPPORTED_SCHEMA_VERSION, SchemaMigrationService, + getOperationSchemaVersion, } from '../persistence/schema-migration.service'; import { SnackService } from '../../core/snack/snack.service'; import { T } from '../../t.const'; @@ -92,7 +93,10 @@ export class RemoteOpsProcessingService { */ async processRemoteOps( remoteOps: Operation[], - options?: { skipConflictDetection?: boolean }, + options?: { + skipConflictDetection?: boolean; + callerHoldsOperationLogLock?: boolean; + }, ): Promise<{ localWinOpsCreated: number; allOpsFilteredBySyncImport: boolean; @@ -125,12 +129,22 @@ export class RemoteOpsProcessingService { const currentVersion = this.schemaMigrationService.getCurrentVersion(); const migratedOps: Operation[] = []; const droppedEntityIds = new Set(); - let blockReason: 'VERSION_UNSUPPORTED' | 'VERSION_TOO_NEW' | 'MIGRATION_FAILED' = - 'MIGRATION_FAILED'; + let blockReason: + | 'VERSION_UNSUPPORTED' + | 'VERSION_TOO_NEW' + | 'INVALID_SCHEMA_VERSION' + | 'MIGRATION_FAILED' = 'MIGRATION_FAILED'; let blockedOp: Operation | null = null; for (const op of remoteOps) { - const opVersion = op.schemaVersion ?? 1; + let opVersion: number; + try { + opVersion = getOperationSchemaVersion(op as { schemaVersion?: unknown }); + } catch { + blockedOp = op; + blockReason = 'INVALID_SCHEMA_VERSION'; + break; + } // Op below minimum supported version: no migration path exists. if (opVersion < MIN_SUPPORTED_SCHEMA_VERSION) { @@ -281,8 +295,11 @@ export class RemoteOpsProcessingService { 'RemoteOpsProcessingService: Skipping conflict detection (skipConflictDetection=true). ' + `Applying ${validOps.length} ops directly.`, ); - await this.applyNonConflictingOps(validOps); - await this.validateAfterSync(); + await this.applyNonConflictingOps( + validOps, + options.callerHoldsOperationLogLock ?? false, + ); + await this.validateAfterSync(options.callerHoldsOperationLogLock ?? false); return { localWinOpsCreated: 0, allOpsFilteredBySyncImport: false, @@ -322,7 +339,15 @@ export class RemoteOpsProcessingService { if (conflicts.length > 0) { OpLog.warn( `RemoteOpsProcessingService: Detected ${conflicts.length} conflicts. Auto-resolving with LWW.`, - conflicts, + { + conflicts: conflicts.map((conflict) => ({ + entityType: conflict.entityType, + entityId: conflict.entityId, + localOpIds: conflict.localOps.map((op) => op.id), + remoteOpIds: conflict.remoteOps.map((op) => op.id), + suggestedResolution: conflict.suggestedResolution, + })), + }, ); // Auto-resolve conflicts using Last-Write-Wins strategy. // Piggyback non-conflicting ops so they're applied with resolved conflicts. @@ -359,9 +384,13 @@ export class RemoteOpsProcessingService { * until an app update or migration fix, and every retry re-hits it). */ private _notifyBlockedOp( - reason: 'VERSION_UNSUPPORTED' | 'VERSION_TOO_NEW' | 'MIGRATION_FAILED', + reason: + | 'VERSION_UNSUPPORTED' + | 'VERSION_TOO_NEW' + | 'INVALID_SCHEMA_VERSION' + | 'MIGRATION_FAILED', ): void { - if (reason === 'MIGRATION_FAILED') { + if (reason === 'MIGRATION_FAILED' || reason === 'INVALID_SCHEMA_VERSION') { if (!this._hasWarnedMigrationFailureThisSession) { this._hasWarnedMigrationFailureThisSession = true; this.snackService.open({ @@ -423,9 +452,10 @@ export class RemoteOpsProcessingService { ops: locallyReplayableOps, store: this.opLogStore, applier: { - applyOperations: (opsToApply) => + applyOperations: (opsToApply, applyOptions) => this.operationApplier.applyOperations(opsToApply, { skipDeferredLocalActions: true, + onReducersCommitted: applyOptions?.onReducersCommitted, }), }, isFullStateOperation: this._isFullStateOperation, diff --git a/src/app/op-log/sync/server-migration.service.spec.ts b/src/app/op-log/sync/server-migration.service.spec.ts index 08861775a8..2eb1041845 100644 --- a/src/app/op-log/sync/server-migration.service.spec.ts +++ b/src/app/op-log/sync/server-migration.service.spec.ts @@ -7,7 +7,7 @@ import { ServerMigrationService } from './server-migration.service'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { VectorClockService } from './vector-clock.service'; import { ValidateStateService } from '../validation/validate-state.service'; -import { StateSnapshotService } from '../backup/state-snapshot.service'; +import { AppStateSnapshot, StateSnapshotService } from '../backup/state-snapshot.service'; import { SnackService } from '../../core/snack/snack.service'; import { UserInputWaitStateService } from '../../imex/sync/user-input-wait-state.service'; import { @@ -20,6 +20,10 @@ import { SYSTEM_TAG_IDS } from '../../features/tag/tag.const'; import { INBOX_PROJECT } from '../../features/project/project.const'; import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { CLIENT_ID_PROVIDER, ClientIdProvider } from '../util/client-id.provider'; +import { LockService } from './lock.service'; +import { OperationWriteFlushService } from './operation-write-flush.service'; +import { LOCK_NAMES } from '../core/operation-log.const'; +import { OperationCaptureService } from '../capture/operation-capture.service'; describe('ServerMigrationService', () => { let service: ServerMigrationService; @@ -32,6 +36,9 @@ describe('ServerMigrationService', () => { let clientIdProviderSpy: jasmine.SpyObj; let matDialogSpy: jasmine.SpyObj; let userInputWaitStateSpy: jasmine.SpyObj; + let lockServiceSpy: jasmine.SpyObj; + let writeFlushServiceSpy: jasmine.SpyObj; + let operationCaptureServiceSpy: jasmine.SpyObj; let defaultProvider: OperationSyncProvider; // Type for operation-sync-capable provider @@ -86,6 +93,18 @@ describe('ServerMigrationService', () => { 'startWaiting', ]); userInputWaitStateSpy.startWaiting.and.returnValue(() => {}); + lockServiceSpy = jasmine.createSpyObj('LockService', ['request']); + lockServiceSpy.request.and.callFake(async (_name: string, fn: () => Promise) => + fn(), + ); + writeFlushServiceSpy = jasmine.createSpyObj('OperationWriteFlushService', [ + 'flushPendingWrites', + ]); + writeFlushServiceSpy.flushPendingWrites.and.resolveTo(); + operationCaptureServiceSpy = jasmine.createSpyObj('OperationCaptureService', [ + 'getPendingCount', + ]); + operationCaptureServiceSpy.getPendingCount.and.returnValue(0); // Default mock returns opLogStoreSpy.hasSyncedOps.and.returnValue(Promise.resolve(true)); @@ -130,6 +149,9 @@ describe('ServerMigrationService', () => { { provide: CLIENT_ID_PROVIDER, useValue: clientIdProviderSpy }, { provide: MatDialog, useValue: matDialogSpy }, { provide: UserInputWaitStateService, useValue: userInputWaitStateSpy }, + { provide: LockService, useValue: lockServiceSpy }, + { provide: OperationWriteFlushService, useValue: writeFlushServiceSpy }, + { provide: OperationCaptureService, useValue: operationCaptureServiceSpy }, ], }); @@ -471,6 +493,52 @@ describe('ServerMigrationService', () => { }); }); + it('should capture and append the full-state operation inside one operation-log barrier', async () => { + const events: string[] = []; + writeFlushServiceSpy.flushPendingWrites.and.callFake(async () => { + events.push('flush'); + }); + lockServiceSpy.request.and.callFake(async (name: string, fn: () => Promise) => { + events.push(`lock:${name}:start`); + const result = await fn(); + events.push(`lock:${name}:end`); + return result; + }); + stateSnapshotServiceSpy.getStateSnapshotAsync.and.callFake(async () => { + events.push('snapshot'); + return { + task: { ids: ['task-1'], entities: { 'task-1': { id: 'task-1' } } }, + project: { ids: [], entities: {} }, + tag: { ids: [], entities: {} }, + } as unknown as AppStateSnapshot; + }); + opLogStoreSpy.append.and.callFake(async () => { + events.push('append'); + return 1; + }); + + await service.handleServerMigration(defaultProvider); + + expect(events).toEqual([ + 'flush', + `lock:${LOCK_NAMES.OPERATION_LOG}:start`, + 'snapshot', + 'append', + `lock:${LOCK_NAMES.OPERATION_LOG}:end`, + ]); + }); + + it('should release, flush, and retry when an action lands before snapshot capture', async () => { + operationCaptureServiceSpy.getPendingCount.and.returnValues(1, 0); + + await service.handleServerMigration(defaultProvider); + + expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalledTimes(2); + expect(lockServiceSpy.request).toHaveBeenCalledTimes(2); + expect(stateSnapshotServiceSpy.getStateSnapshotAsync).toHaveBeenCalledTimes(1); + expect(opLogStoreSpy.append).toHaveBeenCalledTimes(1); + }); + describe('system-tag empty-state detection (tested via handleServerMigration)', () => { it('should identify system tags correctly', async () => { for (const systemTagId of SYSTEM_TAG_IDS) { diff --git a/src/app/op-log/sync/server-migration.service.ts b/src/app/op-log/sync/server-migration.service.ts index 12acfaa95a..5e8330b3aa 100644 --- a/src/app/op-log/sync/server-migration.service.ts +++ b/src/app/op-log/sync/server-migration.service.ts @@ -24,7 +24,11 @@ import { OpLog } from '../../core/log'; import { CLIENT_ID_PROVIDER } from '../util/client-id.provider'; import { DialogServerMigrationConfirmComponent } from './dialog-server-migration-confirm/dialog-server-migration-confirm.component'; import { hasMeaningfulStateData } from '../validation/has-meaningful-state-data.util'; -import { MODEL_CONFIGS } from '../model/model-config'; +import { AppDataComplete, MODEL_CONFIGS } from '../model/model-config'; +import { LockService } from './lock.service'; +import { OperationWriteFlushService } from './operation-write-flush.service'; +import { LOCK_NAMES } from '../core/operation-log.const'; +import { OperationCaptureService } from '../capture/operation-capture.service'; const MEANINGFUL_ENTITY_STATE_KEYS = new Set(['task', 'project', 'tag', 'note']); @@ -86,6 +90,9 @@ export class ServerMigrationService { private clientIdProvider = inject(CLIENT_ID_PROVIDER); private _matDialog = inject(MatDialog); private _userInputWaitState = inject(UserInputWaitStateService); + private lockService = inject(LockService); + private writeFlushService = inject(OperationWriteFlushService); + private operationCaptureService = inject(OperationCaptureService); /** * Checks if we're connecting to a new/empty server and handles migration if needed. @@ -198,104 +205,129 @@ export class ServerMigrationService { 'ServerMigrationService: Server migration detected. Creating full state SYNC_IMPORT.', ); - // Get current full state from NgRx store (async to include archives from IndexedDB) - // Cast to Record for validation compatibility - let currentState: Record = - (await this.stateSnapshotService.getStateSnapshotAsync()) as unknown as Record< - string, - unknown - >; + // Drain already-captured writes, then keep snapshot capture, validation, clock + // construction, and append behind the same mutation barrier. This makes the + // full-state operation's local seq an exact cutoff: every earlier op is in the + // snapshot, and any action captured while this runs is appended afterwards. + let retryForPendingCapture: boolean; + do { + retryForPendingCapture = false; + await this.writeFlushService.flushPendingWrites(); + await this.lockService.request(LOCK_NAMES.OPERATION_LOG, async () => { + // A reducer action can land in the tiny gap between flush() releasing + // this lock and our acquisition. Its pending counter increments + // synchronously, before the persistence effect waits for the lock. Do + // not take a snapshot that includes that reducer state but precedes its + // operation; release, drain it, and retry the cutoff instead. + if (this.operationCaptureService.getPendingCount() > 0) { + retryForPendingCapture = true; + return; + } - // Skip if local state is effectively empty - if (!hasServerMigrationStateData(currentState)) { - OpLog.warn('ServerMigrationService: Skipping SYNC_IMPORT - local state is empty.'); - return; - } + // Get current full state from NgRx store (async to include archives from IndexedDB) + // Cast to Record for validation compatibility + let currentState: Record = + (await this.stateSnapshotService.getStateSnapshotAsync()) as unknown as Record< + string, + unknown + >; - // Validate and repair state before creating SYNC_IMPORT - // This prevents corrupted state (e.g., orphaned menuTree references) from - // propagating to other clients via the full state import. - const validationResult = - await this.validateStateService.validateAndRepair(currentState); + // Skip if local state is effectively empty + if (!hasServerMigrationStateData(currentState)) { + OpLog.warn( + 'ServerMigrationService: Skipping SYNC_IMPORT - local state is empty.', + ); + return; + } - // If state is invalid and couldn't be repaired, abort - don't propagate corruption - if (!validationResult.isValid) { - OpLog.err( - 'ServerMigrationService: Cannot create SYNC_IMPORT - state validation failed.', - validationResult.error || validationResult.crossModelError, - ); - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.SERVER_MIGRATION_VALIDATION_FAILED, + // Validate and repair state before creating SYNC_IMPORT + // This prevents corrupted state (e.g., orphaned menuTree references) from + // propagating to other clients via the full state import. + const validationResult = + await this.validateStateService.validateAndRepair(currentState); + + // If state is invalid and couldn't be repaired, abort - don't propagate corruption + if (!validationResult.isValid) { + OpLog.err( + 'ServerMigrationService: Cannot create SYNC_IMPORT - state validation failed.', + validationResult.error || validationResult.crossModelError, + ); + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.SERVER_MIGRATION_VALIDATION_FAILED, + }); + return; + } + + // If state was repaired, use the repaired version + if (validationResult.repairedState) { + OpLog.warn( + 'ServerMigrationService: State repaired before creating SYNC_IMPORT', + validationResult.repairSummary, + ); + currentState = validationResult.repairedState; + + // Also update NgRx store with repaired state so local client is consistent + this.store.dispatch( + loadAllData({ + appDataComplete: validationResult.repairedState as AppDataComplete, + }), + ); + } + + // Get client ID + const clientId = await this.clientIdProvider.loadClientId(); + if (!clientId) { + OpLog.err( + 'ServerMigrationService: Cannot create SYNC_IMPORT - no client ID available.', + ); + return; + } + + // Build vector clock by merging ALL local operation clocks. + // This ensures the SYNC_IMPORT's clock dominates all pre-import ops, + // so when SyncImportFilterService compares them, all prior ops are + // LESS_THAN (not CONCURRENT) and can be properly filtered. + const allLocalOps = await this.opLogStore.getOpsAfterSeq(0); + let mergedClock = await this.vectorClockService.getCurrentVectorClock(); + for (const entry of allLocalOps) { + mergedClock = mergeVectorClocks(mergedClock, entry.op.vectorClock); + } + const newClock = limitVectorClockSize( + incrementVectorClock(mergedClock, clientId), + clientId, + ); + + OpLog.normal( + `ServerMigrationService: Merged ${allLocalOps.length} local op clocks into SYNC_IMPORT vector clock.`, + ); + + // Create SYNC_IMPORT operation with full state + // NOTE: Use raw state directly (not wrapped in appDataComplete). + // The snapshot endpoint expects raw state, and the hydrator handles + // both formats on extraction. + const op: Operation = { + id: uuidv7(), + actionType: ActionType.LOAD_ALL_DATA, + opType: OpType.SyncImport, + entityType: 'ALL', + payload: currentState, + clientId, + vectorClock: newClock, + timestamp: Date.now(), + schemaVersion: CURRENT_SCHEMA_VERSION, + syncImportReason: options?.syncImportReason ?? 'SERVER_MIGRATION', + }; + + // Append to operation log - will be uploaded via snapshot endpoint + await this.opLogStore.append(op, 'local'); + + OpLog.normal( + 'ServerMigrationService: Created SYNC_IMPORT operation for server migration. ' + + 'Will be uploaded immediately via follow-up upload.', + ); }); - return; - } - - // If state was repaired, use the repaired version - if (validationResult.repairedState) { - OpLog.warn( - 'ServerMigrationService: State repaired before creating SYNC_IMPORT', - validationResult.repairSummary, - ); - currentState = validationResult.repairedState; - - // Also update NgRx store with repaired state so local client is consistent - this.store.dispatch( - loadAllData({ appDataComplete: validationResult.repairedState as any }), - ); - } - - // Get client ID - const clientId = await this.clientIdProvider.loadClientId(); - if (!clientId) { - OpLog.err( - 'ServerMigrationService: Cannot create SYNC_IMPORT - no client ID available.', - ); - return; - } - - // Build vector clock by merging ALL local operation clocks. - // This ensures the SYNC_IMPORT's clock dominates all pre-import ops, - // so when SyncImportFilterService compares them, all prior ops are - // LESS_THAN (not CONCURRENT) and can be properly filtered. - const allLocalOps = await this.opLogStore.getOpsAfterSeq(0); - let mergedClock = await this.vectorClockService.getCurrentVectorClock(); - for (const entry of allLocalOps) { - mergedClock = mergeVectorClocks(mergedClock, entry.op.vectorClock); - } - const newClock = limitVectorClockSize( - incrementVectorClock(mergedClock, clientId), - clientId, - ); - - OpLog.normal( - `ServerMigrationService: Merged ${allLocalOps.length} local op clocks into SYNC_IMPORT vector clock.`, - ); - - // Create SYNC_IMPORT operation with full state - // NOTE: Use raw state directly (not wrapped in appDataComplete). - // The snapshot endpoint expects raw state, and the hydrator handles - // both formats on extraction. - const op: Operation = { - id: uuidv7(), - actionType: ActionType.LOAD_ALL_DATA, - opType: OpType.SyncImport, - entityType: 'ALL', - payload: currentState, - clientId, - vectorClock: newClock, - timestamp: Date.now(), - schemaVersion: CURRENT_SCHEMA_VERSION, - syncImportReason: options?.syncImportReason ?? 'SERVER_MIGRATION', - }; - - // Append to operation log - will be uploaded via snapshot endpoint - await this.opLogStore.append(op, 'local'); - - OpLog.normal( - 'ServerMigrationService: Created SYNC_IMPORT operation for server migration. ' + - 'Will be uploaded immediately via follow-up upload.', - ); + } while (retryForPendingCapture); } /** diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts index 2718eeddb4..0adea0c7d2 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts @@ -94,7 +94,7 @@ describe('SyncImportConflictGateService', () => { }); }); - it('should not produce dialog data when pending ops are config-only', async () => { + it('should produce dialog data when pending ops contain user config changes', async () => { const incomingSyncImport = createOperation(); const pendingConfigEntry = createEntry( createOperation({ @@ -113,8 +113,8 @@ describe('SyncImportConflictGateService', () => { const result = await service.checkIncomingFullStateConflict([incomingSyncImport]); expect(result.fullStateOp).toBe(incomingSyncImport); - expect(result.hasMeaningfulPending).toBeFalse(); - expect(result.dialogData).toBeUndefined(); + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); }); it('should not produce dialog data when pending task creates are startup example tasks', async () => { @@ -272,19 +272,27 @@ describe('SyncImportConflictGateService', () => { expect(opLogStoreSpy.hasSyncedOps).not.toHaveBeenCalled(); }); - it('should not consult sync history when there are no meaningful pending ops', async () => { + it('should not consult sync history when only startup example tasks are pending', async () => { const incomingSyncImport = createOperation(); - const pendingConfigEntry = createEntry( + const pendingExampleTaskEntry = createEntry( createOperation({ - id: 'local-config-update', - opType: OpType.Update, - entityType: 'GLOBAL_CONFIG', - entityId: 'sync', + id: 'local-example-task-create', + actionType: ActionType.TASK_SHARED_ADD, + opType: OpType.Create, + entityType: 'TASK', + entityId: 'example-task-1', + payload: { + actionPayload: { + task: { id: 'example-task-1' }, + isExampleTask: true, + }, + entityChanges: [], + }, clientId: 'client-A', vectorClock: { clientA: 1 }, }), ); - opLogStoreSpy.getUnsynced.and.resolveTo([pendingConfigEntry]); + opLogStoreSpy.getUnsynced.and.resolveTo([pendingExampleTaskEntry]); await service.checkIncomingFullStateConflict([incomingSyncImport]); @@ -383,25 +391,41 @@ describe('SyncImportConflictGateService', () => { expect(service.hasMeaningfulPendingOps([pendingCounter])).toBeTrue(); }); - it('should still treat MIGRATION/RECOVERY bookkeeping ops as not meaningful', async () => { + it('should treat MIGRATION/RECOVERY genesis batches as meaningful recovered user data', async () => { const pendingMigration = createEntry( createOperation({ id: 'local-genesis', actionType: '[Migration] Genesis' as ActionType, - opType: OpType.Create, + opType: OpType.Batch, entityType: 'MIGRATION', entityId: 'genesis', + payload: { task: { ids: ['recovered-task'] } }, clientId: 'client-A', vectorClock: { clientA: 1 }, }), ); - expect(service.hasMeaningfulPendingOps([pendingMigration])).toBeFalse(); + const pendingRecovery = createEntry( + createOperation({ + id: 'local-recovery', + actionType: '[Recovery] Data Import' as ActionType, + opType: OpType.Batch, + entityType: 'RECOVERY', + entityId: 'genesis', + payload: { project: { ids: ['recovered-project'] } }, + clientId: 'client-A', + vectorClock: { clientA: 2 }, + }), + { seq: 2 }, + ); + + expect(service.hasMeaningfulPendingOps([pendingMigration])).toBeTrue(); + expect(service.hasMeaningfulPendingOps([pendingRecovery])).toBeTrue(); }); }); describe('preCapturedPendingOps (piggyback-upload race)', () => { - it('should judge meaningfulness against the pre-upload snapshot, not the live pending set', async () => { + it('should judge meaningfulness against the union of the upload snapshot and live pending set', async () => { // Live pending set is empty — the op was accepted and marked synced during // the same upload round that piggybacked the import. opLogStoreSpy.getUnsynced.and.resolveTo([]); @@ -426,6 +450,54 @@ describe('SyncImportConflictGateService', () => { expect(result.dialogData).toBeDefined(); }); + it('should include meaningful work created after the upload snapshot', async () => { + const createdDuringUpload = createEntry( + createOperation({ + id: 'created-during-upload', + actionType: '[SimpleCounter] Increase Counter Today' as ActionType, + opType: OpType.Update, + entityType: 'SIMPLE_COUNTER', + entityId: 'counter-1', + clientId: 'client-A', + vectorClock: { clientA: 2 }, + }), + { seq: 2 }, + ); + opLogStoreSpy.getUnsynced.and.resolveTo([createdDuringUpload]); + + const result = await service.checkIncomingFullStateConflict([createOperation()], { + flushPendingWrites: true, + preCapturedPendingOps: [], + }); + + expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalled(); + expect(result.pendingOps).toEqual([createdDuringUpload]); + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); + }); + + it('should de-duplicate operations present in both upload and live snapshots', async () => { + const selectedForUpload = createEntry( + createOperation({ + id: 'same-op', + actionType: '[Task] Update task' as ActionType, + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-1', + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ); + opLogStoreSpy.getUnsynced.and.resolveTo([selectedForUpload]); + + const result = await service.checkIncomingFullStateConflict([createOperation()], { + preCapturedPendingOps: [selectedForUpload], + }); + + expect(result.pendingOps).toEqual([selectedForUpload]); + expect(result.dialogData?.filteredOpCount).toBe(1); + }); + it('should derive discardable example-task ids from the LIVE pending set', async () => { const exampleCreate = (id: string): OperationLogEntry => createEntry( diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index bc255685de..5a530864fa 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -10,17 +10,6 @@ import { OperationWriteFlushService } from './operation-write-flush.service'; import { SyncImportConflictData } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; -/** - * Pending ops on these entity types are startup/system noise, not user work: - * config writes happen automatically (defaults, migrations) and sync settings - * are intentionally local; MIGRATION/RECOVERY are bookkeeping genesis ops. - * Everything else — including MOV/BATCH ops and entities like TIME_TRACKING, - * SIMPLE_COUNTER, TASK_REPEAT_CFG, PLANNER, BOARD — is user work an incoming - * full-state import would silently discard (imports drop concurrent ops by - * design), so it must count as meaningful and trigger the conflict dialog. - */ -const DISCARDABLE_ENTITY_TYPES = new Set(['GLOBAL_CONFIG', 'MIGRATION', 'RECOVERY']); - export interface IncomingFullStateConflictGateResult { fullStateOp?: Operation; pendingOps: OperationLogEntry[]; @@ -45,10 +34,11 @@ export class SyncImportConflictGateService { private writeFlushService = inject(OperationWriteFlushService); /** - * Every pending op is user work unless it is on a DISCARDABLE_ENTITY_TYPES - * entity or is an onboarding example-task create. Full-state ops are always - * meaningful because applying a newer full-state op can invalidate their - * local import/repair semantics. + * Every pending op is user work unless it is an onboarding example-task create. + * Entity-wide exemptions are unsafe: GLOBAL_CONFIG contains synced preferences, + * while MIGRATION and RECOVERY genesis operations contain the user's full recovered + * database. Full-state ops are always meaningful because applying a newer full-state + * op can invalidate their local import/repair semantics. */ hasMeaningfulPendingOps(ops: OperationLogEntry[]): boolean { return ops.some((entry) => { @@ -58,16 +48,15 @@ export class SyncImportConflictGateService { if (isExampleTaskCreateOp(entry)) { return false; } - return !DISCARDABLE_ENTITY_TYPES.has(entry.op.entityType); + return true; }); } /** - * @param options.preCapturedPendingOps - Pending ops captured BEFORE the upload - * round started. The piggyback-upload path MUST pass this: by the time - * its gate runs, ops accepted in the same round were already marked - * synced, so a live getUnsynced() read would no longer see local work - * that the piggybacked import is about to discard. + * @param options.preCapturedPendingOps - Exact pending ops selected by the upload + * round. The piggyback path unions this snapshot with a live read so it + * protects both accepted work from that upload and work created while the + * network request was in flight. */ async checkIncomingFullStateConflict( incomingOps: Operation[], @@ -94,8 +83,10 @@ export class SyncImportConflictGateService { await this.writeFlushService.flushPendingWrites(); } - const pendingOps = - options.preCapturedPendingOps ?? (await this.opLogStore.getUnsynced()); + const livePendingOps = await this.opLogStore.getUnsynced(); + const pendingOps = options.preCapturedPendingOps + ? this._mergePendingOps(options.preCapturedPendingOps, livePendingOps) + : livePendingOps; const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); // Example-task ops that the caller may reject when it accepts the import silently. // When `hasMeaningfulPending` is true (real work pending alongside example tasks), @@ -105,9 +96,6 @@ export class SyncImportConflictGateService { // These must come from a LIVE read: with a pre-captured snapshot, example ops // accepted earlier in the same upload round are already marked synced and must // not be re-marked rejected by the caller. - const livePendingOps = options.preCapturedPendingOps - ? await this.opLogStore.getUnsynced() - : pendingOps; const discardablePendingOpIds = livePendingOps .filter(isExampleTaskCreateOp) .map((entry) => entry.op.id); @@ -148,4 +136,21 @@ export class SyncImportConflictGateService { }, }; } + + private _mergePendingOps( + uploadSnapshot: OperationLogEntry[], + livePendingOps: OperationLogEntry[], + ): OperationLogEntry[] { + const merged = [...uploadSnapshot]; + const seenOpIds = new Set(uploadSnapshot.map((entry) => entry.op.id)); + + for (const entry of livePendingOps) { + if (!seenOpIds.has(entry.op.id)) { + merged.push(entry); + seenOpIds.add(entry.op.id); + } + } + + return merged; + } } diff --git a/src/app/op-log/sync/ws-triggered-download.service.spec.ts b/src/app/op-log/sync/ws-triggered-download.service.spec.ts index 8309919a51..df86aa1f47 100644 --- a/src/app/op-log/sync/ws-triggered-download.service.spec.ts +++ b/src/app/op-log/sync/ws-triggered-download.service.spec.ts @@ -39,7 +39,7 @@ describe('WsTriggeredDownloadService', () => { mockProviderManager = jasmine.createSpyObj( 'SyncProviderManager', - ['getActiveProvider'], + ['getActiveProvider', 'setSyncStatus'], { isSyncInProgress: false, }, @@ -244,9 +244,6 @@ describe('WsTriggeredDownloadService', () => { // reset clears them) or leak into the next session. The service must // be its own session boundary. it('sets sync status ERROR when the download flips the validation latch', fakeAsync(() => { - if (mockProviderManager.setSyncStatus === undefined) { - mockProviderManager.setSyncStatus = jasmine.createSpy('setSyncStatus'); - } const latch = TestBed.inject(SyncSessionValidationService); mockSyncService.downloadRemoteOps.and.callFake(async () => { latch.setFailed(); @@ -262,9 +259,6 @@ describe('WsTriggeredDownloadService', () => { })); it('does not flag ERROR when the download leaves the latch reset', fakeAsync(() => { - if (mockProviderManager.setSyncStatus === undefined) { - mockProviderManager.setSyncStatus = jasmine.createSpy('setSyncStatus'); - } const latch = TestBed.inject(SyncSessionValidationService); latch._resetForTest(); @@ -276,6 +270,19 @@ describe('WsTriggeredDownloadService', () => { expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalledWith('ERROR'); })); + it('sets sync status ERROR when processing is blocked by an incompatible op', fakeAsync(() => { + mockSyncService.downloadRemoteOps.and.resolveTo({ + kind: 'blocked_incompatible', + }); + + service.start(); + notification$.next({ latestSeq: 1 }); + tick(500); + flushMicrotasks(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + })); + // Defense against stale latch from a prior path: the WS service opens its // own session, which resets the latch up front so the read at the end // reflects only this session's outcome. diff --git a/src/app/op-log/sync/ws-triggered-download.service.ts b/src/app/op-log/sync/ws-triggered-download.service.ts index 35b607fb65..a2ecfed4a7 100644 --- a/src/app/op-log/sync/ws-triggered-download.service.ts +++ b/src/app/op-log/sync/ws-triggered-download.service.ts @@ -151,6 +151,14 @@ export class WsTriggeredDownloadService implements OnDestroy { SyncLog.log(`WsTriggeredDownloadService: Download complete. kind=${result.kind}`); + if (result.kind === 'blocked_incompatible') { + SyncLog.warn( + 'WsTriggeredDownloadService: Download blocked by an incompatible operation', + ); + this._providerManager.setSyncStatus('ERROR'); + return; + } + if (this._sessionValidation.hasFailed()) { SyncLog.err( 'WsTriggeredDownloadService: Post-sync validation failed during WS download — reporting ERROR', diff --git a/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts b/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts index e56b8f87af..269f19ceed 100644 --- a/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts +++ b/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts @@ -56,7 +56,10 @@ const defineStorePortContract = ( } => { const applyOperationsSpy = jasmine .createSpy('applyOperations') - .and.resolveTo(result); + .and.callFake(async (ops: Operation[], options) => { + await options?.onReducersCommitted?.(ops); + return result; + }); return { applier: { applyOperations: applyOperationsSpy }, @@ -97,7 +100,10 @@ const defineStorePortContract = ( applier, }); - expect(applyOperationsSpy).toHaveBeenCalledOnceWith([newOp]); + expect(applyOperationsSpy).toHaveBeenCalledOnceWith( + [newOp], + jasmine.objectContaining({ onReducersCommitted: jasmine.any(Function) }), + ); expect(result.appendedOps).toEqual([newOp]); expect(result.skippedCount).toBe(1); expect(result.appliedOps).toEqual([newOp]); @@ -141,7 +147,7 @@ const defineStorePortContract = ( }); expect(result.failedOp).toEqual({ op: failedOp, error }); - expect(result.failedOpIds).toEqual([failedOp.id, remainingOp.id]); + expect(result.failedOpIds).toEqual([failedOp.id]); const appliedEntry = await store.getOpById(appliedOp.id); const failedEntry = await store.getOpById(failedOp.id); @@ -149,12 +155,16 @@ const defineStorePortContract = ( expect(appliedEntry?.applicationStatus).toBe('applied'); expect(failedEntry?.applicationStatus).toBe('failed'); expect(failedEntry?.retryCount).toBe(1); - expect(remainingEntry?.applicationStatus).toBe('failed'); - expect(remainingEntry?.retryCount).toBe(1); + expect(remainingEntry?.applicationStatus).toBe('archive_pending'); + expect(remainingEntry?.retryCount).toBeUndefined(); expect( (await store.getFailedRemoteOps()).map((entry) => entry.op.id).sort(), ).toEqual([failedOp.id, remainingOp.id].sort()); - expect(await store.getVectorClock()).toEqual({ clientA: 2 }); + expect(await store.getVectorClock()).toEqual({ + clientA: 2, + clientB: 4, + clientC: 6, + }); }); it('should clear older full-state ops and reset the vector clock after an applied remote import', async () => { From d2127402544f88423608401b849bcce303695338 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:09:35 +0200 Subject: [PATCH 06/47] test(e2e): resolve A's post-import conflict as USE_LOCAL in gate test Client A already synced a populated state, so importing a backup diverges from the server and A's own sync raises the sync-import conflict gate. syncAndWait() defaulted to USE_REMOTE, which discarded A's import before it reached the server, leaving Client B with nothing to conflict against, so the PHASE 5 conflict dialog never appeared (run 29090279243, SuperSync 1/6). Pass { useLocal: true } so A force-uploads the import as a new SYNC_IMPORT the server keeps, letting B's pending simple-counter change trigger the conflict dialog as intended. --- .../sync/supersync-conflict-gate-non-task-work.spec.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index 5c38ccea49..9eb4ad930e 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -141,10 +141,16 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( console.log('[Gate] Client B incremented the counter locally (pending, unsynced)'); // ===== PHASE 4: A imports a backup (SYNC_IMPORT) and syncs it ===== + // A already synced a populated state (PHASE 1), so the import diverges from + // the server: A's own sync raises the sync-import conflict gate against its + // pending import op. Resolve it as USE_LOCAL ({ useLocal: true }) so A force + // uploads the import as a NEW SYNC_IMPORT the server keeps. The default + // (USE_REMOTE) would discard A's import here, leaving the server unchanged + // and B with nothing to conflict against — so the PHASE 5 dialog never shows. const importPageA = new ImportPage(clientA.page); await importPageA.navigateToImportPage(); await importPageA.importBackupFile(ImportPage.getFixturePath('test-backup.json')); - await clientA.sync.syncAndWait(); + await clientA.sync.syncAndWait({ useLocal: true }); console.log('[Gate] Client A imported a backup and synced the SYNC_IMPORT'); // ===== PHASE 5: B syncs — the conflict dialog MUST appear ===== From eecf33a4e88b8cab613fa62b1d7eba33847910b7 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:38:16 +0200 Subject: [PATCH 07/47] fix(sync): remediate multi-agent review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - USE_REMOTE: fix guaranteed self-deadlock — flushPendingWrites() was called while already holding the non-reentrant sp_op_log lock its own Phase 2 re-acquires, so every "Use Server Data" hung 30s and aborted. New OperationWriteFlushService.flushThenRunExclusive owns the bounded flush->lock->recheck barrier; ServerMigrationService reuses it, which also bounds its previously unbounded snapshot-cutoff retry loop. - USE_REMOTE: make the rebuild crash-resumable. A raw_rebuild_incomplete META marker is set atomically with the baseline replacement and cleared after the replay commits; the next sync redoes the raw rebuild instead of the normal download (which excludes own ops server-side and would silently lose the un-replayed suffix). The resume keeps the first attempt's pre-replace backup instead of overwriting the single slot with the partial baseline. - Deferred actions: serialize overlapping drains (two concurrent drains could mint two ops for one user intent), stop the drain at the first transient failure instead of persisting successors out of order (the older edit would win LWW everywhere via clock inversion), abandon permanently invalid actions after one attempt (typed PermanentDeferredWriteError) instead of retrying + snacking on every sync forever, and latch the buffer-size warnings to once per stuck window (previously a blocking dev dialog per action past 100); drop the no-op 5000 "hard cap" tier. - writeOperation: post-append bookkeeping failures no longer propagate into the deferred retry loop, which re-appended the same action under a fresh op id (double-apply of additive payloads). - remote-apply: full-state cleanup retains every full-state op of the current batch, so an archive_pending import can no longer be deleted before its archive retry (startup replay would lose a change already visible at runtime). - Hydration: sanitize a malformed stored schemaVersion per-op instead of failing the whole boot into recovery; strict parsing (floor now 1, matching the server contract) stays on the receive/upload paths. - Server: request fingerprints are computed lazily — only when a dedup entry exists, and otherwise after the rate-limit/pre-quota gates but before uploadOps mutates the parsed ops. Fixes the pre-gate hashing of full payloads (authenticated DoS-cost regression) and repairs three sync-fixes retry tests to model true identical-body retries. - Delete dead code: failed-op-ids util (+spec) and MAX_VERSION_SKIP (the removed forward-compat band); fix the stale clock-merge parity comment. sync-core 210/210, shared-schema 50/50, super-sync-server 825/825, and all touched Angular specs pass. --- packages/shared-schema/src/index.ts | 6 +- packages/shared-schema/src/schema-version.ts | 9 +- .../services/request-deduplication.service.ts | 19 +- .../src/sync/sync.routes.ops-handler.ts | 29 +- .../src/sync/sync.routes.snapshot-handler.ts | 44 ++- .../src/sync/sync.service.ts | 8 +- .../request-deduplication.service.spec.ts | 27 +- .../tests/sync-compressed-body.routes.spec.ts | 2 +- .../tests/sync-fixes.spec.ts | 17 +- packages/sync-core/src/remote-apply.ts | 15 +- packages/sync-core/tests/remote-apply.spec.ts | 32 ++ .../op-log/apply/failed-op-ids.util.spec.ts | 27 -- src/app/op-log/apply/failed-op-ids.util.ts | 25 -- .../operation-capture.meta-reducer.spec.ts | 43 ++- .../capture/operation-capture.meta-reducer.ts | 43 +-- .../capture/operation-log.effects.spec.ts | 98 +++++- .../op-log/capture/operation-log.effects.ts | 156 +++++++--- src/app/op-log/core/errors/sync-errors.ts | 10 + src/app/op-log/persistence/db-keys.const.ts | 9 + .../operation-log-hydrator.service.spec.ts | 39 ++- .../operation-log-hydrator.service.ts | 37 ++- .../operation-log-store.service.spec.ts | 20 ++ .../operation-log-store.service.ts | 30 ++ .../persistence/schema-migration.service.ts | 10 +- .../sync/operation-log-sync.service.spec.ts | 122 ++++++++ .../op-log/sync/operation-log-sync.service.ts | 279 ++++++++++-------- .../operation-write-flush.service.spec.ts | 72 +++++ .../sync/operation-write-flush.service.ts | 47 +++ .../sync/server-migration.service.spec.ts | 21 +- .../op-log/sync/server-migration.service.ts | 219 +++++++------- .../schema-version-sync.integration.spec.ts | 6 - 31 files changed, 1069 insertions(+), 452 deletions(-) delete mode 100644 src/app/op-log/apply/failed-op-ids.util.spec.ts delete mode 100644 src/app/op-log/apply/failed-op-ids.util.ts diff --git a/packages/shared-schema/src/index.ts b/packages/shared-schema/src/index.ts index 27a706441a..ee17caaf40 100644 --- a/packages/shared-schema/src/index.ts +++ b/packages/shared-schema/src/index.ts @@ -1,9 +1,5 @@ // Schema version constants -export { - CURRENT_SCHEMA_VERSION, - MIN_SUPPORTED_SCHEMA_VERSION, - MAX_VERSION_SKIP, -} from './schema-version'; +export { CURRENT_SCHEMA_VERSION, MIN_SUPPORTED_SCHEMA_VERSION } from './schema-version'; // Types export type { diff --git a/packages/shared-schema/src/schema-version.ts b/packages/shared-schema/src/schema-version.ts index 92997bb47d..b98ad6097d 100644 --- a/packages/shared-schema/src/schema-version.ts +++ b/packages/shared-schema/src/schema-version.ts @@ -10,9 +10,6 @@ export const CURRENT_SCHEMA_VERSION = 2; */ export const MIN_SUPPORTED_SCHEMA_VERSION = 1; -/** - * Maximum version difference we tolerate before forcing an app update. - * If remote data is more than MAX_VERSION_SKIP versions ahead, - * the user must update their app. - */ -export const MAX_VERSION_SKIP = 3; +// NOTE: there is deliberately NO forward-compat band: any op from a NEWER +// schema version is blocked outright (the client cannot know how to interpret +// it), and the user is prompted to update the app. diff --git a/packages/super-sync-server/src/sync/services/request-deduplication.service.ts b/packages/super-sync-server/src/sync/services/request-deduplication.service.ts index 2ea3b1e496..b7fc4e2142 100644 --- a/packages/super-sync-server/src/sync/services/request-deduplication.service.ts +++ b/packages/super-sync-server/src/sync/services/request-deduplication.service.ts @@ -63,13 +63,19 @@ export class RequestDeduplicationService { /** * Check if a request has already been processed for this user + namespace. - * @returns Cached results if found and not expired, null otherwise. + * + * @param getFingerprint lazy fingerprint supplier — hashing the full request + * body is expensive (stable stringify + SHA-256 over up to multi-MB + * payloads), so it must only run when an entry for this requestId actually + * exists (genuine retries are rare) and never for first-time requests. + * @returns Cached results if found, not expired, and fingerprint-matching; + * null otherwise. */ checkDeduplication( userId: number, namespace: N, requestId: string, - fingerprint?: string, + getFingerprint?: () => string, ): DedupPayload | null { const key = this._key(userId, namespace, requestId); const entry = this.cache.get(key); @@ -80,7 +86,14 @@ export class RequestDeduplicationService { } // Defensive: keying already isolates namespaces, but verify before casting. if (entry.namespace !== namespace) return null; - if (fingerprint !== undefined && entry.fingerprint !== fingerprint) return null; + if (getFingerprint !== undefined) { + // A pre-fingerprint (legacy) entry cannot prove body equality — treat as + // a miss so the request is re-processed (safe: the server dedups ops by + // id anyway). + if (entry.fingerprint === undefined || entry.fingerprint !== getFingerprint()) { + return null; + } + } return entry.results as DedupPayload; } diff --git a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts index 24ed6eed35..56b260cdc5 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts @@ -87,9 +87,15 @@ export const uploadOpsHandler = async ( } const { ops, clientId, lastKnownServerSeq, requestId } = parseResult.data; - const requestFingerprint = requestId - ? createOpsRequestFingerprint(clientId, ops as unknown as Operation[]) - : undefined; + // Lazy + memoized: hashing the full ops payload is expensive and must not + // run before the rate-limit gate below, nor on first-time requests — the + // dedup check only invokes it when an entry for this requestId exists. + let memoizedFingerprint: string | undefined; + const getRequestFingerprint = (): string => + (memoizedFingerprint ??= createOpsRequestFingerprint( + clientId, + ops as unknown as Operation[], + )); const syncService = getSyncService(); Logger.info( @@ -119,7 +125,7 @@ export const uploadOpsHandler = async ( const cachedResults = syncService.checkOpsRequestDedup( userId, requestId, - requestFingerprint, + getRequestFingerprint, ); if (cachedResults) { Logger.info(`[user:${userId}] Returning cached results for request ${requestId}`); @@ -170,6 +176,14 @@ export const uploadOpsHandler = async ( } } + // Pin the fingerprint BEFORE processing: uploadOps mutates the parsed ops + // (e.g. vector-clock pruning), so hashing after it would never match the + // retry's pre-processing hash. Still after the rate-limit gate above, so a + // rate-limited client cannot burn CPU on it. + if (requestId) { + getRequestFingerprint(); + } + const results = await syncService.runWithStorageUsageLock( userId, async () => { @@ -222,7 +236,12 @@ export const uploadOpsHandler = async ( requestId && !results.some((r) => r.errorCode === SYNC_ERROR_CODES.INTERNAL_ERROR) ) { - syncService.cacheOpsRequestResults(userId, requestId, results, requestFingerprint); + syncService.cacheOpsRequestResults( + userId, + requestId, + results, + getRequestFingerprint(), + ); } const accepted = results.filter((r) => r.accepted).length; diff --git a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts index 8f59879597..281e695406 100644 --- a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts @@ -100,26 +100,31 @@ export const uploadSnapshotHandler = async ( requestId, } = snapshotRequest; const syncService = getSyncService(); - const requestFingerprint = requestId - ? createSnapshotRequestFingerprint({ - state, - clientId, - reason, - vectorClock, - schemaVersion, - isPayloadEncrypted, - syncImportReason, - opId, - isCleanSlate, - snapshotOpType, - }) - : undefined; + // Lazy + memoized: fingerprinting stable-stringifies + SHA-256-hashes the + // full (possibly multi-MB plaintext) snapshot state. It must not run + // before the pre-quota gate below, and first-time requests never need it — + // the dedup check only invokes it when an entry for this requestId exists + // (a genuine retry). + let memoizedFingerprint: string | undefined; + const getRequestFingerprint = (): string => + (memoizedFingerprint ??= createSnapshotRequestFingerprint({ + state, + clientId, + reason, + vectorClock, + schemaVersion, + isPayloadEncrypted, + syncImportReason, + opId, + isCleanSlate, + snapshotOpType, + })); if (requestId) { const cachedResponse = syncService.checkSnapshotRequestDedup( userId, requestId, - requestFingerprint, + getRequestFingerprint, ); if (cachedResponse) { Logger.info( @@ -156,6 +161,13 @@ export const uploadSnapshotHandler = async ( } } + // Pin the fingerprint BEFORE processing mutates anything, but AFTER the + // pre-quota gate above so quota-exhausted clients cannot burn CPU on the + // full-state hash. + if (requestId) { + getRequestFingerprint(); + } + // Reject duplicate SYNC_IMPORT before we acquire the per-user lock — a // duplicate rejection is cheap (one indexed lookup) and skipping the // lock lets concurrent legitimate clients keep moving. @@ -363,7 +375,7 @@ export const uploadSnapshotHandler = async ( userId, requestId, responseBody, - requestFingerprint, + getRequestFingerprint(), ); } diff --git a/packages/super-sync-server/src/sync/sync.service.ts b/packages/super-sync-server/src/sync/sync.service.ts index d4ac70c58d..113781f822 100644 --- a/packages/super-sync-server/src/sync/sync.service.ts +++ b/packages/super-sync-server/src/sync/sync.service.ts @@ -457,13 +457,13 @@ export class SyncService { checkOpsRequestDedup( userId: number, requestId: string, - fingerprint?: string, + getFingerprint?: () => string, ): UploadResult[] | null { return this.requestDeduplicationService.checkDeduplication( userId, 'ops', requestId, - fingerprint, + getFingerprint, ); } @@ -485,13 +485,13 @@ export class SyncService { checkSnapshotRequestDedup( userId: number, requestId: string, - fingerprint?: string, + getFingerprint?: () => string, ): SnapshotDedupResponse | null { return this.requestDeduplicationService.checkDeduplication( userId, 'snapshot', requestId, - fingerprint, + getFingerprint, ); } diff --git a/packages/super-sync-server/tests/request-deduplication.service.spec.ts b/packages/super-sync-server/tests/request-deduplication.service.spec.ts index 41acd782df..d64ca375a5 100644 --- a/packages/super-sync-server/tests/request-deduplication.service.spec.ts +++ b/packages/super-sync-server/tests/request-deduplication.service.spec.ts @@ -44,13 +44,36 @@ describe('RequestDeduplicationService', () => { service.cacheResults(1, 'ops', 'request-123', mockResults, 'fingerprint-a'); expect( - service.checkDeduplication(1, 'ops', 'request-123', 'fingerprint-a'), + service.checkDeduplication(1, 'ops', 'request-123', () => 'fingerprint-a'), ).toEqual(mockResults); expect( - service.checkDeduplication(1, 'ops', 'request-123', 'fingerprint-b'), + service.checkDeduplication(1, 'ops', 'request-123', () => 'fingerprint-b'), ).toBeNull(); }); + it('should not compute the fingerprint when no entry exists for the requestId', () => { + // The fingerprint hashes the full request body — first-time requests + // (the overwhelming majority) must not pay that cost. + const getFingerprint = vi.fn(() => 'fingerprint-a'); + + expect(service.checkDeduplication(1, 'ops', 'request-999', getFingerprint)).toBe( + null, + ); + expect(getFingerprint).not.toHaveBeenCalled(); + }); + + it('should treat a legacy entry without fingerprint as a miss when a fingerprint is supplied', () => { + const mockResults = createMockResults(1); + service.cacheResults(1, 'ops', 'request-123', mockResults); + + const getFingerprint = vi.fn(() => 'fingerprint-a'); + expect( + service.checkDeduplication(1, 'ops', 'request-123', getFingerprint), + ).toBeNull(); + // No entry fingerprint to compare against — the hash is never computed. + expect(getFingerprint).not.toHaveBeenCalled(); + }); + it('should return null for expired request', () => { const mockResults = createMockResults(1); service.cacheResults(1, 'ops', 'request-123', mockResults); diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index 60d282be76..0acbb475f0 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -810,7 +810,7 @@ describe('Sync compressed body routes', () => { expect(mocks.syncService.checkSnapshotRequestDedup).toHaveBeenCalledWith( 1, 'snapshot-v1-retry', - expect.any(String), + expect.any(Function), ); expect(mocks.syncService.checkOpsRequestDedup).not.toHaveBeenCalled(); expect(mocks.prisma.operation.findFirst).not.toHaveBeenCalled(); diff --git a/packages/super-sync-server/tests/sync-fixes.spec.ts b/packages/super-sync-server/tests/sync-fixes.spec.ts index a0a163bddd..96a1a65469 100644 --- a/packages/super-sync-server/tests/sync-fixes.spec.ts +++ b/packages/super-sync-server/tests/sync-fixes.spec.ts @@ -303,6 +303,9 @@ describe('Sync System Fixes', () => { const clientA = 'client-a'; const clientB = 'client-b'; const requestId = 'req-123'; + // A true retry resends the IDENTICAL body — the request fingerprint + // treats a same-requestId/different-body request as a cache miss. + const opA = createOp(clientA, { entityId: 'task-a' }); // Step 1: Client A uploads with requestId const firstResponse = await app.inject({ @@ -310,7 +313,7 @@ describe('Sync System Fixes', () => { url: '/api/sync/ops', headers: { authorization: `Bearer ${authToken}` }, payload: { - ops: [createOp(clientA, { entityId: 'task-a' })], + ops: [opA], clientId: clientA, lastKnownServerSeq: 0, requestId, @@ -339,7 +342,7 @@ describe('Sync System Fixes', () => { url: '/api/sync/ops', headers: { authorization: `Bearer ${authToken}` }, payload: { - ops: [createOp(clientA, { entityId: 'task-a' })], + ops: [opA], clientId: clientA, lastKnownServerSeq: clientASeq, requestId, @@ -366,6 +369,7 @@ describe('Sync System Fixes', () => { const clientA = 'client-a'; const clientB = 'client-b'; const requestId = 'req-456'; + const opA = createOp(clientA, { entityId: 'task-1' }); // Client A uploads await app.inject({ @@ -373,7 +377,7 @@ describe('Sync System Fixes', () => { url: '/api/sync/ops', headers: { authorization: `Bearer ${authToken}` }, payload: { - ops: [createOp(clientA, { entityId: 'task-1' })], + ops: [opA], clientId: clientA, lastKnownServerSeq: 0, requestId, @@ -397,7 +401,7 @@ describe('Sync System Fixes', () => { url: '/api/sync/ops', headers: { authorization: `Bearer ${authToken}` }, payload: { - ops: [createOp(clientA, { entityId: 'task-1' })], + ops: [opA], clientId: clientA, lastKnownServerSeq: 0, requestId, @@ -488,6 +492,7 @@ describe('Sync System Fixes', () => { it('should return cached results even when quota would be exceeded on retry', async () => { const clientId = uuidv7(); const requestId = uuidv7(); + const op = createOp(clientId, { entityId: 'task-1' }); // First request with requestId const firstResponse = await app.inject({ @@ -495,7 +500,7 @@ describe('Sync System Fixes', () => { url: '/api/sync/ops', headers: { authorization: `Bearer ${authToken}` }, payload: { - ops: [createOp(clientId, { entityId: 'task-1' })], + ops: [op], clientId, lastKnownServerSeq: 0, requestId, @@ -513,7 +518,7 @@ describe('Sync System Fixes', () => { url: '/api/sync/ops', headers: { authorization: `Bearer ${authToken}` }, payload: { - ops: [createOp(clientId, { entityId: 'task-1' })], + ops: [op], clientId, lastKnownServerSeq: 0, requestId, diff --git a/packages/sync-core/src/remote-apply.ts b/packages/sync-core/src/remote-apply.ts index 3cc8f32f77..9676c3a311 100644 --- a/packages/sync-core/src/remote-apply.ts +++ b/packages/sync-core/src/remote-apply.ts @@ -74,7 +74,8 @@ const emptyRemoteApplyResult = < * 2. apply only the newly appended ops through the host applier; * 3. mark successfully applied seqs; * 4. merge applied remote vector clocks; - * 5. retain only the newest applied full-state ops when configured; + * 5. after applying a full-state op, clear older full-state ops while + * retaining every full-state op of this batch (incl. archive_pending); * 6. mark the failed op and remaining ops as failed on partial error — their * reducer effects committed with the bulk dispatch; only their archive side * effects are outstanding (see ApplyOperationsResult.failedOp). @@ -145,8 +146,16 @@ export const applyRemoteOperations = async < .map((op) => op.id); if (appliedFullStateOpIds.length > 0) { - clearedFullStateOpCount = - await store.clearFullStateOpsExcept(appliedFullStateOpIds); + // Exclude every full-state op of THIS batch, not only the applied ones: + // a later full-state op whose reducers committed but whose archive + // handling failed is still archive_pending/failed and must survive + // cleanup so the retry path can finish it. Deleting it here would let + // markFailed miss it and lose a change already visible at runtime on + // the next startup replay. + const batchFullStateOpIds = appendResult.writtenOps + .filter(isFullStateOperation) + .map((op) => op.id); + clearedFullStateOpCount = await store.clearFullStateOpsExcept(batchFullStateOpIds); } } diff --git a/packages/sync-core/tests/remote-apply.spec.ts b/packages/sync-core/tests/remote-apply.spec.ts index 9aa8e29a06..3c71023caa 100644 --- a/packages/sync-core/tests/remote-apply.spec.ts +++ b/packages/sync-core/tests/remote-apply.spec.ts @@ -112,6 +112,38 @@ describe('applyRemoteOperations', () => { expect(result.clearedFullStateOpCount).toBe(3); }); + it('cleanup preserves a batch full-state op whose archive handling failed (archive_pending)', async () => { + // Both imports reducer-committed; the LATER one failed only its archive + // side effects. Cleanup keyed on the applied one must not delete the + // archive_pending entry, or markFailed misses it and the change is lost + // from the next startup replay. + const appliedImport = createOperation('sync-import-1', 'SYNC_IMPORT'); + const failedImport = createOperation('sync-import-2', 'SYNC_IMPORT'); + const error = new Error('archive failure'); + const store = createStore({ + seqs: [1, 2], + writtenOps: [appliedImport, failedImport], + skippedCount: 0, + }); + const applier = createApplier({ + appliedOps: [appliedImport], + failedOp: { op: failedImport, error }, + }); + + await applyRemoteOperations({ + ops: [appliedImport, failedImport], + store, + applier, + isFullStateOperation: (op) => op.opType === 'SYNC_IMPORT', + }); + + expect(store.clearFullStateOpsExcept).toHaveBeenCalledWith([ + 'sync-import-1', + 'sync-import-2', + ]); + expect(store.markFailed).toHaveBeenCalledWith(['sync-import-2']); + }); + it('checkpoints the whole reducer batch but charges only the attempted archive failure', async () => { const op1 = createOperation('op-1'); const op2 = createOperation('op-2'); diff --git a/src/app/op-log/apply/failed-op-ids.util.spec.ts b/src/app/op-log/apply/failed-op-ids.util.spec.ts deleted file mode 100644 index b3e10d6b09..0000000000 --- a/src/app/op-log/apply/failed-op-ids.util.spec.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { getFailedOpIdsFromBatch } from './failed-op-ids.util'; -import { Operation } from '../core/operation.types'; - -const op = (id: string): Operation => ({ id }) as Operation; - -describe('getFailedOpIdsFromBatch', () => { - it('returns the failed op and every op after it (slice-from-failure)', () => { - const ops = [op('a'), op('b'), op('c'), op('d')]; - expect(getFailedOpIdsFromBatch(ops, op('b'))).toEqual(['b', 'c', 'd']); - }); - - it('returns just the last op when it is the one that failed', () => { - const ops = [op('a'), op('b'), op('c')]; - expect(getFailedOpIdsFromBatch(ops, op('c'))).toEqual(['c']); - }); - - it('returns all ops when the first one failed', () => { - const ops = [op('a'), op('b'), op('c')]; - expect(getFailedOpIdsFromBatch(ops, op('a'))).toEqual(['a', 'b', 'c']); - }); - - it('falls back to only the failed op when it is not in the batch (defensive -1 guard)', () => { - // Guards against slice(-1) wrongly selecting just the last op. - const ops = [op('a'), op('b'), op('c')]; - expect(getFailedOpIdsFromBatch(ops, op('x'))).toEqual(['x']); - }); -}); diff --git a/src/app/op-log/apply/failed-op-ids.util.ts b/src/app/op-log/apply/failed-op-ids.util.ts deleted file mode 100644 index bc74863c8c..0000000000 --- a/src/app/op-log/apply/failed-op-ids.util.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Operation } from '../core/operation.types'; - -/** - * Given the ops handed to a single `applyOperations()` batch in causal (seq) - * order and the op whose archive side effect threw, return the ids of every op - * that stays unapplied: the failed op plus everything after it. - * - * The batch applier (`replayOperationBatch`) stops at the first failing archive - * side effect, so ops after `failedOp` were never processed and must be retried - * together with it. This is the slice-from-failure handling shared by every - * caller that applies a batch and then marks the remainder failed (the remote - * apply path, conflict resolution, and the failed-op retry on hydration). - * - * The `-1` guard is defensive: `failedOp` always originates from `opsToApply`, - * but if it somehow isn't found we mark only the failed op rather than letting - * `slice(-1)` wrongly select just the last op. - */ -export const getFailedOpIdsFromBatch = ( - opsToApply: Operation[], - failedOp: Operation, -): string[] => { - const failedIndex = opsToApply.findIndex((op) => op.id === failedOp.id); - const stillFailed = failedIndex === -1 ? [failedOp] : opsToApply.slice(failedIndex); - return stillFailed.map((op) => op.id); -}; diff --git a/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts b/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts index 44a57dc990..d395ec010e 100644 --- a/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts +++ b/src/app/op-log/capture/operation-capture.meta-reducer.spec.ts @@ -8,7 +8,6 @@ import { getDeferredActions, clearDeferredActions, DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD, - DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP, } from './operation-capture.meta-reducer'; import { OperationCaptureService } from './operation-capture.service'; import { Action } from '@ngrx/store'; @@ -329,25 +328,43 @@ describe('operationCaptureMetaReducer', () => { expect(getDeferredActions()).toEqual(actions); }); - it('should retain every accepted action past the pathological warning cap', () => { + it('should fire the reload warning only once per stuck window (no per-action spam)', () => { const confirmSpy = spyNativeDialogs(); - const actions = createManyActions(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP); - const dropErrorCalls = (): unknown[][] => + const actions = createManyActions( + DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD + 200, + ); + const reloadWarningCalls = (): unknown[][] => confirmSpy.calls .allArgs() - .filter((args) => /pathological cap/.test(String(args[0]))); + .filter((args) => /consider reloading/.test(String(args[0]))); actions.forEach((a) => bufferDeferredAction(a)); - expect(dropErrorCalls().length).toBe(0); - const extraAction = createMockAction({ type: '[Test] Extra Action' }); - bufferDeferredAction(extraAction); - expect(dropErrorCalls().length).toBe(1); + // In dev builds devError opens a blocking dialog; firing it on every + // buffered action past the threshold would freeze the session exactly + // when sync is stuck. + expect(reloadWarningCalls().length).toBe(1); + expect(getDeferredActions()).toEqual(actions); + }); - const buffered = getDeferredActions(); - expect(buffered.length).toBe(DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP + 1); - expect(buffered[0]).toBe(actions[0]); - expect(buffered[buffered.length - 1]).toBe(extraAction); + it('should warn again after the buffer drained and a new stuck window crosses the threshold', () => { + const confirmSpy = spyNativeDialogs(); + const reloadWarningCalls = (): unknown[][] => + confirmSpy.calls + .allArgs() + .filter((args) => /consider reloading/.test(String(args[0]))); + + createManyActions(DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD).forEach((a) => + bufferDeferredAction(a), + ); + expect(reloadWarningCalls().length).toBe(1); + + clearDeferredActions(); + + createManyActions(DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD).forEach((a) => + bufferDeferredAction(a), + ); + expect(reloadWarningCalls().length).toBe(2); }); }); }); diff --git a/src/app/op-log/capture/operation-capture.meta-reducer.ts b/src/app/op-log/capture/operation-capture.meta-reducer.ts index eefd5476ee..775d053909 100644 --- a/src/app/op-log/capture/operation-capture.meta-reducer.ts +++ b/src/app/op-log/capture/operation-capture.meta-reducer.ts @@ -148,41 +148,38 @@ const DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD = 10; export const DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD = 100; /** - * Pathological high-water mark for the deferred actions buffer. ~100 buffered actions are - * plausibly reachable by a real user during a stuck/very long remote-apply - * window, so we must never drop there (see reload-warning threshold above, - * which tells the user to reload long before this mark can be hit through - * real interaction). Crossing it logs loudly but still cannot discard an - * action whose reducer change has already been accepted. - * Exported for tests. + * Highest threshold already warned about in the current stuck window; each + * warning fires once per crossing instead of on every buffered action (in dev + * builds devError opens a blocking dialog, so per-action firing would freeze + * the session exactly when sync is stuck). Reset when the buffer drains. */ -export const DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP = 5000; +let highestWarnedThreshold = 0; /** * Buffers an action for processing after sync completes. * Called by the meta-reducer when a persistent action arrives during sync. + * + * Never drops an accepted persistent action: its reducer mutation already + * exists in NgRx, so removal here would create permanent unsyncable state. */ export const bufferDeferredAction = (action: PersistentAction): void => { - // Never drop an accepted persistent action: its reducer mutation already - // exists in NgRx, so removal here would create permanent unsyncable state. - // The cap remains a loud runaway-loop diagnostic, not a lossy flow-control - // mechanism. - if (deferredActions.length >= DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP) { - devError( - `[operationCaptureMetaReducer] Deferred actions buffer exceeded pathological cap of ${DEFERRED_ACTIONS_PATHOLOGICAL_HARD_CAP} items. ` + - 'Retaining all actions to preserve sync correctness. Likely a runaway dispatch loop; reload the app.', - ); - } - deferredActions.push(action); deferredActionSet.add(action); - if (deferredActions.length >= DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD) { + if ( + deferredActions.length >= DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD && + highestWarnedThreshold < DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD + ) { + highestWarnedThreshold = DEFERRED_ACTIONS_RELOAD_WARNING_THRESHOLD; devError( `[operationCaptureMetaReducer] Deferred actions buffer has ${deferredActions.length} items. ` + `Sync may be stuck - consider reloading the app. Nothing is dropped; actions remain buffered.`, ); - } else if (deferredActions.length > DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD) { + } else if ( + deferredActions.length > DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD && + highestWarnedThreshold < DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD + ) { + highestWarnedThreshold = DEFERRED_ACTIONS_SOFT_WARNING_THRESHOLD; devError( `[operationCaptureMetaReducer] Deferred actions buffer has ${deferredActions.length} items - sync may be stuck or taking too long`, ); @@ -204,6 +201,9 @@ export const acknowledgeDeferredAction = (action: PersistentAction): void => { } deferredActions.splice(index, 1); deferredActionSet.delete(action); + if (deferredActions.length === 0) { + highestWarnedThreshold = 0; + } }; /** @@ -217,6 +217,7 @@ export const clearDeferredActions = (): void => { deferredActionSet.delete(action); } deferredActions = []; + highestWarnedThreshold = 0; }; /** diff --git a/src/app/op-log/capture/operation-log.effects.spec.ts b/src/app/op-log/capture/operation-log.effects.spec.ts index b8881bb659..cba00d4ac5 100644 --- a/src/app/op-log/capture/operation-log.effects.spec.ts +++ b/src/app/op-log/capture/operation-log.effects.spec.ts @@ -881,29 +881,105 @@ describe('OperationLogEffects', () => { expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(3); }); - it('should keep an exhausted deferred write queued while acknowledging later successes', async () => { + it('should stop the drain at an exhausted transient failure, keeping it AND its successors queued in order', async () => { + // Persisting a successor before the failed action would record them in + // reversed order with inverted vector clocks — the OLDER same-entity + // edit would win LWW on every client. const failedAction = createPersistentAction(ActionType.TASK_SHARED_ADD); - const successfulAction = createPersistentAction(ActionType.TASK_SHARED_UPDATE); + const successorAction = createPersistentAction(ActionType.TASK_SHARED_UPDATE); bufferDeferredAction(failedAction); - bufferDeferredAction(successfulAction); - let attempts = 0; - mockOpLogStore.appendWithVectorClockUpdate.and.callFake(() => { - attempts++; - return attempts <= 3 - ? Promise.reject(new Error('persistent failure')) - : Promise.resolve(1); - }); + bufferDeferredAction(successorAction); + mockOpLogStore.appendWithVectorClockUpdate.and.rejectWith( + new Error('transient failure'), + ); await effects.processDeferredActions(); - expect(getDeferredActions()).toEqual([failedAction]); + // Only the failed action was attempted (3 retries); the successor was + // never written out of order and both remain buffered. + expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(3); + expect(getDeferredActions()).toEqual([failedAction, successorAction]); + mockOpLogStore.appendWithVectorClockUpdate.calls.reset(); mockOpLogStore.appendWithVectorClockUpdate.and.resolveTo(2); await effects.processDeferredActions(); + // Next window drains both in the original order. + const calls = mockOpLogStore.appendWithVectorClockUpdate.calls.all(); + expect(calls.length).toBe(2); + expect(calls[0].args[0].actionType).toBe(ActionType.TASK_SHARED_ADD); + expect(calls[1].args[0].actionType).toBe(ActionType.TASK_SHARED_UPDATE); expect(getDeferredActions()).toEqual([]); }); + it('should abandon a permanently invalid deferred action and continue with its successors', async () => { + // Invalid entity identifiers are deterministic: retrying every sync + // window forever (with a sticky error snack each time) can never succeed. + const invalidAction = createPersistentAction(ActionType.TASK_SHARED_ADD); + (invalidAction.meta as { entityId?: string }).entityId = ''; + const validAction = createPersistentAction(ActionType.TASK_SHARED_UPDATE); + bufferDeferredAction(invalidAction); + bufferDeferredAction(validAction); + + await effects.processDeferredActions(); + + // Invalid action: single attempt, no retries, abandoned. Valid successor + // persisted (its relative order w.r.t. an unpersistable action is moot). + expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(1); + expect( + mockOpLogStore.appendWithVectorClockUpdate.calls.mostRecent().args[0].actionType, + ).toBe(ActionType.TASK_SHARED_UPDATE); + expect(getDeferredActions()).toEqual([]); + expect(mockSnackService.open).toHaveBeenCalledWith( + jasmine.objectContaining({ msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED }), + ); + }); + + it('should serialize overlapping drains so one buffered action is persisted exactly once', async () => { + // getDeferredActions() is a non-destructive snapshot: without + // serialization two concurrent drains would both see the same + // unacknowledged action and mint two ops for one user intent. + const action = createPersistentAction(ActionType.TASK_SHARED_UPDATE); + bufferDeferredAction(action); + + let resolveFirstWrite!: (seq: number) => void; + mockOpLogStore.appendWithVectorClockUpdate.and.returnValue( + new Promise((resolve) => { + resolveFirstWrite = resolve; + }), + ); + + const firstDrain = effects.processDeferredActions(); + const secondDrain = effects.processDeferredActions(); + // Let the first drain reach its (pending) write before releasing it. + await new Promise((resolve) => setTimeout(resolve, 0)); + resolveFirstWrite(1); + await Promise.all([firstDrain, secondDrain]); + + expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(1); + expect(getDeferredActions()).toEqual([]); + }); + + it('should not re-append a deferred action when post-append bookkeeping fails', async () => { + // After appendWithVectorClockUpdate commits, a bookkeeping throw (e.g. + // getCompactionCounter) must not bubble into the retry loop — that would + // append the same user action again under a fresh op id and double-apply + // additive payloads on every client. + const action = createPersistentAction(ActionType.TASK_SHARED_UPDATE); + bufferDeferredAction(action); + mockOpLogStore.getCompactionCounter.and.rejectWith( + new Error('bookkeeping failure'), + ); + + await effects.processDeferredActions(); + + expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(1); + expect(getDeferredActions()).toEqual([]); + expect(mockSnackService.open).not.toHaveBeenCalledWith( + jasmine.objectContaining({ msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED }), + ); + }); + it('should use fresh vector clock for deferred actions', async () => { // Set up vector clock to return a specific value mockVectorClockService.getCurrentVectorClock.and.returnValue( diff --git a/src/app/op-log/capture/operation-log.effects.ts b/src/app/op-log/capture/operation-log.effects.ts index 8b68de3511..68c4b70165 100644 --- a/src/app/op-log/capture/operation-log.effects.ts +++ b/src/app/op-log/capture/operation-log.effects.ts @@ -4,7 +4,10 @@ import type { DeferredLocalActionsPort } from '@sp/sync-core'; import { ALL_ACTIONS } from '../../util/local-actions.token'; import { concatMap, filter } from 'rxjs/operators'; import { LockService } from '../sync/lock.service'; -import { LockAcquisitionTimeoutError } from '../core/errors/sync-errors'; +import { + LockAcquisitionTimeoutError, + PermanentDeferredWriteError, +} from '../core/errors/sync-errors'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { isPersistentAction, @@ -63,6 +66,12 @@ export class OperationLogEffects implements DeferredLocalActionsPort { * Initialized lazily from persisted value on first operation. */ private inMemoryCompactionCounter: number | null = null; + + /** + * Serialization chain for processDeferredActions — overlapping drains would + * double-persist the same buffered action (see processDeferredActions). + */ + private _deferredProcessingChain: Promise = Promise.resolve(); /** * Dedupe timestamp for the storage-quota snackbar. #7700: when quota fires * inside the deferred-action retry loop, the retry loop calls handleQuotaExceeded @@ -182,12 +191,17 @@ export class OperationLogEffects implements DeferredLocalActionsPort { if (!isBulkAllOperation && !hasValidEntityId && !hasValidEntityIds) { // No queue bookkeeping needed here: the effect wrapper's `finally` // decrements the pending counter for this action even on early return. + if (isDeferredWrite) { + // Typed throw BEFORE devError: in dev builds devError itself can throw + // a generic Error, which the deferred drain would misclassify as a + // transient (retryable) failure. The drain logs the abandonment loudly. + throw new PermanentDeferredWriteError( + `Deferred action ${action.type} has invalid entity identifiers.`, + ); + } devError( `[OperationLogEffects] Action ${action.type} has invalid entityId/entityIds (${action.meta.entityId}) - skipping persistence`, ); - if (isDeferredWrite) { - throw new Error(`Deferred action ${action.type} has invalid entity identifiers.`); - } return; } @@ -300,7 +314,9 @@ export class OperationLogEffects implements DeferredLocalActionsPort { }, }); if (isDeferredWrite) { - throw new Error(`Deferred action ${action.type} has an invalid payload.`); + throw new PermanentDeferredWriteError( + `Deferred action ${action.type} has an invalid payload.`, + ); } return; // Skip persisting invalid operation } @@ -320,32 +336,44 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // The op.vectorClock already contains the incremented clock (from newClock above). await this.opLogStore.appendWithVectorClockUpdate(op, 'local'); - // Mark that we have pending ops (not yet uploaded) for UI indicator - this.superSyncStatusService.updatePendingOpsStatus(true); + // The op is durably committed past this point. Bookkeeping failures + // below must NOT propagate: a throw would send the deferred retry loop + // back through writeOperation, appending the same user action again + // under a fresh op id — additive payloads (time tracking, counters) + // would then double-apply on every client. + try { + // Mark that we have pending ops (not yet uploaded) for UI indicator + this.superSyncStatusService.updatePendingOpsStatus(true); - // Track write count for high-volume debugging - this.writeCount++; - if (this.writeCount % 50 === 0) { - OpLog.normal( - `OperationLogEffects: Wrote ${this.writeCount} operations to IndexedDB`, + // Track write count for high-volume debugging + this.writeCount++; + if (this.writeCount % 50 === 0) { + OpLog.normal( + `OperationLogEffects: Wrote ${this.writeCount} operations to IndexedDB`, + ); + } + + // 1b. Trigger immediate upload to SuperSync (async, non-blocking) + this.immediateUploadService.trigger(); + + // 2. Check if compaction is needed + // PERF: Use in-memory counter instead of IndexedDB transaction on every operation. + // Initialize from persisted value on first use (for crash recovery). + if (this.inMemoryCompactionCounter === null) { + this.inMemoryCompactionCounter = await this.opLogStore.getCompactionCounter(); + } + this.inMemoryCompactionCounter++; + if (this.inMemoryCompactionCounter >= COMPACTION_THRESHOLD) { + // Trigger compaction asynchronously (don't block write operation) + // Counter is reset in compaction service on success + this.triggerCompaction(); + } + } catch (bookkeepingError) { + OpLog.err( + 'OperationLogEffects: Post-append bookkeeping failed (operation is already durable; not retrying the append)', + { name: (bookkeepingError as Error | undefined)?.name }, ); } - - // 1b. Trigger immediate upload to SuperSync (async, non-blocking) - this.immediateUploadService.trigger(); - - // 2. Check if compaction is needed - // PERF: Use in-memory counter instead of IndexedDB transaction on every operation. - // Initialize from persisted value on first use (for crash recovery). - if (this.inMemoryCompactionCounter === null) { - this.inMemoryCompactionCounter = await this.opLogStore.getCompactionCounter(); - } - this.inMemoryCompactionCounter++; - if (this.inMemoryCompactionCounter >= COMPACTION_THRESHOLD) { - // Trigger compaction asynchronously (don't block write operation) - // Counter is reset in compaction service on success - this.triggerCompaction(); - } }; if (options.callerHoldsOperationLogLock) { @@ -365,6 +393,11 @@ export class OperationLogEffects implements DeferredLocalActionsPort { } catch (e) { // 4.1.1 Error Handling for Optimistic Updates OpLog.err('OperationLogEffects: Failed to persist operation', e); + if (e instanceof PermanentDeferredWriteError) { + // Deterministic invalidity, already surfaced via its own snack — + // rethrow untouched so the deferred drain abandons instead of retrying. + throw e; + } if (e instanceof LockAcquisitionTimeoutError) { // #7700: do NOT silently swallow lock timeouts. Pre-fix, a reentrant // sp_op_log timeout was caught here, the user got a snackbar, and the @@ -620,6 +653,28 @@ export class OperationLogEffects implements DeferredLocalActionsPort { * Called after sync operations are applied and remote op bookkeeping is complete. */ async processDeferredActions(options: WriteOperationOptions = {}): Promise { + // Serialize overlapping invocations: getDeferredActions() is a + // non-destructive snapshot, so two concurrent drains (e.g. a WS-triggered + // download finishing while a piggyback apply flushes — they run under + // different locks) would both see the same unacknowledged action and each + // mint an op with a fresh id for it. One user intent = one op; additive + // payloads would otherwise double-apply on every client. Chaining makes + // the second caller wait and re-snapshot after the first finishes. + const run = this._deferredProcessingChain.then(() => + this._processDeferredActionsImpl(options), + ); + // Keep the chain alive even if this run rejects; errors still surface to + // this invocation's caller via `run`. + this._deferredProcessingChain = run.then( + () => undefined, + () => undefined, + ); + return run; + } + + private async _processDeferredActionsImpl( + options: WriteOperationOptions, + ): Promise { const deferredActions = getDeferredActions(); if (deferredActions.length === 0) { return; @@ -647,6 +702,10 @@ export class OperationLogEffects implements DeferredLocalActionsPort { break; } catch (e) { lastError = e; + if (e instanceof PermanentDeferredWriteError) { + // Deterministic invalidity — retrying cannot succeed. + break; + } if (attempt < MAX_RETRIES - 1) { // Exponential backoff: 100ms, 200ms, 400ms const delay = BASE_DELAY_MS * Math.pow(2, attempt); @@ -659,19 +718,38 @@ export class OperationLogEffects implements DeferredLocalActionsPort { } } - if (!success) { - failedCount++; - // Log error after all retries exhausted, continue processing remaining actions - OpLog.err( - `OperationLogEffects: Failed to process deferred action after ${MAX_RETRIES} retries`, - { - actionType: action.type, - errorName: (lastError as Error | undefined)?.name, - }, - ); - } else { + if (success) { acknowledgeDeferredAction(action); + continue; } + + failedCount++; + + if (lastError instanceof PermanentDeferredWriteError) { + // Terminal: the action can never be persisted (invalid identifiers or + // payload). Abandon it — keeping it queued would retry it with backoff + // and re-raise the failure snack on every future sync window while the + // in-memory buffer's "durability" ends at the advised reload anyway. + acknowledgeDeferredAction(action); + OpLog.err('OperationLogEffects: Abandoning permanently invalid deferred action', { + actionType: action.type, + }); + continue; + } + + // Transient failure: STOP the drain here instead of persisting later + // actions first. Writing a successor now and this action on a future + // drain would record them in reversed order with inverted vector clocks, + // so the OLDER same-entity edit would win LWW on every client. The whole + // suffix stays buffered and is retried in order on the next window. + OpLog.err( + `OperationLogEffects: Deferred action failed after ${MAX_RETRIES} retries; keeping it and all later actions queued in order`, + { + actionType: action.type, + errorName: (lastError as Error | undefined)?.name, + }, + ); + break; } // Show notification if any actions failed diff --git a/src/app/op-log/core/errors/sync-errors.ts b/src/app/op-log/core/errors/sync-errors.ts index b7fa8d71a8..f6f33e9002 100644 --- a/src/app/op-log/core/errors/sync-errors.ts +++ b/src/app/op-log/core/errors/sync-errors.ts @@ -112,6 +112,16 @@ export class UnknownSyncStateError extends Error { override name = 'UnknownSyncStateError'; } +/** + * A deferred action can never be persisted (invalid entity identifiers or an + * invalid operation payload) — a deterministic condition, not a transient + * I/O failure. processDeferredActions abandons such actions instead of + * retrying them on every sync window forever. + */ +export class PermanentDeferredWriteError extends Error { + override name = 'PermanentDeferredWriteError'; +} + // -----ENCRYPTION & COMPRESSION---- export class DecryptNoPasswordError extends AdditionalLogErrorBase { override name = 'DecryptNoPasswordError'; diff --git a/src/app/op-log/persistence/db-keys.const.ts b/src/app/op-log/persistence/db-keys.const.ts index 6d96c32a41..7d59f6319f 100644 --- a/src/app/op-log/persistence/db-keys.const.ts +++ b/src/app/op-log/persistence/db-keys.const.ts @@ -45,6 +45,15 @@ export const BACKUP_KEY = 'backup' as const; /** Meta key for derived full-state operation refs */ export const FULL_STATE_OPS_META_KEY = 'full_state_ops' as const; +/** + * Meta key marking an interrupted USE_REMOTE raw rebuild: set atomically with + * the baseline replacement, cleared after the server-history replay commits. + * While set, the next sync must redo the raw (own-ops-included) rebuild — + * the normal download path excludes this client's own ops server-side, so an + * interrupted replay would otherwise silently lose them. + */ +export const RAW_REBUILD_INCOMPLETE_META_KEY = 'raw_rebuild_incomplete' as const; + /** Index names for ops object store */ export const OPS_INDEXES = { BY_ID: 'byId' as const, diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts index 308be9126e..815ae29a50 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts @@ -410,6 +410,37 @@ describe('OperationLogHydratorService', () => { expect(mockHydrationStateService.endApplyingRemoteOps).toHaveBeenCalled(); }); + it('should replay a tail op with a malformed stored schemaVersion verbatim instead of failing into recovery', async () => { + // Strict schemaVersion parsing guards the receive/upload paths; on the + // local hydration path one legacy/corrupt entry must not turn every + // boot into attemptRecovery() (which can lose tail data). + const snapshot = createMockSnapshot({ lastAppliedOpSeq: 5 }); + const malformedOp = createMockOperation('op-6', OpType.Update, { + schemaVersion: '2' as unknown as number, + }); + const tailOps = [ + createMockEntry(6, malformedOp), + createMockEntry(7, createMockOperation('op-7')), + ]; + mockOpLogStore.loadStateCache.and.returnValue(Promise.resolve(snapshot)); + mockOpLogStore.getOpsAfterSeq.and.returnValue(Promise.resolve(tailOps)); + + await service.hydrateStore(); + + expect(mockRecoveryService.attemptRecovery).not.toHaveBeenCalled(); + // Both ops replay in order; the malformed one is passed through with a + // sanitized version stamp (payload untouched). + expect(mockStore.dispatch).toHaveBeenCalledWith( + bulkApplyHydrationOperations({ + operations: [ + { ...malformedOp, schemaVersion: CURRENT_SCHEMA_VERSION }, + tailOps[1].op, + ], + localClientId: 'test-client', + }), + ); + }); + it('should apply a failed tail op exactly once across hydration replay + retry (one boot)', async () => { // Regression: a remote op marked 'failed' (archive side effect threw // after its reducer committed) with seq > lastAppliedOpSeq used to get @@ -993,10 +1024,12 @@ describe('OperationLogHydratorService', () => { lastAppliedOpSeq: 5, }); + // schemaVersion 1 = a legitimately old version (the floor); 0 would be + // malformed and is sanitized by the lenient hydration boundary instead. const tailOps = [ createMockEntry( 6, - createMockOperation('op-6', OpType.Update, { schemaVersion: 0 }), // Old + createMockOperation('op-6', OpType.Update, { schemaVersion: 1 }), // Old ), createMockEntry( 7, @@ -1006,7 +1039,7 @@ describe('OperationLogHydratorService', () => { ), createMockEntry( 8, - createMockOperation('op-8', OpType.Update, { schemaVersion: 0 }), // Old + createMockOperation('op-8', OpType.Update, { schemaVersion: 1 }), // Old ), ]; @@ -1015,7 +1048,7 @@ describe('OperationLogHydratorService', () => { mockSchemaMigrationService.needsMigration.and.returnValue(false); // At least one operation needs migration mockSchemaMigrationService.operationNeedsMigration.and.callFake( - (op: Operation) => op.schemaVersion === 0, + (op: Operation) => op.schemaVersion === 1, ); await service.hydrateStore(); diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.ts b/src/app/op-log/persistence/operation-log-hydrator.service.ts index e972905af4..05b86adfe2 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.ts @@ -5,6 +5,7 @@ import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { OperationLogMigrationService } from './operation-log-migration.service'; import { CURRENT_SCHEMA_VERSION, + getOperationSchemaVersion, SchemaMigrationService, } from './schema-migration.service'; import { OperationLogSnapshotService } from './operation-log-snapshot.service'; @@ -484,20 +485,39 @@ export class OperationLogHydratorService { * @returns Array of migrated operations */ private _migrateTailOps(ops: Operation[]): Operation[] { + // Lenient boundary: a malformed stored schemaVersion (legacy or corrupt + // entry) must not abort the WHOLE hydration into attemptRecovery() — that + // trades one questionable op for possible tail-data loss on every boot. + // Strict parsing stays on the receive/upload paths; locally we replay the + // op verbatim as a best effort (stamping the current version so + // migrateOperations passes it through unchanged, preserving order). + const sanitizedOps = ops.map((op) => { + try { + getOperationSchemaVersion(op); + return op; + } catch { + OpLog.warn( + 'OperationLogHydratorService: Stored op has a malformed schemaVersion; replaying verbatim without migration.', + { id: op.id }, + ); + return { ...op, schemaVersion: CURRENT_SCHEMA_VERSION }; + } + }); + // Check if any ops need migration - const needsMigration = ops.some((op) => + const needsMigration = sanitizedOps.some((op) => this.schemaMigrationService.operationNeedsMigration(op), ); if (!needsMigration) { - return ops; + return sanitizedOps; } OpLog.normal( - `OperationLogHydratorService: Migrating ${ops.length} tail ops to current schema version...`, + `OperationLogHydratorService: Migrating ${sanitizedOps.length} tail ops to current schema version...`, ); - return this.schemaMigrationService.migrateOperations(ops); + return this.schemaMigrationService.migrateOperations(sanitizedOps); } /** @@ -619,9 +639,12 @@ export class OperationLogHydratorService { .filter((seq): seq is number => seq !== undefined); if (appliedSeqs.length > 0) { await this.opLogStore.markApplied(appliedSeqs); - // Parity with the primary remote-apply path: applyRemoteOperations only - // merges clocks for ops it marked applied, so a failed op's clock is - // merged here, when its application completes. + // The primary remote-apply path (applyRemoteOperations) merges clocks at + // reducer commit for the WHOLE batch, including ops whose archive + // handling later fails — so these clocks were usually merged already. + // Re-merging here is a harmless component-wise max and also covers ops + // that reached `failed`/`archive_pending` via crash recovery, where the + // reducer-commit callback (and its clock merge) may never have run. await this.opLogStore.mergeRemoteOpClocks(result.appliedOps); OpLog.normal( `OperationLogHydratorService: Successfully retried ${appliedSeqs.length} failed ops`, diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 2f1da31953..422d7e8a97 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -2384,6 +2384,26 @@ describe('OperationLogStoreService', () => { ); }); + it('sets the raw-rebuild-incomplete marker atomically with the replacement and clears it on demand', async () => { + expect(await service.isRawRebuildIncomplete()).toBe(false); + + await service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('young'), + archiveOld: createArchive('old'), + }); + + // A crash after the replacement but before the replay commits must leave + // the marker set so the next sync redoes the raw rebuild. + expect(await service.isRawRebuildIncomplete()).toBe(true); + + await service.clearRawRebuildIncomplete(); + expect(await service.isRawRebuildIncomplete()).toBe(false); + }); + it('rolls back every store if one archive write fails', async () => { const priorOp = createTestOperation({ entityId: 'prior-task' }); const priorState = { sentinel: 'prior-state' }; diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index 0db53f6a65..1b387219ba 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -18,6 +18,7 @@ import { SINGLETON_KEY, BACKUP_KEY, FULL_STATE_OPS_META_KEY, + RAW_REBUILD_INCOMPLETE_META_KEY, OPS_INDEXES, ArchiveStoreEntry, ProfileDataStoreEntry, @@ -1577,6 +1578,15 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + await this._ensureInit(); + const entry = await this._adapter.get<{ incomplete?: boolean }>( + STORE_NAMES.META, + RAW_REBUILD_INCOMPLETE_META_KEY, + ); + return entry?.incomplete === true; + } + + /** Marks the USE_REMOTE raw rebuild as fully completed. */ + async clearRawRebuildIncomplete(): Promise { + await this._ensureInit(); + await this._adapter.delete(STORE_NAMES.META, RAW_REBUILD_INCOMPLETE_META_KEY); + } + // ============================================================ // Vector Clock Management (Performance Optimization) // ============================================================ diff --git a/src/app/op-log/persistence/schema-migration.service.ts b/src/app/op-log/persistence/schema-migration.service.ts index 4fbaaafc5a..a154f2f7ad 100644 --- a/src/app/op-log/persistence/schema-migration.service.ts +++ b/src/app/op-log/persistence/schema-migration.service.ts @@ -3,7 +3,6 @@ import { Operation, VectorClock } from '../core/operation.types'; import { OpLog } from '../../core/log'; import { CURRENT_SCHEMA_VERSION as SHARED_CURRENT_SCHEMA_VERSION, - MAX_VERSION_SKIP as SHARED_MAX_VERSION_SKIP, MIN_SUPPORTED_SCHEMA_VERSION as SHARED_MIN_SUPPORTED_SCHEMA_VERSION, MIGRATIONS, migrateState, @@ -17,7 +16,6 @@ import { // Re-export shared constants for backwards compatibility export const CURRENT_SCHEMA_VERSION = SHARED_CURRENT_SCHEMA_VERSION; -export const MAX_VERSION_SKIP = SHARED_MAX_VERSION_SKIP; export const MIN_SUPPORTED_SCHEMA_VERSION = SHARED_MIN_SUPPORTED_SCHEMA_VERSION; // Re-export types @@ -30,11 +28,11 @@ export const getOperationSchemaVersion = (op: { schemaVersion?: unknown }): numb if ( typeof op.schemaVersion !== 'number' || !Number.isInteger(op.schemaVersion) || - op.schemaVersion < 0 + // Floor of 1 matches the server contract (SuperSyncOperationSchema min(1)) + // so the layers cannot drift apart. + op.schemaVersion < 1 ) { - throw new Error( - 'Operation schemaVersion must be a non-negative integer when present.', - ); + throw new Error('Operation schemaVersion must be a positive integer when present.'); } return op.schemaVersion; }; diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 30ea0d7307..7f38e0de66 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -70,6 +70,9 @@ describe('OperationLogSyncService', () => { 'appendBatchSkipDuplicates', 'hasSyncedOps', 'runRemoteStateReplacement', + 'isRawRebuildIncomplete', + 'clearRawRebuildIncomplete', + 'loadImportBackup', ]); opLogStoreSpy.hasSyncedOps.and.resolveTo(true); opLogStoreSpy.markSynced.and.resolveTo(); @@ -82,6 +85,9 @@ describe('OperationLogSyncService', () => { skippedCount: 0, }); opLogStoreSpy.runRemoteStateReplacement.and.resolveTo(); + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); + opLogStoreSpy.clearRawRebuildIncomplete.and.resolveTo(); + opLogStoreSpy.loadImportBackup.and.resolveTo(null); schemaMigrationServiceSpy = jasmine.createSpyObj('SchemaMigrationService', [ 'getCurrentVersion', @@ -147,8 +153,16 @@ describe('OperationLogSyncService', () => { writeFlushServiceSpy = jasmine.createSpyObj('OperationWriteFlushService', [ 'flushPendingWrites', + 'flushThenRunExclusive', ]); writeFlushServiceSpy.flushPendingWrites.and.resolveTo(); + // Mirror the real barrier semantics: flush BEFORE the exclusive section runs. + writeFlushServiceSpy.flushThenRunExclusive.and.callFake( + async (fn: () => Promise) => { + await writeFlushServiceSpy.flushPendingWrites(); + return fn(); + }, + ); superSyncStatusServiceSpy = jasmine.createSpyObj('SuperSyncStatusService', [ 'updatePendingOpsStatus', @@ -879,6 +893,27 @@ describe('OperationLogSyncService', () => { }); describe('downloadRemoteOps', () => { + it('should redo the raw rebuild when a prior USE_REMOTE replay was interrupted', async () => { + // The normal download path excludes this client's own ops server-side, + // so resuming an interrupted rebuild through it would silently lose them. + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + const forceDownloadSpy = spyOn( + service, + 'forceDownloadRemoteState', + ).and.resolveTo(); + const mockProvider = { + isReady: () => Promise.resolve(true), + } as any; + + const result = await service.downloadRemoteOps(mockProvider); + + expect(forceDownloadSpy).toHaveBeenCalledWith(mockProvider, { + isCrashResume: true, + }); + expect(result.kind).toBe('snapshot_hydrated'); + expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); + }); + it('should return localWinOpsCreated: 0 and newOpsCount: 0 when no new ops', async () => { downloadServiceSpy.downloadRemoteOps.and.returnValue( Promise.resolve({ @@ -2533,6 +2568,93 @@ describe('OperationLogSyncService', () => { expect(backupServiceSpy.captureImportBackup).toHaveBeenCalled(); }); + it('should clear the raw-rebuild-incomplete marker only after the replay committed', async () => { + const callOrder: string[] = []; + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + remoteOpsProcessingServiceSpy.processRemoteOps.and.callFake(async () => { + callOrder.push('processRemoteOps'); + return { + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, + }; + }); + opLogStoreSpy.clearRawRebuildIncomplete.and.callFake(async () => { + callOrder.push('clearRawRebuildIncomplete'); + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as any; + + await service.forceDownloadRemoteState(mockProvider); + + expect(callOrder).toEqual(['processRemoteOps', 'clearRawRebuildIncomplete']); + }); + + it('should NOT clear the raw-rebuild-incomplete marker when the replay is blocked', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as any; + + await expectAsync(service.forceDownloadRemoteState(mockProvider)).toBeRejected(); + + expect(opLogStoreSpy.clearRawRebuildIncomplete).not.toHaveBeenCalled(); + }); + + it('should keep the first attempt backup on crash resume instead of re-capturing', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ state: {}, savedAt: 12345 }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as any; + + await service.forceDownloadRemoteState(mockProvider, { isCrashResume: true }); + + // Re-capturing would overwrite the single backup slot with the partial + // baseline; the original pre-replace snapshot must survive the resume. + expect(backupServiceSpy.captureImportBackup).not.toHaveBeenCalled(); + expect(snackServiceSpy.open).toHaveBeenCalledWith( + jasmine.objectContaining({ + msg: T.F.SYNC.S.LOCAL_DATA_REPLACE_UNDO, + actionStr: T.G.UNDO, + }), + ); + }); + it('should offer to restore the previous data after replacing (#8107)', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 6d97e0d7ce..3050d00754 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -51,8 +51,6 @@ import { SyncLocalStateService } from './sync-local-state.service'; import { SyncImportConflictCoordinatorService } from './sync-import-conflict-coordinator.service'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; import { Operation } from '../core/operation.types'; -import { LockService } from './lock.service'; -import { LOCK_NAMES } from '../core/operation-log.const'; import { ValidateStateService } from '../validation/validate-state.service'; import { extractEntityKeysFromState } from '../persistence/extract-entity-keys'; @@ -132,7 +130,6 @@ export class OperationLogSyncService { private superSyncStatusService = inject(SuperSyncStatusService); private serverMigrationService = inject(ServerMigrationService); private writeFlushService = inject(OperationWriteFlushService); - private lockService = inject(LockService); private schemaMigrationService = inject(SchemaMigrationService); private validateStateService = inject(ValidateStateService); @@ -439,6 +436,19 @@ export class OperationLogSyncService { syncProvider: OperationSyncCapable, options?: { forceFromSeq0?: boolean; isNeverSynced?: boolean }, ): Promise { + // Crash-resume: a prior USE_REMOTE rebuild committed its baseline + // replacement but crashed before the replay finished. The normal download + // path excludes this client's own ops server-side, so resuming through it + // would silently lose them — redo the raw rebuild instead. + if (await this.opLogStore.isRawRebuildIncomplete()) { + OpLog.warn( + 'OperationLogSyncService: Interrupted USE_REMOTE rebuild detected — redoing the raw rebuild.', + ); + await this.forceDownloadRemoteState(syncProvider, { isCrashResume: true }); + // State was replaced wholesale, exactly like a snapshot hydration. + return { kind: 'snapshot_hydrated' }; + } + const result = await this.downloadService.downloadRemoteOps(syncProvider, options); // FIX #6571: Check download success before processing results. @@ -1174,7 +1184,18 @@ export class OperationLogSyncService { * * @param syncProvider - The sync provider to download from */ - async forceDownloadRemoteState(syncProvider: OperationSyncCapable): Promise { + async forceDownloadRemoteState( + syncProvider: OperationSyncCapable, + options?: { + /** + * Resuming an interrupted rebuild (crash between the baseline replacement + * and the replay commit). Keeps the FIRST attempt's pre-replace safety + * backup: the single backup slot still holds the user's original data, + * and re-capturing here would overwrite it with the partial baseline. + */ + isCrashResume?: boolean; + }, + ): Promise { OpLog.warn( 'OperationLogSyncService: Force downloading remote state for a full rebuild.', ); @@ -1255,15 +1276,20 @@ export class OperationLogSyncService { let capturedBackupSavedAt: number | undefined; let replacementCommitted = false; - let backupSavedAt: number; + let backupSavedAt: number | undefined; try { - backupSavedAt = await this.lockService.request( - LOCK_NAMES.OPERATION_LOG, - async () => { - // Include actions dispatched while the network request and preflight were - // in flight in the reversible safety backup. - await this.writeFlushService.flushPendingWrites(); - let savedAt: number; + // flushThenRunExclusive drains the capture pipeline BEFORE acquiring the + // op-log lock (flushPendingWrites re-acquires the same non-reentrant lock, + // so flushing while holding it deadlocks) and re-checks inside the lock — + // actions dispatched while the network request and preflight were in + // flight are durably written and included in the reversible safety backup. + backupSavedAt = await this.writeFlushService.flushThenRunExclusive(async () => { + let savedAt: number | undefined; + if (options?.isCrashResume) { + // Keep the first attempt's pre-replace backup (see option JSDoc). + savedAt = (await this.opLogStore.loadImportBackup())?.savedAt; + capturedBackupSavedAt = savedAt; + } else { try { savedAt = await this.backupService.captureImportBackup(); capturedBackupSavedAt = savedAt; @@ -1276,121 +1302,129 @@ export class OperationLogSyncService { 'Pre-replace safety backup failed; aborting to preserve local state.', ); } + } - // The provider cursor lives outside SUP_OPS, so it cannot join the IDB - // transaction. Reset it first: a crash/failure before the transaction - // merely causes a safe re-download onto intact local state, while a - // commit can never become visible with a stale cursor that skips the - // remote history required to rebuild the baseline. - await syncProvider.setLastServerSeq(0); - await this.opLogStore.runRemoteStateReplacement({ - baselineState, - vectorClock: rebuiltClock, - schemaVersion: CURRENT_SCHEMA_VERSION, - snapshotEntityKeys: extractEntityKeysFromState( - baselineState as Parameters[0], - ), - archiveYoung, - archiveOld, - }); - replacementCommitted = true; + // The provider cursor lives outside SUP_OPS, so it cannot join the IDB + // transaction. Reset it first: a crash/failure before the transaction + // merely causes a safe re-download onto intact local state, while a + // commit can never become visible with a stale cursor that skips the + // remote history required to rebuild the baseline. + await syncProvider.setLastServerSeq(0); + await this.opLogStore.runRemoteStateReplacement({ + baselineState, + vectorClock: rebuiltClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + snapshotEntityKeys: extractEntityKeysFromState( + baselineState as Parameters[0], + ), + archiveYoung, + archiveOld, + }); + replacementCommitted = true; + OpLog.normal( + 'OperationLogSyncService: Replaced local persistence with remote baseline.', + ); + + // FILE-BASED SYNC: Handle snapshot state from force download. + // When downloading from seq 0 on file-based providers, we may receive a + // snapshotState instead of incremental ops. This happens when the remote + // has a SYNC_IMPORT (full state snapshot) with empty recentOps. + // hydrateFromRemoteSync persists its own state cache + vector clock. + if (result.providerMode === 'fileSnapshotOps' && snapshotState) { OpLog.normal( - 'OperationLogSyncService: Replaced local persistence with remote baseline.', + 'OperationLogSyncService: Force download received snapshotState. Hydrating...', ); - // FILE-BASED SYNC: Handle snapshot state from force download. - // When downloading from seq 0 on file-based providers, we may receive a - // snapshotState instead of incremental ops. This happens when the remote - // has a SYNC_IMPORT (full state snapshot) with empty recentOps. - // hydrateFromRemoteSync persists its own state cache + vector clock. - if (result.providerMode === 'fileSnapshotOps' && snapshotState) { + // Hydrate from snapshot - DON'T create SYNC_IMPORT since we're + // accepting remote state, not uploading local state. + await this.syncHydrationService.hydrateFromRemoteSync( + snapshotState, + result.snapshotVectorClock, + false, // Don't create SYNC_IMPORT + ); + + // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. + // Same rationale as downloadRemoteOps: file-based providers return ALL + // recentOps on every download and rely on getAppliedOpIds() to filter them. + if (migratedRemoteOps.length > 0) { + const appendResult = await this.opLogStore.appendBatchSkipDuplicates( + migratedRemoteOps, + 'remote', + ); OpLog.normal( - 'OperationLogSyncService: Force download received snapshotState. Hydrating...', - ); - - // Hydrate from snapshot - DON'T create SYNC_IMPORT since we're - // accepting remote state, not uploading local state. - await this.syncHydrationService.hydrateFromRemoteSync( - snapshotState, - result.snapshotVectorClock, - false, // Don't create SYNC_IMPORT - ); - - // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. - // Same rationale as downloadRemoteOps: file-based providers return ALL - // recentOps on every download and rely on getAppliedOpIds() to filter them. - if (migratedRemoteOps.length > 0) { - const appendResult = await this.opLogStore.appendBatchSkipDuplicates( - migratedRemoteOps, - 'remote', - ); - OpLog.normal( - `OperationLogSyncService: Wrote ${appendResult.writtenOps.length} snapshot ops to IndexedDB ` + - 'after force-download hydration.' + - (appendResult.skippedCount > 0 - ? ` Skipped ${appendResult.skippedCount} duplicate(s).` - : ''), - ); - } - - // Update lastServerSeq after hydration - if (result.latestServerSeq !== undefined) { - await syncProvider.setLastServerSeq(result.latestServerSeq); - } - - OpLog.normal( - 'OperationLogSyncService: Force download snapshot hydration complete.', - ); - return savedAt; - } - - // Reset live state to defaults, then replay the COMPLETE server history on - // top. A full-state op in the history replaces state again by its own - // semantics; a purely incremental history rebuilds from this baseline. - this.store.dispatch( - loadAllData({ - appDataComplete: defaultData as Parameters< - typeof loadAllData - >[0]['appDataComplete'], - }), - ); - // Brief yield to let NgRx process the state reset - await new Promise((resolve) => setTimeout(resolve, 0)); - - // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). - // Skip conflict detection because the NgRx store was just reset to empty state, - // which causes all entities to appear missing and CONCURRENT ops to be discarded. - // Validation failure is surfaced via the session-validation latch. (#7330) - const processResult = await this.remoteOpsProcessingService.processRemoteOps( - migratedRemoteOps, - { - skipConflictDetection: true, - callerHoldsOperationLogLock: true, - }, - ); - - if (processResult.blockedByIncompatibleOp) { - // Version blocks were pre-checked above; only a migration exception lands - // here. The rebuild is partial: keep the cursor at 0 so the next sync - // retries the remainder, and surface the failure — the Undo snack still - // offers the pre-replace backup. - throw new Error( - 'USE_REMOTE incomplete: an op failed schema migration during replay.', + `OperationLogSyncService: Wrote ${appendResult.writtenOps.length} snapshot ops to IndexedDB ` + + 'after force-download hydration.' + + (appendResult.skippedCount > 0 + ? ` Skipped ${appendResult.skippedCount} duplicate(s).` + : ''), ); } - // Update lastServerSeq + // Update lastServerSeq after hydration if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); } + // Replay committed — the rebuild is complete and crash-resume is + // no longer needed. + await this.opLogStore.clearRawRebuildIncomplete(); + OpLog.normal( - `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, + 'OperationLogSyncService: Force download snapshot hydration complete.', ); return savedAt; - }, - ); + } + + // Reset live state to defaults, then replay the COMPLETE server history on + // top. A full-state op in the history replaces state again by its own + // semantics; a purely incremental history rebuilds from this baseline. + this.store.dispatch( + loadAllData({ + appDataComplete: defaultData as Parameters< + typeof loadAllData + >[0]['appDataComplete'], + }), + ); + // Brief yield to let NgRx process the state reset + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). + // Skip conflict detection because the NgRx store was just reset to empty state, + // which causes all entities to appear missing and CONCURRENT ops to be discarded. + // Validation failure is surfaced via the session-validation latch. (#7330) + const processResult = await this.remoteOpsProcessingService.processRemoteOps( + migratedRemoteOps, + { + skipConflictDetection: true, + callerHoldsOperationLogLock: true, + }, + ); + + if (processResult.blockedByIncompatibleOp) { + // Version blocks were pre-checked above; only a migration exception lands + // here. The rebuild is partial: keep the cursor at 0 so the next sync + // retries the remainder, and surface the failure — the Undo snack still + // offers the pre-replace backup. + throw new Error( + 'USE_REMOTE incomplete: an op failed schema migration during replay.', + ); + } + + // Update lastServerSeq + if (result.latestServerSeq !== undefined) { + await syncProvider.setLastServerSeq(result.latestServerSeq); + } + + // Replay committed — the rebuild is complete and crash-resume is + // no longer needed. + await this.opLogStore.clearRawRebuildIncomplete(); + + OpLog.normal( + `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, + ); + return savedAt; + }); } catch (e) { if (replacementCommitted && capturedBackupSavedAt !== undefined) { this._showRestorePreviousDataSnack(capturedBackupSavedAt); @@ -1398,7 +1432,10 @@ export class OperationLogSyncService { throw e; } - this._showRestorePreviousDataSnack(backupSavedAt); + // On a crash resume without a surviving backup there is nothing to offer. + if (backupSavedAt !== undefined) { + this._showRestorePreviousDataSnack(backupSavedAt); + } } private _preflightRemoteOperations(remoteOps: Operation[]): Operation[] { @@ -1406,9 +1443,16 @@ export class OperationLogSyncService { let version: number; try { version = getOperationSchemaVersion(op as { schemaVersion?: unknown }); - } catch { + } catch (e) { + // Keep the root cause diagnosable (id-only, no payloads) — this is a + // rare, support-heavy failure path. + OpLog.err('OperationLogSyncService: USE_REMOTE preflight version parse failed', { + id: op.id, + name: (e as Error | undefined)?.name, + }); throw new Error( 'USE_REMOTE aborted: remote history has an invalid schema version.', + { cause: e }, ); } @@ -1433,8 +1477,13 @@ export class OperationLogSyncService { try { return this.schemaMigrationService.migrateOperations(remoteOps); - } catch { - throw new Error('USE_REMOTE aborted: remote operation migration failed.'); + } catch (e) { + OpLog.err('OperationLogSyncService: USE_REMOTE preflight migration failed', { + name: (e as Error | undefined)?.name, + }); + throw new Error('USE_REMOTE aborted: remote operation migration failed.', { + cause: e, + }); } } diff --git a/src/app/op-log/sync/operation-write-flush.service.spec.ts b/src/app/op-log/sync/operation-write-flush.service.spec.ts index 9052c23735..a868d129a8 100644 --- a/src/app/op-log/sync/operation-write-flush.service.spec.ts +++ b/src/app/op-log/sync/operation-write-flush.service.spec.ts @@ -1,21 +1,28 @@ import { TestBed } from '@angular/core/testing'; import { OperationWriteFlushService } from './operation-write-flush.service'; import { LockService } from './lock.service'; +import { OperationCaptureService } from '../capture/operation-capture.service'; describe('OperationWriteFlushService', () => { let service: OperationWriteFlushService; let lockServiceSpy: jasmine.SpyObj; + let captureServiceSpy: jasmine.SpyObj; beforeEach(() => { lockServiceSpy = jasmine.createSpyObj('LockService', ['request']); lockServiceSpy.request.and.callFake( async (_name: string, callback: () => Promise) => callback(), ); + captureServiceSpy = jasmine.createSpyObj('OperationCaptureService', [ + 'getPendingCount', + ]); + captureServiceSpy.getPendingCount.and.returnValue(0); TestBed.configureTestingModule({ providers: [ OperationWriteFlushService, { provide: LockService, useValue: lockServiceSpy }, + { provide: OperationCaptureService, useValue: captureServiceSpy }, ], }); service = TestBed.inject(OperationWriteFlushService); @@ -102,6 +109,71 @@ describe('OperationWriteFlushService', () => { }); }); + describe('flushThenRunExclusive', () => { + it('should flush BEFORE acquiring the lock (calling flush inside the held lock deadlocks)', async () => { + const events: string[] = []; + spyOn(service, 'flushPendingWrites').and.callFake(async () => { + events.push('flush'); + }); + lockServiceSpy.request.and.callFake( + async (_name: string, callback: () => Promise) => { + events.push('lock-acquired'); + const result = await callback(); + events.push('lock-released'); + return result; + }, + ); + + await service.flushThenRunExclusive(async () => { + events.push('fn'); + }); + + expect(events).toEqual(['flush', 'lock-acquired', 'fn', 'lock-released']); + }); + + it('should return the value produced by fn', async () => { + spyOn(service, 'flushPendingWrites').and.resolveTo(); + + const result = await service.flushThenRunExclusive(async () => 42); + + expect(result).toBe(42); + }); + + it('should release, re-flush, and retry when a capture lands between flush and lock acquisition', async () => { + spyOn(service, 'flushPendingWrites').and.resolveTo(); + captureServiceSpy.getPendingCount.and.returnValues(1, 0); + const fn = jasmine.createSpy('fn').and.resolveTo('done'); + + const result = await service.flushThenRunExclusive(fn); + + expect(result).toBe('done'); + expect(service.flushPendingWrites).toHaveBeenCalledTimes(2); + expect(lockServiceSpy.request).toHaveBeenCalledTimes(2); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('should abort after bounded attempts under continuous dispatch activity', async () => { + spyOn(service, 'flushPendingWrites').and.resolveTo(); + captureServiceSpy.getPendingCount.and.returnValue(1); + const fn = jasmine.createSpy('fn'); + + await expectAsync(service.flushThenRunExclusive(fn)).toBeRejectedWithError( + /cutoff not reached/, + ); + expect(fn).not.toHaveBeenCalled(); + expect(service.flushPendingWrites).toHaveBeenCalledTimes(5); + }); + + it('should propagate fn rejections without retrying', async () => { + spyOn(service, 'flushPendingWrites').and.resolveTo(); + const testError = new Error('fn failed'); + const fn = jasmine.createSpy('fn').and.rejectWith(testError); + + await expectAsync(service.flushThenRunExclusive(fn)).toBeRejectedWith(testError); + expect(fn).toHaveBeenCalledTimes(1); + }); + }); + describe('FIFO ordering guarantee', () => { it('should ensure prior lock holders complete before flush resolves', async () => { // This test verifies the core guarantee: when flushPendingWrites resolves, diff --git a/src/app/op-log/sync/operation-write-flush.service.ts b/src/app/op-log/sync/operation-write-flush.service.ts index b36456380d..c3d6ab1bb4 100644 --- a/src/app/op-log/sync/operation-write-flush.service.ts +++ b/src/app/op-log/sync/operation-write-flush.service.ts @@ -39,6 +39,14 @@ export class OperationWriteFlushService { */ private readonly MAX_WAIT_TIME = 30000; + /** + * Maximum flush→lock→recheck attempts in flushThenRunExclusive before + * aborting. Bounds the retry loop so continuous dispatch activity (e.g. a + * runaway effect or 1Hz tracking ticks on a very slow device) cannot + * livelock the caller; the operation re-triggers on the next sync. + */ + private readonly MAX_CUTOFF_ATTEMPTS = 5; + /** * Polling interval to check queue size (ms). */ @@ -107,4 +115,43 @@ export class OperationWriteFlushService { `Initial pending: ${initialPendingCount}, Final pending: ${finalPendingCount}`, ); } + + /** + * Runs `fn` inside the operation-log lock with the capture pipeline drained — + * every action dispatched before the lock was acquired is durably written. + * + * flushPendingWrites() must NOT be called while holding the lock: its Phase 2 + * re-acquires the same non-reentrant lock and deadlocks until the acquisition + * timeout. So the flush runs BEFORE acquisition. A reducer action can still + * land in the tiny gap between the flush releasing the lock and our + * acquisition — its pending counter increments synchronously, before the + * persistence effect waits for the lock. Taking a snapshot/backup then would + * include that reducer state but precede its operation, so instead the lock + * is released, the writes are re-flushed, and the acquisition retried + * (bounded by MAX_CUTOFF_ATTEMPTS). + */ + async flushThenRunExclusive(fn: () => Promise): Promise { + for (let attempt = 0; attempt < this.MAX_CUTOFF_ATTEMPTS; attempt++) { + await this.flushPendingWrites(); + const outcome = await this.lockService.request( + LOCK_NAMES.OPERATION_LOG, + async (): Promise<{ retry: true } | { retry: false; value: T }> => { + if (this.captureService.getPendingCount() > 0) { + return { retry: true }; + } + return { retry: false, value: await fn() }; + }, + ); + if (!outcome.retry) { + return outcome.value; + } + OpLog.warn( + `OperationWriteFlushService: Capture landed between flush and lock acquisition; ` + + `retrying cutoff (attempt ${attempt + 1}/${this.MAX_CUTOFF_ATTEMPTS}).`, + ); + } + throw new Error( + `Operation write cutoff not reached after ${this.MAX_CUTOFF_ATTEMPTS} attempts — continuous dispatch activity. Try again.`, + ); + } } diff --git a/src/app/op-log/sync/server-migration.service.spec.ts b/src/app/op-log/sync/server-migration.service.spec.ts index 2eb1041845..03bbaa7adf 100644 --- a/src/app/op-log/sync/server-migration.service.spec.ts +++ b/src/app/op-log/sync/server-migration.service.spec.ts @@ -99,8 +99,16 @@ describe('ServerMigrationService', () => { ); writeFlushServiceSpy = jasmine.createSpyObj('OperationWriteFlushService', [ 'flushPendingWrites', + 'flushThenRunExclusive', ]); writeFlushServiceSpy.flushPendingWrites.and.resolveTo(); + // Mirror the real barrier semantics: flush, acquire the op-log lock, run fn. + writeFlushServiceSpy.flushThenRunExclusive.and.callFake( + async (fn: () => Promise) => { + await writeFlushServiceSpy.flushPendingWrites(); + return lockServiceSpy.request(LOCK_NAMES.OPERATION_LOG, fn); + }, + ); operationCaptureServiceSpy = jasmine.createSpyObj('OperationCaptureService', [ 'getPendingCount', ]); @@ -528,16 +536,9 @@ describe('ServerMigrationService', () => { ]); }); - it('should release, flush, and retry when an action lands before snapshot capture', async () => { - operationCaptureServiceSpy.getPendingCount.and.returnValues(1, 0); - - await service.handleServerMigration(defaultProvider); - - expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalledTimes(2); - expect(lockServiceSpy.request).toHaveBeenCalledTimes(2); - expect(stateSnapshotServiceSpy.getStateSnapshotAsync).toHaveBeenCalledTimes(1); - expect(opLogStoreSpy.append).toHaveBeenCalledTimes(1); - }); + // The release-flush-retry behavior when an action lands between flush and lock + // acquisition now lives in OperationWriteFlushService.flushThenRunExclusive — + // covered by operation-write-flush.service.spec.ts. describe('system-tag empty-state detection (tested via handleServerMigration)', () => { it('should identify system tags correctly', async () => { diff --git a/src/app/op-log/sync/server-migration.service.ts b/src/app/op-log/sync/server-migration.service.ts index 5e8330b3aa..b58985fdd1 100644 --- a/src/app/op-log/sync/server-migration.service.ts +++ b/src/app/op-log/sync/server-migration.service.ts @@ -25,10 +25,7 @@ import { CLIENT_ID_PROVIDER } from '../util/client-id.provider'; import { DialogServerMigrationConfirmComponent } from './dialog-server-migration-confirm/dialog-server-migration-confirm.component'; import { hasMeaningfulStateData } from '../validation/has-meaningful-state-data.util'; import { AppDataComplete, MODEL_CONFIGS } from '../model/model-config'; -import { LockService } from './lock.service'; import { OperationWriteFlushService } from './operation-write-flush.service'; -import { LOCK_NAMES } from '../core/operation-log.const'; -import { OperationCaptureService } from '../capture/operation-capture.service'; const MEANINGFUL_ENTITY_STATE_KEYS = new Set(['task', 'project', 'tag', 'note']); @@ -90,9 +87,7 @@ export class ServerMigrationService { private clientIdProvider = inject(CLIENT_ID_PROVIDER); private _matDialog = inject(MatDialog); private _userInputWaitState = inject(UserInputWaitStateService); - private lockService = inject(LockService); private writeFlushService = inject(OperationWriteFlushService); - private operationCaptureService = inject(OperationCaptureService); /** * Checks if we're connecting to a new/empty server and handles migration if needed. @@ -209,125 +204,113 @@ export class ServerMigrationService { // construction, and append behind the same mutation barrier. This makes the // full-state operation's local seq an exact cutoff: every earlier op is in the // snapshot, and any action captured while this runs is appended afterwards. - let retryForPendingCapture: boolean; - do { - retryForPendingCapture = false; - await this.writeFlushService.flushPendingWrites(); - await this.lockService.request(LOCK_NAMES.OPERATION_LOG, async () => { - // A reducer action can land in the tiny gap between flush() releasing - // this lock and our acquisition. Its pending counter increments - // synchronously, before the persistence effect waits for the lock. Do - // not take a snapshot that includes that reducer state but precedes its - // operation; release, drain it, and retry the cutoff instead. - if (this.operationCaptureService.getPendingCount() > 0) { - retryForPendingCapture = true; - return; - } + // flushThenRunExclusive owns the flush→lock→recheck retry loop (bounded, so + // continuous dispatch cannot livelock the migration; it re-triggers on the + // next sync). + await this.writeFlushService.flushThenRunExclusive(async () => { + // Get current full state from NgRx store (async to include archives from IndexedDB) + // Cast to Record for validation compatibility + let currentState: Record = + (await this.stateSnapshotService.getStateSnapshotAsync()) as unknown as Record< + string, + unknown + >; - // Get current full state from NgRx store (async to include archives from IndexedDB) - // Cast to Record for validation compatibility - let currentState: Record = - (await this.stateSnapshotService.getStateSnapshotAsync()) as unknown as Record< - string, - unknown - >; - - // Skip if local state is effectively empty - if (!hasServerMigrationStateData(currentState)) { - OpLog.warn( - 'ServerMigrationService: Skipping SYNC_IMPORT - local state is empty.', - ); - return; - } - - // Validate and repair state before creating SYNC_IMPORT - // This prevents corrupted state (e.g., orphaned menuTree references) from - // propagating to other clients via the full state import. - const validationResult = - await this.validateStateService.validateAndRepair(currentState); - - // If state is invalid and couldn't be repaired, abort - don't propagate corruption - if (!validationResult.isValid) { - OpLog.err( - 'ServerMigrationService: Cannot create SYNC_IMPORT - state validation failed.', - validationResult.error || validationResult.crossModelError, - ); - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.SERVER_MIGRATION_VALIDATION_FAILED, - }); - return; - } - - // If state was repaired, use the repaired version - if (validationResult.repairedState) { - OpLog.warn( - 'ServerMigrationService: State repaired before creating SYNC_IMPORT', - validationResult.repairSummary, - ); - currentState = validationResult.repairedState; - - // Also update NgRx store with repaired state so local client is consistent - this.store.dispatch( - loadAllData({ - appDataComplete: validationResult.repairedState as AppDataComplete, - }), - ); - } - - // Get client ID - const clientId = await this.clientIdProvider.loadClientId(); - if (!clientId) { - OpLog.err( - 'ServerMigrationService: Cannot create SYNC_IMPORT - no client ID available.', - ); - return; - } - - // Build vector clock by merging ALL local operation clocks. - // This ensures the SYNC_IMPORT's clock dominates all pre-import ops, - // so when SyncImportFilterService compares them, all prior ops are - // LESS_THAN (not CONCURRENT) and can be properly filtered. - const allLocalOps = await this.opLogStore.getOpsAfterSeq(0); - let mergedClock = await this.vectorClockService.getCurrentVectorClock(); - for (const entry of allLocalOps) { - mergedClock = mergeVectorClocks(mergedClock, entry.op.vectorClock); - } - const newClock = limitVectorClockSize( - incrementVectorClock(mergedClock, clientId), - clientId, + // Skip if local state is effectively empty + if (!hasServerMigrationStateData(currentState)) { + OpLog.warn( + 'ServerMigrationService: Skipping SYNC_IMPORT - local state is empty.', ); + return; + } - OpLog.normal( - `ServerMigrationService: Merged ${allLocalOps.length} local op clocks into SYNC_IMPORT vector clock.`, + // Validate and repair state before creating SYNC_IMPORT + // This prevents corrupted state (e.g., orphaned menuTree references) from + // propagating to other clients via the full state import. + const validationResult = + await this.validateStateService.validateAndRepair(currentState); + + // If state is invalid and couldn't be repaired, abort - don't propagate corruption + if (!validationResult.isValid) { + OpLog.err( + 'ServerMigrationService: Cannot create SYNC_IMPORT - state validation failed.', + validationResult.error || validationResult.crossModelError, ); + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.SERVER_MIGRATION_VALIDATION_FAILED, + }); + return; + } - // Create SYNC_IMPORT operation with full state - // NOTE: Use raw state directly (not wrapped in appDataComplete). - // The snapshot endpoint expects raw state, and the hydrator handles - // both formats on extraction. - const op: Operation = { - id: uuidv7(), - actionType: ActionType.LOAD_ALL_DATA, - opType: OpType.SyncImport, - entityType: 'ALL', - payload: currentState, - clientId, - vectorClock: newClock, - timestamp: Date.now(), - schemaVersion: CURRENT_SCHEMA_VERSION, - syncImportReason: options?.syncImportReason ?? 'SERVER_MIGRATION', - }; - - // Append to operation log - will be uploaded via snapshot endpoint - await this.opLogStore.append(op, 'local'); - - OpLog.normal( - 'ServerMigrationService: Created SYNC_IMPORT operation for server migration. ' + - 'Will be uploaded immediately via follow-up upload.', + // If state was repaired, use the repaired version + if (validationResult.repairedState) { + OpLog.warn( + 'ServerMigrationService: State repaired before creating SYNC_IMPORT', + validationResult.repairSummary, ); - }); - } while (retryForPendingCapture); + currentState = validationResult.repairedState; + + // Also update NgRx store with repaired state so local client is consistent + this.store.dispatch( + loadAllData({ + appDataComplete: validationResult.repairedState as AppDataComplete, + }), + ); + } + + // Get client ID + const clientId = await this.clientIdProvider.loadClientId(); + if (!clientId) { + OpLog.err( + 'ServerMigrationService: Cannot create SYNC_IMPORT - no client ID available.', + ); + return; + } + + // Build vector clock by merging ALL local operation clocks. + // This ensures the SYNC_IMPORT's clock dominates all pre-import ops, + // so when SyncImportFilterService compares them, all prior ops are + // LESS_THAN (not CONCURRENT) and can be properly filtered. + const allLocalOps = await this.opLogStore.getOpsAfterSeq(0); + let mergedClock = await this.vectorClockService.getCurrentVectorClock(); + for (const entry of allLocalOps) { + mergedClock = mergeVectorClocks(mergedClock, entry.op.vectorClock); + } + const newClock = limitVectorClockSize( + incrementVectorClock(mergedClock, clientId), + clientId, + ); + + OpLog.normal( + `ServerMigrationService: Merged ${allLocalOps.length} local op clocks into SYNC_IMPORT vector clock.`, + ); + + // Create SYNC_IMPORT operation with full state + // NOTE: Use raw state directly (not wrapped in appDataComplete). + // The snapshot endpoint expects raw state, and the hydrator handles + // both formats on extraction. + const op: Operation = { + id: uuidv7(), + actionType: ActionType.LOAD_ALL_DATA, + opType: OpType.SyncImport, + entityType: 'ALL', + payload: currentState, + clientId, + vectorClock: newClock, + timestamp: Date.now(), + schemaVersion: CURRENT_SCHEMA_VERSION, + syncImportReason: options?.syncImportReason ?? 'SERVER_MIGRATION', + }; + + // Append to operation log - will be uploaded via snapshot endpoint + await this.opLogStore.append(op, 'local'); + + OpLog.normal( + 'ServerMigrationService: Created SYNC_IMPORT operation for server migration. ' + + 'Will be uploaded immediately via follow-up upload.', + ); + }); } /** diff --git a/src/app/op-log/testing/integration/schema-version-sync.integration.spec.ts b/src/app/op-log/testing/integration/schema-version-sync.integration.spec.ts index bb3aa32f86..bba9abd9e3 100644 --- a/src/app/op-log/testing/integration/schema-version-sync.integration.spec.ts +++ b/src/app/op-log/testing/integration/schema-version-sync.integration.spec.ts @@ -9,7 +9,6 @@ import { resetTestUuidCounter } from './helpers/test-client.helper'; import { MockSyncServer } from './helpers/mock-sync-server.helper'; import { SimulatedClient } from './helpers/simulated-client.helper'; import { createMinimalTaskPayload } from './helpers/operation-factory.helper'; -import { MAX_VERSION_SKIP } from '@sp/shared-schema'; /** * Integration tests for Schema Version Sync. @@ -175,11 +174,6 @@ describe('Schema Version Sync Integration', () => { expect(CURRENT_SCHEMA_VERSION).toBeGreaterThanOrEqual(1); }); - it('should define MAX_VERSION_SKIP as a positive number', () => { - expect(typeof MAX_VERSION_SKIP).toBe('number'); - expect(MAX_VERSION_SKIP).toBeGreaterThan(0); - }); - it('should not need migration for state at current version', () => { const cache = { state: {}, From a09ef864cf533a8dc3fc5e071d4b958b6259ec54 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:43:23 +0200 Subject: [PATCH 08/47] test(e2e): identify counter without the flaky hover tooltip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getNamedClickCounter hovered the counter button and waited for its matTooltip title, but matTooltip does not open on Playwright's synthetic hover in headless CI — the assertion timed out at PHASE 1 before the gate logic ran (run 29098937743, SuperSync 1/6). Each client has exactly one counter at every interaction point here, so assert the button's rendered initial instead of hovering for a tooltip. --- .../supersync-conflict-gate-non-task-work.spec.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index 9eb4ad930e..4c7995f821 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -75,10 +75,16 @@ const getNamedClickCounter = async ( .last(); await counter.waitFor({ state: 'visible', timeout: 15000 }); - // Counter buttons render only their initial, so verify the exact counter via - // its accessible Material tooltip before interacting with it. - await counter.hover(); - await expect(client.page.getByRole('tooltip').filter({ hasText: title })).toBeVisible(); + // The button renders only the title's initial; the full title lives in a + // matTooltip that does NOT open on Playwright's synthetic hover in headless + // CI, so we can't identify by tooltip. This test only ever has one counter + // per client at each interaction point (A imports after its only counter + // interaction; B keeps local via USE_LOCAL and never adopts the backup's + // counters), so the last button is unambiguous — sanity-check its rendered + // initial matches the expected title instead of hovering for a tooltip. + await expect(counter.locator('.habit-initial')).toHaveText( + title.charAt(0).toUpperCase(), + ); return counter; }; From 0df3e811556f57c37690f5f995e77bfb655a1abf Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:43:23 +0200 Subject: [PATCH 09/47] fix(sync): remediate multi-agent review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - USE_REMOTE: fix guaranteed self-deadlock — flushPendingWrites() was called while already holding the non-reentrant sp_op_log lock its own Phase 2 re-acquires, so every "Use Server Data" hung 30s and aborted. New OperationWriteFlushService.flushThenRunExclusive owns the bounded flush->lock->recheck barrier; ServerMigrationService reuses it, which also bounds its previously unbounded snapshot-cutoff retry loop. - USE_REMOTE: make the rebuild crash-resumable. A raw_rebuild_incomplete META marker is set atomically with the baseline replacement and cleared after the replay commits; the next sync redoes the raw rebuild instead of the normal download (which excludes own ops server-side and would silently lose the un-replayed suffix). The resume keeps the first attempt's pre-replace backup instead of overwriting the single slot with the partial baseline. - Deferred actions: serialize overlapping drains (two concurrent drains could mint two ops for one user intent), stop the drain at the first transient failure instead of persisting successors out of order (the older edit would win LWW everywhere via clock inversion), abandon permanently invalid actions after one attempt (typed PermanentDeferredWriteError) instead of retrying + snacking on every sync forever, and latch the buffer-size warnings to once per stuck window (previously a blocking dev dialog per action past 100); drop the no-op 5000 "hard cap" tier. - writeOperation: post-append bookkeeping failures no longer propagate into the deferred retry loop, which re-appended the same action under a fresh op id (double-apply of additive payloads). - remote-apply: full-state cleanup retains every full-state op of the current batch, so an archive_pending import can no longer be deleted before its archive retry (startup replay would lose a change already visible at runtime). - Hydration: sanitize a malformed stored schemaVersion per-op instead of failing the whole boot into recovery; strict parsing stays on the receive/upload paths (0 still parses so a below-minimum remote op surfaces as VERSION_UNSUPPORTED, not a generic migration failure). - Server: request fingerprints are computed lazily — only when a dedup entry exists, and otherwise after the rate-limit/pre-quota gates but before uploadOps mutates the parsed ops. Fixes the pre-gate hashing of full payloads (authenticated DoS-cost regression) and repairs three sync-fixes retry tests to model true identical-body retries. - Delete dead code: failed-op-ids util (+spec) and MAX_VERSION_SKIP (the removed forward-compat band); fix the stale clock-merge parity comment. sync-core 210/210, shared-schema 50/50, super-sync-server 825/825, and all touched Angular specs pass. --- .../supersync-conflict-gate-non-task-work.spec.ts | 14 ++++++++++---- .../op-log/persistence/schema-migration.service.ts | 12 ++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index 9eb4ad930e..4c7995f821 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -75,10 +75,16 @@ const getNamedClickCounter = async ( .last(); await counter.waitFor({ state: 'visible', timeout: 15000 }); - // Counter buttons render only their initial, so verify the exact counter via - // its accessible Material tooltip before interacting with it. - await counter.hover(); - await expect(client.page.getByRole('tooltip').filter({ hasText: title })).toBeVisible(); + // The button renders only the title's initial; the full title lives in a + // matTooltip that does NOT open on Playwright's synthetic hover in headless + // CI, so we can't identify by tooltip. This test only ever has one counter + // per client at each interaction point (A imports after its only counter + // interaction; B keeps local via USE_LOCAL and never adopts the backup's + // counters), so the last button is unambiguous — sanity-check its rendered + // initial matches the expected title instead of hovering for a tooltip. + await expect(counter.locator('.habit-initial')).toHaveText( + title.charAt(0).toUpperCase(), + ); return counter; }; diff --git a/src/app/op-log/persistence/schema-migration.service.ts b/src/app/op-log/persistence/schema-migration.service.ts index a154f2f7ad..8e5e309a45 100644 --- a/src/app/op-log/persistence/schema-migration.service.ts +++ b/src/app/op-log/persistence/schema-migration.service.ts @@ -28,11 +28,15 @@ export const getOperationSchemaVersion = (op: { schemaVersion?: unknown }): numb if ( typeof op.schemaVersion !== 'number' || !Number.isInteger(op.schemaVersion) || - // Floor of 1 matches the server contract (SuperSyncOperationSchema min(1)) - // so the layers cannot drift apart. - op.schemaVersion < 1 + // Deliberately accepts 0 (unlike the server contract's min(1)): a parsed 0 + // then fails the MIN_SUPPORTED_SCHEMA_VERSION comparison and surfaces as + // VERSION_UNSUPPORTED — a truthful message — instead of a generic + // migration-failure. Safety is identical either way. + op.schemaVersion < 0 ) { - throw new Error('Operation schemaVersion must be a positive integer when present.'); + throw new Error( + 'Operation schemaVersion must be a non-negative integer when present.', + ); } return op.schemaVersion; }; From a8f29a3f627549eab9d44261b5e7120aad292a5a Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:49:07 +0200 Subject: [PATCH 10/47] test(sync): avoid conflict-gate e2e deadlock --- .../sync/supersync-conflict-gate-non-task-work.spec.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index 4c7995f821..03ec247fc1 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -161,7 +161,10 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( // ===== PHASE 5: B syncs — the conflict dialog MUST appear ===== // triggerSync() does NOT auto-resolve dialogs, so we can assert on it. - await clientB.sync.triggerSync(); + const clientBSync = clientB.sync.triggerSync(); + // Prevent a teardown-triggered page close from becoming an unhandled + // rejection if an assertion below fails before we can await the sync. + clientBSync.catch(() => {}); await expect(clientB.sync.syncImportConflictDialog).toBeVisible({ timeout: 30000 }); console.log('[Gate] ✓ Conflict dialog shown for pending non-task work'); @@ -173,7 +176,7 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( state: 'hidden', timeout: 10000, }); - await clientB.sync.waitForSyncToComplete({ timeout: 30000 }); + await clientBSync; // The dialog itself is not the guarantee: prove the exact pending counter // value won, then run another sync and prove it remains durable. From fc20f81c2c147e5bb8fff0b224a524bd38e03af1 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:55:18 +0200 Subject: [PATCH 11/47] test(sync): force the upload-piggyback conflict race --- ...ersync-conflict-gate-non-task-work.spec.ts | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index 03ec247fc1..e2d8dd0f8c 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -27,9 +27,11 @@ import { ImportPage } from '../../pages/import.page'; * 2. Client B syncs and receives the counter (B now has synced history). * 3. Client B increments the counter locally — a pending SIMPLE_COUNTER op it * does NOT sync. - * 4. Client A imports a backup (creates a SYNC_IMPORT) and syncs it. - * 5. Client B triggers a sync. The SYNC_IMPORT conflict dialog MUST appear so - * the user can choose, instead of the counter change being discarded. + * 4. Client B starts syncing, but its completed download response is held. + * 5. Client A imports a backup and uploads its SYNC_IMPORT while B is between + * download and upload. + * 6. B's download is released. The upload response piggybacks A's import, and + * the conflict dialog MUST appear instead of discarding B's counter change. * * Run with: npm run e2e:supersync:file e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts */ @@ -110,7 +112,7 @@ const incrementClickCounter = async ( test.describe.configure({ mode: 'serial' }); test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', () => { - test('a pending simple-counter change prompts the conflict dialog instead of being discarded', async ({ + test('a pending simple-counter change survives an upload-piggybacked import conflict', async ({ browser, baseURL, testRunId, @@ -146,28 +148,57 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( await incrementClickCounter(clientB, counterTitle, 2); console.log('[Gate] Client B incremented the counter locally (pending, unsynced)'); - // ===== PHASE 4: A imports a backup (SYNC_IMPORT) and syncs it ===== + // ===== PHASE 4: B downloads before A's import, then pauses ===== + // The captured response is already fixed at the pre-import server state. + // Holding it here creates the exact download→upload race deterministically: + // A's later import can only reach B through the upload response piggyback. + let markDownloadCaptured!: () => void; + const downloadCaptured = new Promise((resolve) => { + markDownloadCaptured = resolve; + }); + let releaseDownload!: () => void; + const downloadRelease = new Promise((resolve) => { + releaseDownload = resolve; + }); + let hasCapturedDownload = false; + + await clientB.page.route('**/api/sync/ops**', async (route) => { + if (!hasCapturedDownload && route.request().method() === 'GET') { + const response = await route.fetch(); + hasCapturedDownload = true; + markDownloadCaptured(); + await downloadRelease; + await route.fulfill({ response }); + return; + } + await route.continue(); + }); + + await clientB.sync.syncBtn.click(); + await downloadCaptured; + console.log('[Gate] Client B completed its pre-import download; response paused'); + + // ===== PHASE 5: A imports + uploads while B is between download/upload ===== // A already synced a populated state (PHASE 1), so the import diverges from // the server: A's own sync raises the sync-import conflict gate against its // pending import op. Resolve it as USE_LOCAL ({ useLocal: true }) so A force // uploads the import as a NEW SYNC_IMPORT the server keeps. The default // (USE_REMOTE) would discard A's import here, leaving the server unchanged - // and B with nothing to conflict against — so the PHASE 5 dialog never shows. + // and B with nothing to conflict against — so the PHASE 6 dialog never shows. const importPageA = new ImportPage(clientA.page); - await importPageA.navigateToImportPage(); - await importPageA.importBackupFile(ImportPage.getFixturePath('test-backup.json')); - await clientA.sync.syncAndWait({ useLocal: true }); + try { + await importPageA.navigateToImportPage(); + await importPageA.importBackupFile(ImportPage.getFixturePath('test-backup.json')); + await clientA.sync.syncAndWait({ useLocal: true }); + } finally { + // Never strand B's routed request if A's setup/assertion fails. + releaseDownload(); + } console.log('[Gate] Client A imported a backup and synced the SYNC_IMPORT'); - // ===== PHASE 5: B syncs — the conflict dialog MUST appear ===== - // triggerSync() does NOT auto-resolve dialogs, so we can assert on it. - const clientBSync = clientB.sync.triggerSync(); - // Prevent a teardown-triggered page close from becoming an unhandled - // rejection if an assertion below fails before we can await the sync. - clientBSync.catch(() => {}); - + // ===== PHASE 6: B uploads and receives A's import via piggyback ===== await expect(clientB.sync.syncImportConflictDialog).toBeVisible({ timeout: 30000 }); - console.log('[Gate] ✓ Conflict dialog shown for pending non-task work'); + console.log('[Gate] ✓ Piggyback conflict shown for pending non-task work'); // Resolving with "Use My Data" keeps B's local work; the point of the test // is that B was given the choice at all. @@ -176,7 +207,7 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( state: 'hidden', timeout: 10000, }); - await clientBSync; + await clientB.sync.syncSpinner.waitFor({ state: 'hidden', timeout: 30000 }); // The dialog itself is not the guarantee: prove the exact pending counter // value won, then run another sync and prove it remains durable. From 16af83964194baec1b39781ed317e34b249c54d3 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 16:56:50 +0200 Subject: [PATCH 12/47] test(sync): align example-task gate spec with widened conflict gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The integration spec still pinned the pre-branch semantics where a pending GLOBAL_CONFIG op counted as non-meaningful. The widened gate (79f91e36fe) deliberately treats config changes as meaningful user work (entity-wide exemptions are unsafe — GLOBAL_CONFIG carries synced preferences), pinned by the gate unit spec but missed here, so the full suite failed since that commit. Split the case: example-only pending stays silent + discardable; config pending now expects the dialog. --- ...ample-task-import-gate.integration.spec.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts b/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts index f453f12ee3..cadfde53b1 100644 --- a/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts +++ b/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts @@ -73,10 +73,9 @@ describe('Example-task SYNC_IMPORT gate (integration)', () => { resetTestUuidCounter(); }); - it('treats pending example-task creates + config as non-meaningful and reports them as discardable', async () => { + it('treats pending example-task creates as non-meaningful and reports them as discardable', async () => { const example1 = exampleTaskOp('example-task-1'); const example2 = exampleTaskOp('example-task-2'); - await storeService.append(configOp(), 'local'); await storeService.append(example1, 'local'); await storeService.append(example2, 'local'); @@ -90,6 +89,23 @@ describe('Example-task SYNC_IMPORT gate (integration)', () => { ); }); + it('treats a pending config change as meaningful user work (widened gate) and shows the dialog', async () => { + // The gate deliberately has NO entity-wide exemptions beyond example-task + // creates: GLOBAL_CONFIG carries synced preferences, so silently discarding + // a pending config op on an incoming SYNC_IMPORT would lose user work. + const example = exampleTaskOp('example-task-1'); + await storeService.append(configOp(), 'local'); + await storeService.append(example, 'local'); + + const result = await gate.checkIncomingFullStateConflict([incomingSyncImport()]); + + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); + // Example ops stay listed so the caller can discard them if the user + // accepts the import. + expect(result.discardablePendingOpIds).toEqual([example.id]); + }); + it('actually excludes example-task ops from getUnsynced after markRejected (so they are not uploaded)', async () => { const config = configOp(); const example = exampleTaskOp('example-task-1'); From 360f95ed7ae094e0192baada9b55c727f6a4759a Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 17:15:13 +0200 Subject: [PATCH 13/47] fix(sync): preserve device-local sync settings in rebuild baseline USE_REMOTE's atomic rebuild synthesizes a globalConfig shell from DEFAULT_GLOBAL_CONFIG (getDefaultMainModelData omits globalConfig), then re-applies the device's local-only sync settings (provider, isEnabled, isEncryptionEnabled, syncInterval, isManualSyncOnly) onto the baseline. Without this, an interrupted rebuild committed a baseline whose sync config was pure defaults, so the resumed client could lose the provider and schedule it needs to sync again. Adds a unit test asserting the device-local settings survive into runRemoteStateReplacement's baseline. --- .../sync/operation-log-sync.service.spec.ts | 46 ++++++++++++++++++- .../op-log/sync/operation-log-sync.service.ts | 42 ++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 7f38e0de66..dfc8711d86 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -21,7 +21,7 @@ import { RemoteOpsProcessingService } from './remote-ops-processing.service'; import { RejectedOpsHandlerService } from './rejected-ops-handler.service'; import { OperationWriteFlushService } from './operation-write-flush.service'; import { SuperSyncStatusService } from './super-sync-status.service'; -import { provideMockStore } from '@ngrx/store/testing'; +import { MockStore, provideMockStore } from '@ngrx/store/testing'; import { ActionType, Operation, @@ -38,6 +38,9 @@ import { T } from '../../t.const'; import { INBOX_PROJECT } from '../../features/project/project.const'; import { TODAY_TAG, SYSTEM_TAG_IDS } from '../../features/tag/tag.const'; import { OperationSyncCapable } from '../sync-providers/provider.interface'; +import { selectSyncConfig } from '../../features/config/store/global-config.reducer'; +import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; +import { SyncProviderId } from '../sync-providers/provider.const'; describe('OperationLogSyncService', () => { let service: OperationLogSyncService; @@ -2508,6 +2511,47 @@ describe('OperationLogSyncService', () => { expect(callOrder).toEqual(['downloadRemoteOps', 'runRemoteStateReplacement']); }); + it('should preserve device-local sync settings in the atomic rebuild baseline', async () => { + const mockStore = TestBed.inject(MockStore); + mockStore.overrideSelector(selectSyncConfig, { + ...DEFAULT_GLOBAL_CONFIG.sync, + syncProvider: SyncProviderId.SuperSync, + isEnabled: true, + isEncryptionEnabled: true, + syncInterval: 17, + isManualSyncOnly: true, + }); + mockStore.refreshState(); + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await service.forceDownloadRemoteState(mockProvider); + + const baselineState = opLogStoreSpy.runRemoteStateReplacement.calls.mostRecent() + .args[0].baselineState as { + globalConfig: { sync: Record }; + }; + expect(baselineState.globalConfig.sync).toEqual( + jasmine.objectContaining({ + syncProvider: SyncProviderId.SuperSync, + isEnabled: true, + isEncryptionEnabled: true, + syncInterval: 17, + isManualSyncOnly: true, + }), + ); + }); + it('should capture a safety backup after download but before replacement (#8107)', async () => { const callOrder: string[] = []; downloadServiceSpy.downloadRemoteOps.and.callFake(async () => { diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 3050d00754..1d43b0b699 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -53,6 +53,13 @@ import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; import { Operation } from '../core/operation.types'; import { ValidateStateService } from '../validation/validate-state.service'; import { extractEntityKeysFromState } from '../persistence/extract-entity-keys'; +import { firstValueFrom } from 'rxjs'; +import { selectSyncConfig } from '../../features/config/store/global-config.reducer'; +import { + applyLocalOnlySyncSettingsToAppData, + LocalOnlySyncSettings, +} from '../../features/config/local-only-sync-settings.util'; +import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; /** * Orchestrates synchronization of the Operation Log with remote storage. @@ -1264,7 +1271,40 @@ export class OperationLogSyncService { rebuiltClock = mergeVectorClocks(rebuiltClock, opClock); } const defaultData = getDefaultMainModelData(); - const baselineState = snapshotState ?? defaultData; + const baselineSource = snapshotState ?? defaultData; + const baselineGlobalConfig = + baselineSource['globalConfig'] && typeof baselineSource['globalConfig'] === 'object' + ? (baselineSource['globalConfig'] as Record) + : {}; + const baselineSyncConfig = + baselineGlobalConfig['sync'] && typeof baselineGlobalConfig['sync'] === 'object' + ? (baselineGlobalConfig['sync'] as Record) + : {}; + const currentSyncConfig = await firstValueFrom(this.store.select(selectSyncConfig)); + const localOnlySyncSettings: LocalOnlySyncSettings = { + isEnabled: currentSyncConfig.isEnabled, + isEncryptionEnabled: currentSyncConfig.isEncryptionEnabled, + syncProvider: currentSyncConfig.syncProvider, + syncInterval: currentSyncConfig.syncInterval, + isManualSyncOnly: currentSyncConfig.isManualSyncOnly, + }; + // getDefaultMainModelData intentionally excludes globalConfig. Add a + // default config shell before applying the canonical device-local fields + // so an interrupted rebuild can hydrate enough configuration to sync again. + const baselineState = applyLocalOnlySyncSettingsToAppData( + { + ...baselineSource, + globalConfig: { + ...DEFAULT_GLOBAL_CONFIG, + ...baselineGlobalConfig, + sync: { + ...DEFAULT_GLOBAL_CONFIG.sync, + ...baselineSyncConfig, + }, + }, + }, + localOnlySyncSettings, + ); const archiveYoung = (snapshotState?.[ 'archiveYoung' From 19b6fed044fa70ea3f5e04762f930832d353a358 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 17:15:51 +0200 Subject: [PATCH 14/47] test(e2e): cover USE_REMOTE interrupted rebuild crash resume Reloads client B at the exact atomic-baseline-commit cutoff, then verifies the next sync detects the interrupted rebuild, redoes the raw download, keeps the first attempt's pre-replace backup, finishes on the remote state, and offers an Undo that restores B's original import. --- .../supersync-use-remote-crash-resume.spec.ts | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 e2e/tests/sync/supersync-use-remote-crash-resume.spec.ts diff --git a/e2e/tests/sync/supersync-use-remote-crash-resume.spec.ts b/e2e/tests/sync/supersync-use-remote-crash-resume.spec.ts new file mode 100644 index 0000000000..a6deae61d7 --- /dev/null +++ b/e2e/tests/sync/supersync-use-remote-crash-resume.spec.ts @@ -0,0 +1,177 @@ +import { test, expect } from '../../fixtures/supersync.fixture'; +import { ImportPage } from '../../pages/import.page'; +import { + closeClient, + createSimulatedClient, + createTestUser, + getSuperSyncConfig, + type SimulatedE2EClient, +} from '../../utils/supersync-helpers'; +import { waitForAppReady } from '../../utils/waits'; + +const CRASH_STATE_KEY = 'e2e-use-remote-crash-state'; +const CRASH_LOG = '[CrashResume] Simulating reload after remote baseline commit'; +const REBUILD_COMMITTED_LOG = + 'OperationLogSyncService: Replaced local persistence with remote baseline.'; +const RESUME_DETECTED_LOG = + 'OperationLogSyncService: Interrupted USE_REMOTE rebuild detected'; + +/** + * SuperSync USE_REMOTE crash recovery. + * + * The test reloads Client B immediately after runRemoteStateReplacement() has + * atomically committed the remote baseline and its raw-rebuild-incomplete marker, + * but before replay can finish and clear that marker. On the next sync, B must: + * + * 1. detect the interrupted rebuild, + * 2. redo the raw server-history download, + * 3. keep the FIRST attempt's pre-replace import backup, + * 4. finish on the remote state, and + * 5. offer Undo that restores B's original imported state. + * + * Run with: + * npm run e2e:supersync:file e2e/tests/sync/supersync-use-remote-crash-resume.spec.ts + */ +test.describe('@supersync USE_REMOTE interrupted rebuild recovery', () => { + test.describe.configure({ mode: 'serial' }); + + test('resumes after reload and preserves the original Undo backup', async ({ + browser, + baseURL, + testRunId, + }) => { + test.setTimeout(180000); + + let clientA: SimulatedE2EClient | null = null; + let clientB: SimulatedE2EClient | null = null; + + try { + const user = await createTestUser(testRunId); + const syncConfig = getSuperSyncConfig(user); + const remoteTask = `CrashResumeRemote-${testRunId}`; + const importedTask = 'E2E Import Test - Active Task With Subtask'; + + // ===== PHASE 1: A establishes the remote state ===== + clientA = await createSimulatedClient(browser, baseURL!, 'A', testRunId); + await clientA.sync.setupSuperSync(syncConfig); + await clientA.workView.addTask(remoteTask); + await clientA.sync.syncAndWait(); + + // ===== PHASE 2: B installs a one-shot crash failpoint before app boot ===== + clientB = await createSimulatedClient(browser, baseURL!, 'B', testRunId); + await clientB.page.addInitScript( + ({ crashLog, crashStateKey, rebuildCommittedLog }) => { + const e2eGlobal = globalThis as typeof globalThis & { + __SP_E2E_BLOCK_AUTO_SYNC?: boolean; + __SP_E2E_BLOCK_IMMEDIATE_UPLOAD?: boolean; + __SP_E2E_BLOCK_WS_DOWNLOAD?: boolean; + }; + // Allow setup's initial sync before the crash. On the reload, block + // automatic sync so the test can observe and trigger recovery itself. + e2eGlobal.__SP_E2E_BLOCK_AUTO_SYNC = + sessionStorage.getItem(crashStateKey) === 'crashed'; + e2eGlobal.__SP_E2E_BLOCK_IMMEDIATE_UPLOAD = true; + e2eGlobal.__SP_E2E_BLOCK_WS_DOWNLOAD = true; + + // Install before Angular modules load so Log's pre-bound console method + // captures this wrapper. The session marker makes the reload one-shot. + const originalLog = console.log.bind(console); + console.log = (...args: unknown[]): void => { + originalLog(...args); + const message = args.map(String).join(' '); + if ( + sessionStorage.getItem(crashStateKey) === 'armed' && + message.includes(rebuildCommittedLog) + ) { + sessionStorage.setItem(crashStateKey, 'crashed'); + originalLog(crashLog); + // Abort the current replay task at the exact committed-baseline + // cutoff. Playwright reloads the document after observing crashLog. + throw new Error(crashLog); + } + }; + }, + { + crashLog: CRASH_LOG, + crashStateKey: CRASH_STATE_KEY, + rebuildCommittedLog: REBUILD_COMMITTED_LOG, + }, + ); + await clientB.page.reload(); + await waitForAppReady(clientB.page); + + // Importing gives B a known local state and creates a full-state local op, + // guaranteeing the incoming server history requires an explicit choice. + const importPageB = new ImportPage(clientB.page); + await importPageB.navigateToImportPage(); + await importPageB.importBackupFile(ImportPage.getFixturePath('test-backup.json')); + await clientB.page.goto('/#/tag/TODAY/tasks'); + await clientB.page.waitForURL(/(active\/tasks|tag\/TODAY\/tasks)/); + await expect(clientB.page.locator('task', { hasText: importedTask })).toBeVisible({ + timeout: 15000, + }); + + await clientB.page.evaluate( + ({ crashStateKey }) => sessionStorage.setItem(crashStateKey, 'armed'), + { crashStateKey: CRASH_STATE_KEY }, + ); + + // Setup starts the first sync but leaves its conflict dialog for the test. + await clientB.sync.setupSuperSync({ + ...syncConfig, + waitForInitialSync: false, + }); + await expect(clientB.sync.syncImportConflictDialog).toBeVisible({ timeout: 30000 }); + + // ===== PHASE 3: choose USE_REMOTE and reload at the atomic baseline cutoff ===== + const crashObserved = clientB.page.waitForEvent('console', { + predicate: (message) => message.text().includes(CRASH_LOG), + timeout: 30000, + }); + + await clientB.sync.syncImportUseRemoteBtn.click(); + await crashObserved; + await clientB.page.reload(); + await waitForAppReady(clientB.page); + expect( + await clientB.page.evaluate( + ({ crashStateKey }) => sessionStorage.getItem(crashStateKey), + { crashStateKey: CRASH_STATE_KEY }, + ), + ).toBe('crashed'); + + // ===== PHASE 4: next sync must resume raw rebuild and keep first backup ===== + const resumeDetected = clientB.page.waitForEvent('console', { + predicate: (message) => message.text().includes(RESUME_DETECTED_LOG), + timeout: 30000, + }); + await clientB.sync.syncAndWait({ timeout: 60000 }); + await resumeDetected; + + // B was already on Today before the reload. Keep the current work context: + // changing it intentionally dismisses snacks, including the persistent Undo. + await expect(clientB.page.locator('task', { hasText: remoteTask })).toBeVisible({ + timeout: 15000, + }); + await expect( + clientB.page.locator('task', { hasText: importedTask }), + ).not.toBeVisible(); + + const undoSnack = clientB.page.locator('snack-custom', { + hasText: /replaced with the server/i, + }); + await expect(undoSnack).toBeVisible({ timeout: 30000 }); + await undoSnack.locator('button.action').click(); + + // Undo must restore the backup from BEFORE the first, interrupted replace, + // not a backup of the partial remote baseline captured during crash resume. + await expect(clientB.page.locator('task', { hasText: importedTask })).toBeVisible({ + timeout: 15000, + }); + console.log('[CrashResume] ✓ Original imported state restored from first backup'); + } finally { + if (clientA) await closeClient(clientA); + if (clientB) await closeClient(clientB); + } + }); +}); From a3272ed138e2efea4a2286e6ab2d7ae0e44b3b6c Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 17:38:47 +0200 Subject: [PATCH 15/47] fix(sync): keep USE_REMOTE recovery Undo visible after routine sync After a "Use Server Data" replace shows the persistent Undo recovery snack (#8107), a follow-up routine sync-success snack must not silently replace it. SnackService now tracks a pending persistent action (actionStr + duration 0); the header sync() skips its success feedback while one is showing. Unblocks the committed USE_REMOTE crash-resume e2e (supersync-use-remote-crash-resume.spec.ts), whose Undo assertion depends on the recovery snack surviving the sync that resumed the rebuild. --- .../main-header/main-header.component.spec.ts | 25 ++++++++++++++++++- .../main-header/main-header.component.ts | 5 ++++ src/app/core/snack/snack.service.ts | 24 +++++++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/app/core-ui/main-header/main-header.component.spec.ts b/src/app/core-ui/main-header/main-header.component.spec.ts index dc5063e2a9..810b88366c 100644 --- a/src/app/core-ui/main-header/main-header.component.spec.ts +++ b/src/app/core-ui/main-header/main-header.component.spec.ts @@ -25,6 +25,7 @@ import { MetricService } from '../../features/metric/metric.service'; import { DateService } from '../../core/date/date.service'; import { UserProfileService } from '../../features/user-profile/user-profile.service'; import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; +import { SyncStatus } from '../../op-log/sync-exports'; // Regression test for #7477: in a project view a long title pushed the // right-side header actions (simple-counter / habit buttons) off screen. @@ -166,6 +167,7 @@ describe('MainHeaderComponent focus button visibility', () => { { provide: SyncWrapperService, useValue: { + sync: jasmine.createSpy('sync'), isEnabledAndReady$: of(false), syncState$: of('IN_SYNC'), isSyncInProgress$: of(false), @@ -173,7 +175,13 @@ describe('MainHeaderComponent focus button visibility', () => { superSyncIsConfirmedInSync$: of(false), }, }, - { provide: SnackService, useValue: { open: jasmine.createSpy('open') } }, + { + provide: SnackService, + useValue: { + open: jasmine.createSpy('open'), + hasPendingPersistentAction: jasmine.createSpy('hasPendingPersistentAction'), + }, + }, { provide: Router, useValue: { events: EMPTY } }, { provide: GlobalConfigService, @@ -229,4 +237,19 @@ describe('MainHeaderComponent focus button visibility', () => { expect(component.isFocusButtonVisible()).toBe(false); }); + + it('keeps a persistent recovery action instead of showing routine sync success', async () => { + component = createComponent(); + const syncWrapperService = TestBed.inject( + SyncWrapperService, + ) as jasmine.SpyObj; + const snackService = TestBed.inject(SnackService) as jasmine.SpyObj; + syncWrapperService.sync.and.resolveTo(SyncStatus.UpdateRemote); + snackService.hasPendingPersistentAction.and.returnValue(true); + + component.sync(); + await Promise.resolve(); + + expect(snackService.open).not.toHaveBeenCalled(); + }); }); diff --git a/src/app/core-ui/main-header/main-header.component.ts b/src/app/core-ui/main-header/main-header.component.ts index caceffc7f2..6aa1229246 100644 --- a/src/app/core-ui/main-header/main-header.component.ts +++ b/src/app/core-ui/main-header/main-header.component.ts @@ -294,6 +294,11 @@ export class MainHeaderComponent implements OnDestroy { sync(): void { this.syncWrapperService.sync(true).then((r) => { + // Keep persistent recovery actions (for example USE_REMOTE Undo) visible; + // routine sync-success feedback must not replace them. + if (this._snackService.hasPendingPersistentAction()) { + return; + } if ( r === SyncStatus.UpdateLocal || r === SyncStatus.UpdateRemoteAll || diff --git a/src/app/core/snack/snack.service.ts b/src/app/core/snack/snack.service.ts index 9a5833cb4d..52eb7f0e69 100644 --- a/src/app/core/snack/snack.service.ts +++ b/src/app/core/snack/snack.service.ts @@ -3,7 +3,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { Store } from '@ngrx/store'; import { SnackParams } from './snack.model'; import { Observable, Subject } from 'rxjs'; -import { takeUntil } from 'rxjs/operators'; +import { take, takeUntil } from 'rxjs/operators'; import { DEFAULT_SNACK_CFG } from './snack.const'; import { SnackCustomComponent } from './snack-custom/snack-custom.component'; import { TranslateService } from '@ngx-translate/core'; @@ -23,6 +23,7 @@ export class SnackService { private _matSnackBar = inject(MatSnackBar); private _ref?: MatSnackBarRef; + private _hasPendingPersistentAction = false; constructor() { const _onWorkContextChange$: Observable = this._actions$.pipe( @@ -37,10 +38,20 @@ export class SnackService { if (typeof params === 'string') { params = { msg: params }; } + if (params.actionStr && params.config?.duration === 0) { + // Set this before the debounced render so immediate follow-up feedback + // cannot unknowingly replace a persistent recovery action. + this._hasPendingPersistentAction = true; + } this._openSnack(params); } + hasPendingPersistentAction(): boolean { + return this._hasPendingPersistentAction; + } + close(): void { + this._hasPendingPersistentAction = false; if (this._ref) { this._ref.dismiss(); } @@ -107,6 +118,17 @@ export class SnackService { } } + const openedRef = this._ref; + this._hasPendingPersistentAction = !!actionStr && cfg.duration === 0; + openedRef + ?.afterDismissed() + .pipe(take(1)) + .subscribe(() => { + if (this._ref === openedRef) { + this._hasPendingPersistentAction = false; + } + }); + if (actionStr && actionId && this._ref) { this._ref .onAction() From be0d6e7dd01028bfed367132200c4447c49272b4 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 18:41:09 +0200 Subject: [PATCH 16/47] fix(sync): surface Undo when an interrupted USE_REMOTE resume can't finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-review remediation for the USE_REMOTE rebuild path. - Crash-resume recovery gap (confirmed by 2 independent reviewers): when an interrupted-rebuild resume aborts in its download/validate phase (empty or newer-schema remote, or a persistent download failure), forceDownloadRemoteState threw before it could offer Undo. The prior attempt had already committed the destructive baseline, so the pre-replace backup was stranded with no restore affordance — reading as total data loss. downloadRemoteOps now surfaces the recovery Undo on a resume abort (deduped via hasPendingPersistentAction so repeated auto/WS syncs don't respawn it). Covered by two new unit tests. - SnackService: collapse the redundant dual write of the persistent-action flag (set in open() AND recomputed in _openSnack) to one authoritative write in open(); _openSnack keeps only the on-dismiss clear (3-reviewer consensus). - Document the archive-retry idempotency invariant on ARCHIVE_AFFECTING_ACTION_TYPES: the hydrator retry re-runs archive side effects with skipReducerDispatch even after a fully-successful run, so they must stay idempotent (overwrite, never additive) to avoid double-counting time-tracking/counter deltas. --- src/app/core/snack/snack.service.ts | 18 +++++--- .../archive-operation-handler.service.ts | 9 ++++ .../sync/operation-log-sync.service.spec.ts | 42 ++++++++++++++++++- .../op-log/sync/operation-log-sync.service.ts | 32 +++++++++++++- 4 files changed, 93 insertions(+), 8 deletions(-) diff --git a/src/app/core/snack/snack.service.ts b/src/app/core/snack/snack.service.ts index 52eb7f0e69..08eeda69fa 100644 --- a/src/app/core/snack/snack.service.ts +++ b/src/app/core/snack/snack.service.ts @@ -38,11 +38,15 @@ export class SnackService { if (typeof params === 'string') { params = { msg: params }; } - if (params.actionStr && params.config?.duration === 0) { - // Set this before the debounced render so immediate follow-up feedback - // cannot unknowingly replace a persistent recovery action. - this._hasPendingPersistentAction = true; - } + // Track a persistent recovery action (a sticky, actionable snack) + // synchronously, before the debounced render, so an immediate follow-up + // check (e.g. the header's post-sync success feedback) cannot unknowingly + // replace it. `_openSnack` is debounced trailing-edge, so this last-writer + // value matches the snack it will actually render — including the case + // where a non-persistent snack supersedes a persistent one. + this._hasPendingPersistentAction = !!( + params.actionStr && params.config?.duration === 0 + ); this._openSnack(params); } @@ -118,8 +122,10 @@ export class SnackService { } } + // The pending-persistent-action flag is set authoritatively in open() + // before this debounced render. Here we only clear it once this specific + // snack is dismissed (guarded so a newer snack's flag isn't cleared). const openedRef = this._ref; - this._hasPendingPersistentAction = !!actionStr && cfg.duration === 0; openedRef ?.afterDismissed() .pipe(take(1)) diff --git a/src/app/op-log/apply/archive-operation-handler.service.ts b/src/app/op-log/apply/archive-operation-handler.service.ts index d38595e493..b7c98ac393 100644 --- a/src/app/op-log/apply/archive-operation-handler.service.ts +++ b/src/app/op-log/apply/archive-operation-handler.service.ts @@ -36,6 +36,15 @@ const createEmptyArchiveModel = (): ArchiveModel => ({ /** * Action types that affect archive storage and require special handling. + * + * INVARIANT: every archive side effect for these actions must be idempotent + * even when it already fully succeeded once. Crash recovery marks interrupted + * remote ops `archive_pending` (OperationLogRecoveryService) and the hydrator + * retry re-runs their archive work with `skipReducerDispatch` — the crash + * window is between archive completion and `markApplied()`, so a re-run can hit + * an archive that is already up to date. Use id-keyed writes / overwrites, never + * additive merges (e.g. `archiveTT[...] = value`, not `+=`); an additive path + * here would silently double-count time-tracking / counter deltas on this path. */ const ARCHIVE_AFFECTING_ACTION_TYPES: string[] = [ TaskSharedActions.moveToArchive.type, diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index dfc8711d86..05e0cbdd62 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -59,7 +59,11 @@ describe('OperationLogSyncService', () => { let lockServiceSpy: jasmine.SpyObj; beforeEach(() => { - snackServiceSpy = jasmine.createSpyObj('SnackService', ['open']); + snackServiceSpy = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); opLogStoreSpy = jasmine.createSpyObj('OperationLogStoreService', [ 'getUnsynced', 'loadStateCache', @@ -917,6 +921,42 @@ describe('OperationLogSyncService', () => { expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); }); + it('should offer the stranded pre-replace backup when an interrupted rebuild resume cannot finish', async () => { + // Resume path: the prior attempt already committed the destructive + // baseline, but this resume aborts in its download/validate phase (e.g. + // empty/newer-schema remote). Without an escape hatch the user is stuck + // on the baseline with the pre-replace backup hidden — surface Undo. + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + spyOn(service, 'forceDownloadRemoteState').and.rejectWith( + new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'), + ); + opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); + const mockProvider = { isReady: () => Promise.resolve(true) } as any; + + await expectAsync(service.downloadRemoteOps(mockProvider)).toBeRejected(); + + expect(snackServiceSpy.open).toHaveBeenCalledWith( + jasmine.objectContaining({ msg: T.F.SYNC.S.LOCAL_DATA_REPLACE_UNDO }), + ); + }); + + it('should not respawn the recovery snack while one is already showing', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + spyOn(service, 'forceDownloadRemoteState').and.rejectWith( + new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'), + ); + opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + // A persistent recovery snack from a previous resume attempt is still up. + snackServiceSpy.hasPendingPersistentAction.and.returnValue(true); + const mockProvider = { isReady: () => Promise.resolve(true) } as any; + + await expectAsync(service.downloadRemoteOps(mockProvider)).toBeRejected(); + + expect(opLogStoreSpy.loadImportBackup).not.toHaveBeenCalled(); + expect(snackServiceSpy.open).not.toHaveBeenCalled(); + }); + it('should return localWinOpsCreated: 0 and newOpsCount: 0 when no new ops', async () => { downloadServiceSpy.downloadRemoteOps.and.returnValue( Promise.resolve({ diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 1d43b0b699..970491c5d3 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -451,7 +451,19 @@ export class OperationLogSyncService { OpLog.warn( 'OperationLogSyncService: Interrupted USE_REMOTE rebuild detected — redoing the raw rebuild.', ); - await this.forceDownloadRemoteState(syncProvider, { isCrashResume: true }); + try { + await this.forceDownloadRemoteState(syncProvider, { isCrashResume: true }); + } catch (e) { + // The prior attempt already committed the destructive baseline (that is + // why we are resuming), so the user's original data now lives only in + // the pre-replace backup. If this resume cannot finish — empty/newer- + // schema remote, or a persistent download failure — forceDownloadRemoteState + // throws in its download/validate phase, before it can offer Undo, and + // the backup would otherwise stay stranded with no restore affordance + // (reads as total data loss). Surface the recovery Undo before rethrowing. + await this._offerStrandedRebuildBackup(); + throw e; + } // State was replaced wholesale, exactly like a snapshot hydration. return { kind: 'snapshot_hydrated' }; } @@ -1559,6 +1571,24 @@ export class OperationLogSyncService { }); } + /** + * Offer the pre-replace Undo after an interrupted USE_REMOTE rebuild whose + * resume could not finish. The first attempt already committed the destructive + * baseline, so the user's original data survives only in the IMPORT_BACKUP + * slot; without an explicit affordance here it has no restore entry point and + * reads as total data loss. Skipped while a persistent recovery snack is + * already showing, so repeated resume attempts (auto/WS syncs) don't respawn it. + */ + private async _offerStrandedRebuildBackup(): Promise { + if (this.snackService.hasPendingPersistentAction()) { + return; + } + const backup = await this.opLogStore.loadImportBackup(); + if (backup?.savedAt !== undefined) { + this._showRestorePreviousDataSnack(backup.savedAt); + } + } + /** * Checks if there's an encryption state mismatch between local config and server data. * If the server has only unencrypted data but local config has encryption enabled, From ab9e1d38c9902a2653eeabdc1db944c7354a1a36 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:08:38 +0200 Subject: [PATCH 17/47] fix(sync): exempt fresh-client startup ops from conflict gate Fresh clients generate startup config and genesis operations before their first sync. Treat those operations as setup state until a prior sync exists, while continuing to protect later config changes and all ordinary user work from authoritative snapshot replacement. --- .../op-log/sync/operation-log-sync.service.ts | 8 ++- .../sync-import-conflict-gate.service.spec.ts | 36 +++++++--- .../sync/sync-import-conflict-gate.service.ts | 70 ++++++++++++------- 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 970491c5d3..ecc341401b 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -582,9 +582,13 @@ export class OperationLogSyncService { .filter((id): id is string => id !== undefined), ); const exampleTaskOpIds = exampleTaskEntries.map((entry) => entry.op.id); + // Nothing from this sync is persisted yet, so this live read reflects + // whether the client completed a prior sync cycle. + const hasCompletedInitialSync = await this.opLogStore.hasSyncedOps(); const hasMeaningfulUserData = - this.syncImportConflictGateService.hasMeaningfulPendingOps(unsyncedOps) || - this.syncLocalStateService.hasMeaningfulStoreData(exampleTaskIds); + this.syncImportConflictGateService.hasMeaningfulPendingOps(unsyncedOps, { + hasCompletedInitialSync, + }) || this.syncLocalStateService.hasMeaningfulStoreData(exampleTaskIds); if (hasMeaningfulUserData) { // SPAP-9: before surfacing the binary USE_LOCAL/USE_REMOTE dialog, use diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts index 0adea0c7d2..81795af8fc 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts @@ -94,9 +94,8 @@ describe('SyncImportConflictGateService', () => { }); }); - it('should produce dialog data when pending ops contain user config changes', async () => { - const incomingSyncImport = createOperation(); - const pendingConfigEntry = createEntry( + const createPendingConfigEntry = (): OperationLogEntry => + createEntry( createOperation({ id: 'local-config-update', actionType: '[Global Config] Update Global Config Section' as ActionType, @@ -108,15 +107,27 @@ describe('SyncImportConflictGateService', () => { vectorClock: { clientA: 1 }, }), ); - opLogStoreSpy.getUnsynced.and.resolveTo([pendingConfigEntry]); - const result = await service.checkIncomingFullStateConflict([incomingSyncImport]); + it('should produce dialog data for a synced client with a pending config change', async () => { + opLogStoreSpy.hasSyncedOps.and.resolveTo(true); + opLogStoreSpy.getUnsynced.and.resolveTo([createPendingConfigEntry()]); + + const result = await service.checkIncomingFullStateConflict([createOperation()]); - expect(result.fullStateOp).toBe(incomingSyncImport); expect(result.hasMeaningfulPending).toBeTrue(); expect(result.dialogData).toBeDefined(); }); + it('should NOT flag a pending config change on a never-synced client', async () => { + opLogStoreSpy.hasSyncedOps.and.resolveTo(false); + opLogStoreSpy.getUnsynced.and.resolveTo([createPendingConfigEntry()]); + + const result = await service.checkIncomingFullStateConflict([createOperation()]); + + expect(result.hasMeaningfulPending).toBeFalse(); + expect(result.dialogData).toBeUndefined(); + }); + it('should not produce dialog data when pending task creates are startup example tasks', async () => { const incomingSyncImport = createOperation(); const pendingExampleTaskEntry = createEntry( @@ -391,7 +402,7 @@ describe('SyncImportConflictGateService', () => { expect(service.hasMeaningfulPendingOps([pendingCounter])).toBeTrue(); }); - it('should treat MIGRATION/RECOVERY genesis batches as meaningful recovered user data', async () => { + it('should treat MIGRATION/RECOVERY genesis batches as meaningful only once the client has synced', async () => { const pendingMigration = createEntry( createOperation({ id: 'local-genesis', @@ -419,8 +430,15 @@ describe('SyncImportConflictGateService', () => { { seq: 2 }, ); - expect(service.hasMeaningfulPendingOps([pendingMigration])).toBeTrue(); - expect(service.hasMeaningfulPendingOps([pendingRecovery])).toBeTrue(); + const synced = { hasCompletedInitialSync: true }; + expect(service.hasMeaningfulPendingOps([pendingMigration], synced)).toBeTrue(); + expect(service.hasMeaningfulPendingOps([pendingRecovery], synced)).toBeTrue(); + + const neverSynced = { hasCompletedInitialSync: false }; + expect( + service.hasMeaningfulPendingOps([pendingMigration], neverSynced), + ).toBeFalse(); + expect(service.hasMeaningfulPendingOps([pendingRecovery], neverSynced)).toBeFalse(); }); }); diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index 5a530864fa..b90ee427f8 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -10,6 +10,13 @@ import { OperationWriteFlushService } from './operation-write-flush.service'; import { SyncImportConflictData } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; +/** + * Startup/system entities are written automatically during first-time setup. + * Before the client has completed a sync they are setup state, not divergence; + * after that, later writes are genuine local changes. + */ +const STARTUP_ENTITY_TYPES = new Set(['GLOBAL_CONFIG', 'MIGRATION', 'RECOVERY']); + export interface IncomingFullStateConflictGateResult { fullStateOp?: Operation; pendingOps: OperationLogEntry[]; @@ -34,13 +41,20 @@ export class SyncImportConflictGateService { private writeFlushService = inject(OperationWriteFlushService); /** - * Every pending op is user work unless it is an onboarding example-task create. - * Entity-wide exemptions are unsafe: GLOBAL_CONFIG contains synced preferences, - * while MIGRATION and RECOVERY genesis operations contain the user's full recovered - * database. Full-state ops are always meaningful because applying a newer full-state - * op can invalidate their local import/repair semantics. + * Every pending op is user work unless it is an onboarding example-task create, + * or a startup/system write on a client that has not yet completed a sync. + * Full-state ops are always meaningful because applying a newer full-state op + * can invalidate their local import/repair semantics. + * + * The lifecycle default is deliberately conservative: a caller that does not + * know whether initial sync completed must protect startup-entity changes. */ - hasMeaningfulPendingOps(ops: OperationLogEntry[]): boolean { + hasMeaningfulPendingOps( + ops: OperationLogEntry[], + options: { hasCompletedInitialSync: boolean } = { + hasCompletedInitialSync: true, + }, + ): boolean { return ops.some((entry) => { if (FULL_STATE_OP_TYPES.has(entry.op.opType as OpType)) { return true; @@ -48,6 +62,9 @@ export class SyncImportConflictGateService { if (isExampleTaskCreateOp(entry)) { return false; } + if (STARTUP_ENTITY_TYPES.has(entry.op.entityType)) { + return options.hasCompletedInitialSync; + } return true; }); } @@ -87,12 +104,8 @@ export class SyncImportConflictGateService { const pendingOps = options.preCapturedPendingOps ? this._mergePendingOps(options.preCapturedPendingOps, livePendingOps) : livePendingOps; - const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); + // Example-task ops that the caller may reject when it accepts the import silently. - // When `hasMeaningfulPending` is true (real work pending alongside example tasks), - // the conflict dialog is shown instead and these are intentionally left untouched: - // if the user keeps local state, their example tasks ride along with the rest. - // // These must come from a LIVE read: with a pre-captured snapshot, example ops // accepted earlier in the same upload round are already marked synced and must // not be re-marked rejected by the caller. @@ -100,6 +113,27 @@ export class SyncImportConflictGateService { .filter(isExampleTaskCreateOp) .map((entry) => entry.op.id); + // Preserve the cheap example-task-only path. If nothing could be meaningful + // even for a synced client, there is no reason to read sync history. + const canContainMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); + if (!canContainMeaningfulPending) { + return { + fullStateOp, + pendingOps, + hasMeaningfulPending: false, + discardablePendingOpIds, + }; + } + + // Capture lifecycle state before deciding whether startup writes are setup + // noise or post-sync divergence. Piggyback callers pass their pre-sync snapshot + // because a live read there already includes this sync cycle's writes. + const isNeverSynced = + options.isNeverSynced ?? !(await this.opLogStore.hasSyncedOps()); + const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps, { + hasCompletedInitialSync: !isNeverSynced, + }); + const result = { fullStateOp, pendingOps, @@ -111,20 +145,6 @@ export class SyncImportConflictGateService { return result; } - // A client that has never completed a sync cannot have diverged from remote — its - // "meaningful" pending ops are pre-first-sync startup state (e.g. example tasks). - // Flag this so the dialog guards the destructive USE_LOCAL choice with an extra - // confirmation (it would overwrite the populated remote with throwaway data). - // - // Callers SHOULD pass `isNeverSynced` captured at sync-cycle start (pre-download). - // A live `hasSyncedOps()` here is unreliable on the piggyback-upload path: by the - // time that gate runs, this same sync has already persisted downloaded ops - // (syncedAt set) and marked accepted uploads synced, so reading it now would see - // post-sync state and wrongly clear the guard. Fall back to a live read only for - // standalone callers (e.g. the download path, where nothing is persisted yet). - const isNeverSynced = - options.isNeverSynced ?? !(await this.opLogStore.hasSyncedOps()); - return { ...result, dialogData: { From de8a2cb7b03cb1f764d36437cb1a486e532e7e14 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:18:20 +0200 Subject: [PATCH 18/47] fix(sync): preserve fresh-client recovery state --- .../imex/sync/sync-wrapper.service.spec.ts | 19 +++++++++++++- src/app/imex/sync/sync-wrapper.service.ts | 22 ++++++++++------ .../sync-import-conflict-gate.service.spec.ts | 26 +++++++++++++------ .../sync/sync-import-conflict-gate.service.ts | 18 ++++++------- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/app/imex/sync/sync-wrapper.service.spec.ts b/src/app/imex/sync/sync-wrapper.service.spec.ts index cc09584fdf..a4d2112e7e 100644 --- a/src/app/imex/sync/sync-wrapper.service.spec.ts +++ b/src/app/imex/sync/sync-wrapper.service.spec.ts @@ -157,7 +157,11 @@ describe('SyncWrapperService', () => { cfg$: configSubject.asObservable(), }); - mockSnackService = jasmine.createSpyObj('SnackService', ['open']); + mockSnackService = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + mockSnackService.hasPendingPersistentAction.and.returnValue(false); mockMatDialog = jasmine.createSpyObj('MatDialog', ['open'], { openDialogs: [], }); @@ -1843,6 +1847,19 @@ describe('SyncWrapperService', () => { ); }); + it('should preserve a persistent recovery action when sync rethrows', async () => { + mockSnackService.hasPendingPersistentAction.and.returnValue(true); + mockSyncService.downloadRemoteOps.and.rejectWith( + new Error('Interrupted rebuild could not resume'), + ); + + const result = await service.sync(); + + expect(result).toBe('HANDLED_ERROR'); + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).not.toHaveBeenCalled(); + }); + it('should treat UploadRevToMatchMismatchAPIError as transient: set UNKNOWN_OR_CHANGED, no error snackbar', async () => { mockSyncService.uploadPendingOps.and.returnValue( Promise.reject( diff --git a/src/app/imex/sync/sync-wrapper.service.ts b/src/app/imex/sync/sync-wrapper.service.ts index 0815ca6cbf..0c9280756b 100644 --- a/src/app/imex/sync/sync-wrapper.service.ts +++ b/src/app/imex/sync/sync-wrapper.service.ts @@ -935,14 +935,20 @@ export class SyncWrapperService { } else { this._providerManager.setSyncStatus('ERROR'); const errStr = getSyncErrorStr(error); - this._snackService.open({ - // msg: T.F.SYNC.S.UNKNOWN_ERROR, - msg: errStr, - type: 'ERROR', - translateParams: { - err: errStr, - }, - }); + // A lower-level recovery path may already have shown a sticky action + // (for example Undo after an interrupted remote-state rebuild). Snack + // rendering is debounced, so opening the generic error here would win + // the race and silently remove the only recovery action. + if (!this._snackService.hasPendingPersistentAction()) { + this._snackService.open({ + // msg: T.F.SYNC.S.UNKNOWN_ERROR, + msg: errStr, + type: 'ERROR', + translateParams: { + err: errStr, + }, + }); + } return 'HANDLED_ERROR'; } } diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts index 81795af8fc..860e258f09 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts @@ -94,15 +94,15 @@ describe('SyncImportConflictGateService', () => { }); }); - const createPendingConfigEntry = (): OperationLogEntry => + const createPendingConfigEntry = (sectionKey = 'sync'): OperationLogEntry => createEntry( createOperation({ id: 'local-config-update', actionType: '[Global Config] Update Global Config Section' as ActionType, opType: OpType.Update, entityType: 'GLOBAL_CONFIG', - entityId: 'sync', - payload: { sectionKey: 'sync' }, + entityId: sectionKey, + payload: { sectionKey }, clientId: 'client-A', vectorClock: { clientA: 1 }, }), @@ -128,6 +128,18 @@ describe('SyncImportConflictGateService', () => { expect(result.dialogData).toBeUndefined(); }); + it('should flag a user config change on a never-synced client', async () => { + opLogStoreSpy.hasSyncedOps.and.resolveTo(false); + opLogStoreSpy.getUnsynced.and.resolveTo([ + createPendingConfigEntry('productivityHacks'), + ]); + + const result = await service.checkIncomingFullStateConflict([createOperation()]); + + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); + }); + it('should not produce dialog data when pending task creates are startup example tasks', async () => { const incomingSyncImport = createOperation(); const pendingExampleTaskEntry = createEntry( @@ -402,7 +414,7 @@ describe('SyncImportConflictGateService', () => { expect(service.hasMeaningfulPendingOps([pendingCounter])).toBeTrue(); }); - it('should treat MIGRATION/RECOVERY genesis batches as meaningful only once the client has synced', async () => { + it('should always treat MIGRATION/RECOVERY genesis batches as meaningful', async () => { const pendingMigration = createEntry( createOperation({ id: 'local-genesis', @@ -435,10 +447,8 @@ describe('SyncImportConflictGateService', () => { expect(service.hasMeaningfulPendingOps([pendingRecovery], synced)).toBeTrue(); const neverSynced = { hasCompletedInitialSync: false }; - expect( - service.hasMeaningfulPendingOps([pendingMigration], neverSynced), - ).toBeFalse(); - expect(service.hasMeaningfulPendingOps([pendingRecovery], neverSynced)).toBeFalse(); + expect(service.hasMeaningfulPendingOps([pendingMigration], neverSynced)).toBeTrue(); + expect(service.hasMeaningfulPendingOps([pendingRecovery], neverSynced)).toBeTrue(); }); }); diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index b90ee427f8..a0a1f920b2 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -2,6 +2,7 @@ import { inject, Injectable } from '@angular/core'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { FULL_STATE_OP_TYPES, + extractActionPayload, Operation, OperationLogEntry, OpType, @@ -10,13 +11,6 @@ import { OperationWriteFlushService } from './operation-write-flush.service'; import { SyncImportConflictData } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; -/** - * Startup/system entities are written automatically during first-time setup. - * Before the client has completed a sync they are setup state, not divergence; - * after that, later writes are genuine local changes. - */ -const STARTUP_ENTITY_TYPES = new Set(['GLOBAL_CONFIG', 'MIGRATION', 'RECOVERY']); - export interface IncomingFullStateConflictGateResult { fullStateOp?: Operation; pendingOps: OperationLogEntry[]; @@ -42,7 +36,7 @@ export class SyncImportConflictGateService { /** * Every pending op is user work unless it is an onboarding example-task create, - * or a startup/system write on a client that has not yet completed a sync. + * or the sync-config write required to enable first sync on a fresh client. * Full-state ops are always meaningful because applying a newer full-state op * can invalidate their local import/repair semantics. * @@ -62,8 +56,12 @@ export class SyncImportConflictGateService { if (isExampleTaskCreateOp(entry)) { return false; } - if (STARTUP_ENTITY_TYPES.has(entry.op.entityType)) { - return options.hasCompletedInitialSync; + if ( + !options.hasCompletedInitialSync && + entry.op.entityType === 'GLOBAL_CONFIG' && + extractActionPayload(entry.op.payload).sectionKey === 'sync' + ) { + return false; } return true; }); From 87353d4b0aeeda46e931b7461f4e5db80d3227a6 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:23:07 +0200 Subject: [PATCH 19/47] fix(sync): preserve edits across rebuild resume --- .../operation-log-store.service.spec.ts | 27 +++++++ .../operation-log-store.service.ts | 35 ++++++++- .../sync/operation-log-sync.service.spec.ts | 74 ++++++++++++++++++- .../op-log/sync/operation-log-sync.service.ts | 67 +++++++++++++++++ 4 files changed, 199 insertions(+), 4 deletions(-) diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 422d7e8a97..ee4b544471 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -2404,6 +2404,33 @@ describe('OperationLogStoreService', () => { expect(await service.isRawRebuildIncomplete()).toBe(false); }); + it('durably carries post-crash local ops in the rebuild marker', async () => { + const preservedLocalOp = createTestOperation({ + id: '01900000-0000-7000-8000-000000000091', + entityId: 'edited-after-crash', + clientId: 'localClient', + vectorClock: { localClient: 2, remote: 1 }, + }); + + await service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('young'), + archiveOld: createArchive('old'), + preservedLocalOps: [preservedLocalOp], + }); + + expect(await service.getOpsAfterSeq(0)).toEqual([]); + expect(await service.loadRawRebuildIncomplete()).toEqual( + jasmine.objectContaining({ + incomplete: true, + preservedLocalOps: [preservedLocalOp], + }), + ); + }); + it('rolls back every store if one archive write fails', async () => { const priorOp = createTestOperation({ entityId: 'prior-task' }); const priorState = { sentinel: 'prior-state' }; diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index 1b387219ba..5bb985048c 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -80,6 +80,12 @@ interface StateCacheEntry { snapshotEntityKeys?: string[]; } +export interface RawRebuildIncompleteEntry { + incomplete: true; + startedAt: number; + preservedLocalOps: Operation[]; +} + /** * Stored operation log entry that can hold either compact or full operation format. * Used internally for backwards compatibility with existing data. @@ -1556,6 +1562,7 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { await this._ensureInit(); @@ -1584,7 +1591,11 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + return (await this.loadRawRebuildIncomplete()) !== null; + } + + /** + * Loads the durable resume marker, including local operations created after + * an interrupted replacement. Older markers did not contain the operation + * array, so they normalize to an empty list. + */ + async loadRawRebuildIncomplete(): Promise { await this._ensureInit(); - const entry = await this._adapter.get<{ incomplete?: boolean }>( + const entry = await this._adapter.get>( STORE_NAMES.META, RAW_REBUILD_INCOMPLETE_META_KEY, ); - return entry?.incomplete === true; + if (entry?.incomplete !== true) { + return null; + } + return { + incomplete: true, + startedAt: typeof entry.startedAt === 'number' ? entry.startedAt : 0, + preservedLocalOps: Array.isArray(entry.preservedLocalOps) + ? entry.preservedLocalOps + : [], + }; } /** Marks the USE_REMOTE raw rebuild as fully completed. */ diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 05e0cbdd62..1189c3f3a4 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -57,6 +57,7 @@ describe('OperationLogSyncService', () => { let schemaMigrationServiceSpy: jasmine.SpyObj; let validateStateServiceSpy: jasmine.SpyObj; let lockServiceSpy: jasmine.SpyObj; + let operationApplierSpy: jasmine.SpyObj; beforeEach(() => { snackServiceSpy = jasmine.createSpyObj('SnackService', [ @@ -78,10 +79,12 @@ describe('OperationLogSyncService', () => { 'hasSyncedOps', 'runRemoteStateReplacement', 'isRawRebuildIncomplete', + 'loadRawRebuildIncomplete', 'clearRawRebuildIncomplete', 'loadImportBackup', ]); opLogStoreSpy.hasSyncedOps.and.resolveTo(true); + opLogStoreSpy.getUnsynced.and.resolveTo([]); opLogStoreSpy.markSynced.and.resolveTo(); opLogStoreSpy.setVectorClock.and.resolveTo(); opLogStoreSpy.clearFullStateOps.and.resolveTo(); @@ -93,6 +96,7 @@ describe('OperationLogSyncService', () => { }); opLogStoreSpy.runRemoteStateReplacement.and.resolveTo(); opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo(null); opLogStoreSpy.clearRawRebuildIncomplete.and.resolveTo(); opLogStoreSpy.loadImportBackup.and.resolveTo(null); @@ -180,6 +184,10 @@ describe('OperationLogSyncService', () => { ['showConflictDialog'], ); syncImportConflictDialogServiceSpy.showConflictDialog.and.resolveTo('CANCEL'); + operationApplierSpy = jasmine.createSpyObj('OperationApplierService', [ + 'applyOperations', + ]); + operationApplierSpy.applyOperations.and.resolveTo({ appliedOps: [] }); TestBed.configureTestingModule({ providers: [ @@ -199,7 +207,7 @@ describe('OperationLogSyncService', () => { }, { provide: OperationApplierService, - useValue: jasmine.createSpyObj('OperationApplierService', ['applyOperations']), + useValue: operationApplierSpy, }, { provide: ConflictResolutionService, @@ -2739,6 +2747,70 @@ describe('OperationLogSyncService', () => { ); }); + it('should preserve and replay local edits made after an interrupted rebuild', async () => { + const previouslyPreserved = makeRemoteOp('local-before-second-crash'); + previouslyPreserved.clientId = 'local'; + previouslyPreserved.vectorClock = { local: 2, remote: 1 }; + const liveLocalEdit = makeRemoteOp('local-after-restart'); + liveLocalEdit.clientId = 'local'; + liveLocalEdit.vectorClock = { local: 3, remote: 1 }; + const liveEntry = { + seq: 2, + op: liveLocalEdit, + source: 'local' as const, + appliedAt: Date.now(), + }; + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp('remote-op')], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 4, + }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ state: {}, savedAt: 12345 }); + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [previouslyPreserved], + }); + opLogStoreSpy.getUnsynced.and.resolveTo([liveEntry]); + opLogStoreSpy.appendBatchSkipDuplicates.and.callFake(async (ops, source) => ({ + seqs: ops.map((_, index) => index + 10), + writtenOps: source === 'local' ? ops : [], + skippedCount: 0, + })); + operationApplierSpy.applyOperations.and.callFake(async (ops) => ({ + appliedOps: ops, + })); + opLogStoreSpy.getVectorClock.and.resolveTo({ remote: 4 }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await service.forceDownloadRemoteState(mockProvider, { isCrashResume: true }); + + const preservedLocalOps = [previouslyPreserved, liveLocalEdit]; + expect( + opLogStoreSpy.runRemoteStateReplacement.calls.mostRecent().args[0] + .preservedLocalOps, + ).toEqual(preservedLocalOps); + expect(opLogStoreSpy.appendBatchSkipDuplicates).toHaveBeenCalledWith( + preservedLocalOps, + 'local', + ); + expect(operationApplierSpy.applyOperations).toHaveBeenCalledWith( + preservedLocalOps, + jasmine.objectContaining({ isLocalHydration: false }), + ); + expect(opLogStoreSpy.setVectorClock).toHaveBeenCalledWith({ + remote: 4, + local: 3, + }); + expect(opLogStoreSpy.clearRawRebuildIncomplete).toHaveBeenCalled(); + }); + it('should offer to restore the previous data after replacing (#8107)', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index ecc341401b..0aff4ddf64 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -60,6 +60,7 @@ import { LocalOnlySyncSettings, } from '../../features/config/local-only-sync-settings.util'; import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; +import { OperationApplierService } from '../apply/operation-applier.service'; /** * Orchestrates synchronization of the Operation Log with remote storage. @@ -149,6 +150,7 @@ export class OperationLogSyncService { private syncLocalStateService = inject(SyncLocalStateService); private syncImportConflictCoordinator = inject(SyncImportConflictCoordinatorService); private providerManager = inject(SyncProviderManager); + private operationApplier = inject(OperationApplierService); /** * Checks if this client is "wholly fresh" - meaning it has never synced before @@ -1333,6 +1335,7 @@ export class OperationLogSyncService { let capturedBackupSavedAt: number | undefined; let replacementCommitted = false; let backupSavedAt: number | undefined; + let preservedLocalOps: Operation[] = []; try { // flushThenRunExclusive drains the capture pipeline BEFORE acquiring the // op-log lock (flushPendingWrites re-acquires the same non-reentrant lock, @@ -1345,6 +1348,14 @@ export class OperationLogSyncService { // Keep the first attempt's pre-replace backup (see option JSDoc). savedAt = (await this.opLogStore.loadImportBackup())?.savedAt; capturedBackupSavedAt = savedAt; + const marker = await this.opLogStore.loadRawRebuildIncomplete(); + const liveLocalOps = (await this.opLogStore.getUnsynced()).map( + (entry) => entry.op, + ); + preservedLocalOps = this._mergeOperationsById( + marker?.preservedLocalOps ?? [], + liveLocalOps, + ); } else { try { savedAt = await this.backupService.captureImportBackup(); @@ -1375,6 +1386,7 @@ export class OperationLogSyncService { ), archiveYoung, archiveOld, + preservedLocalOps, }); replacementCommitted = true; @@ -1417,6 +1429,8 @@ export class OperationLogSyncService { ); } + await this._restorePreservedLocalOps(preservedLocalOps); + // Update lastServerSeq after hydration if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); @@ -1467,6 +1481,8 @@ export class OperationLogSyncService { ); } + await this._restorePreservedLocalOps(preservedLocalOps); + // Update lastServerSeq if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); @@ -1494,6 +1510,57 @@ export class OperationLogSyncService { } } + private _mergeOperationsById(...operationGroups: Operation[][]): Operation[] { + const merged: Operation[] = []; + const seenIds = new Set(); + for (const operations of operationGroups) { + for (const op of operations) { + if (!seenIds.has(op.id)) { + seenIds.add(op.id); + merged.push(op); + } + } + } + return merged; + } + + /** + * Replays edits captured after an interrupted rebuild on top of the complete + * authoritative history. They stay local/unsynced so the next upload carries + * the user's post-crash intent to the server. + */ + private async _restorePreservedLocalOps(operations: Operation[]): Promise { + if (operations.length === 0) { + return; + } + + const { writtenOps } = await this.opLogStore.appendBatchSkipDuplicates( + operations, + 'local', + ); + if (writtenOps.length === 0) { + return; + } + + const applyResult = await this.operationApplier.applyOperations(writtenOps, { + // The authoritative replacement also overwrote archive IndexedDB stores, + // so archive-affecting post-crash edits must replay their side effects. + // The entries themselves remain source=local and unsynced in the op-log. + isLocalHydration: false, + }); + if (applyResult.failedOp || applyResult.appliedOps.length !== writtenOps.length) { + throw new Error( + 'USE_REMOTE incomplete: post-crash local operations could not be restored.', + ); + } + + let restoredClock = (await this.opLogStore.getVectorClock()) ?? {}; + for (const op of writtenOps) { + restoredClock = mergeVectorClocks(restoredClock, op.vectorClock); + } + await this.opLogStore.setVectorClock(restoredClock); + } + private _preflightRemoteOperations(remoteOps: Operation[]): Operation[] { for (const op of remoteOps) { let version: number; From 3480d749d75539e71e166152d92ef0755021c780 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:29:23 +0200 Subject: [PATCH 20/47] fix(sync): block on incomplete local persistence --- .../capture/operation-log.effects.spec.ts | 23 +++++-- .../op-log/capture/operation-log.effects.ts | 39 +++++++----- ...ration-log-lock-reentry.regression.spec.ts | 18 +++--- .../sync/operation-log-sync.service.spec.ts | 62 ++++++++++++++++++- .../op-log/sync/operation-log-sync.service.ts | 30 ++++++++- .../process-deferred-actions-flush.util.ts | 4 +- src/assets/i18n/en.json | 2 +- 7 files changed, 145 insertions(+), 33 deletions(-) diff --git a/src/app/op-log/capture/operation-log.effects.spec.ts b/src/app/op-log/capture/operation-log.effects.spec.ts index cba00d4ac5..b28d9c05c4 100644 --- a/src/app/op-log/capture/operation-log.effects.spec.ts +++ b/src/app/op-log/capture/operation-log.effects.spec.ts @@ -805,7 +805,7 @@ describe('OperationLogEffects', () => { }); it('should do nothing when no deferred actions are buffered', async () => { - await effects.processDeferredActions(); + await expectAsync(effects.processDeferredActions()).toBeResolved(); expect(mockOpLogStore.appendWithVectorClockUpdate).not.toHaveBeenCalled(); }); @@ -893,12 +893,18 @@ describe('OperationLogEffects', () => { new Error('transient failure'), ); - await effects.processDeferredActions(); + await expectAsync(effects.processDeferredActions()).toBeRejected(); // Only the failed action was attempted (3 retries); the successor was // never written out of order and both remain buffered. expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(3); expect(getDeferredActions()).toEqual([failedAction, successorAction]); + expect(mockSnackService.open).toHaveBeenCalledWith( + jasmine.objectContaining({ + msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED, + actionStr: T.G.DISMISS, + }), + ); mockOpLogStore.appendWithVectorClockUpdate.calls.reset(); mockOpLogStore.appendWithVectorClockUpdate.and.resolveTo(2); @@ -931,7 +937,10 @@ describe('OperationLogEffects', () => { ).toBe(ActionType.TASK_SHARED_UPDATE); expect(getDeferredActions()).toEqual([]); expect(mockSnackService.open).toHaveBeenCalledWith( - jasmine.objectContaining({ msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED }), + jasmine.objectContaining({ + msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED, + actionStr: T.G.DISMISS, + }), ); }); @@ -1022,7 +1031,9 @@ describe('OperationLogEffects', () => { new DOMException('Quota exceeded', 'QuotaExceededError'), ); - await effects.processDeferredActions({ callerHoldsOperationLogLock: true }); + await expectAsync( + effects.processDeferredActions({ callerHoldsOperationLogLock: true }), + ).toBeRejected(); expect(mockCompactionService.emergencyCompact).not.toHaveBeenCalled(); expect(mockLockService.request).not.toHaveBeenCalledWith( @@ -1055,7 +1066,9 @@ describe('OperationLogEffects', () => { new DOMException('Quota exceeded', 'QuotaExceededError'), ); - await effects.processDeferredActions({ callerHoldsOperationLogLock: true }); + await expectAsync( + effects.processDeferredActions({ callerHoldsOperationLogLock: true }), + ).toBeRejected(); // 1. The bail path actually ran (proves handleQuotaExceeded was invoked // AND took the caller-holds-lock branch — not some other code path). diff --git a/src/app/op-log/capture/operation-log.effects.ts b/src/app/op-log/capture/operation-log.effects.ts index 68c4b70165..76edf6a342 100644 --- a/src/app/op-log/capture/operation-log.effects.ts +++ b/src/app/op-log/capture/operation-log.effects.ts @@ -305,6 +305,11 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // (recordCriticalErrorTime) — that is fed by GlobalErrorHandler and the // validateState seam only; snackbar-surfaced conditions are out of scope // by design. See util/critical-error-signal.ts. + if (isDeferredWrite) { + throw new PermanentDeferredWriteError( + `Deferred action ${action.type} has an invalid payload.`, + ); + } this.snackService.open({ type: 'ERROR', msg: T.F.SYNC.S.INVALID_OPERATION_PAYLOAD, @@ -313,11 +318,6 @@ export class OperationLogEffects implements DeferredLocalActionsPort { window.location.reload(); }, }); - if (isDeferredWrite) { - throw new PermanentDeferredWriteError( - `Deferred action ${action.type} has an invalid payload.`, - ); - } return; // Skip persisting invalid operation } @@ -409,7 +409,9 @@ export class OperationLogEffects implements DeferredLocalActionsPort { // #8306: the effect path catches this throw in writeOperationFromEffect // — the shared persistOperation$ stream must NOT be torn down by one // failed write — while still showing the sticky snackbar below. - this.notifyUserAndTriggerRollback(); + if (!isDeferredWrite) { + this.notifyUserAndTriggerRollback(); + } throw e; } if (this.isQuotaExceededError(e)) { @@ -418,13 +420,17 @@ export class OperationLogEffects implements DeferredLocalActionsPort { OpLog.err( 'OperationLogEffects: Quota exceeded during retry - aborting to prevent loop', ); - this.notifyUserAndTriggerRollback(); + if (!isDeferredWrite) { + this.notifyUserAndTriggerRollback(); + } } else { await this.handleQuotaExceeded(action, isDeferredWrite, options); return; } } else { - this.notifyUserAndTriggerRollback(); + if (!isDeferredWrite) { + this.notifyUserAndTriggerRollback(); + } } throw e; } @@ -687,6 +693,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { const MAX_RETRIES = 3; const BASE_DELAY_MS = 100; let failedCount = 0; + let transientFailure: unknown; for (const action of deferredActions) { let lastError: unknown; @@ -728,8 +735,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { if (lastError instanceof PermanentDeferredWriteError) { // Terminal: the action can never be persisted (invalid identifiers or // payload). Abandon it — keeping it queued would retry it with backoff - // and re-raise the failure snack on every future sync window while the - // in-memory buffer's "durability" ends at the advised reload anyway. + // and re-raise the failure snack on every future sync window. acknowledgeDeferredAction(action); OpLog.err('OperationLogEffects: Abandoning permanently invalid deferred action', { actionType: action.type, @@ -749,6 +755,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { errorName: (lastError as Error | undefined)?.name, }, ); + transientFailure = lastError; break; } @@ -757,14 +764,18 @@ export class OperationLogEffects implements DeferredLocalActionsPort { this.snackService.open({ type: 'ERROR', msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED, - actionStr: T.PS.RELOAD, - actionFn: (): void => { - window.location.reload(); - }, + actionStr: T.G.DISMISS, config: { duration: 0, // Sticky - don't auto-dismiss critical errors }, }); } + + if (transientFailure !== undefined) { + throw new Error( + 'Deferred local actions remain unpersisted after retry exhaustion.', + { cause: transientFailure }, + ); + } } } diff --git a/src/app/op-log/sync/operation-log-lock-reentry.regression.spec.ts b/src/app/op-log/sync/operation-log-lock-reentry.regression.spec.ts index 1fe578bbba..e7d0dd52c9 100644 --- a/src/app/op-log/sync/operation-log-lock-reentry.regression.spec.ts +++ b/src/app/op-log/sync/operation-log-lock-reentry.regression.spec.ts @@ -218,14 +218,16 @@ describe('regression #7700: operation-log lock reentry', () => { // Hold the lock comfortably longer than the full retry budget: // 3 attempts × ~1000ms timeout + 100ms + 200ms backoffs ≈ 3300ms. // 10000ms is generous. - await lockService.request( - LOCK_NAMES.OPERATION_LOG, - async () => { - // Caller forgot the flag — same as a buggy refactor. - await effects.processDeferredActions(); - }, - 10000, - ); + await expectAsync( + lockService.request( + LOCK_NAMES.OPERATION_LOG, + async () => { + // Caller forgot the flag — same as a buggy refactor. + await effects.processDeferredActions(); + }, + 10000, + ), + ).toBeRejected(); // Action was NOT written — no silent persistence. expect(opLogStoreSpy.appendWithVectorClockUpdate).not.toHaveBeenCalled(); diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 1189c3f3a4..ef3a282ebe 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -6,6 +6,7 @@ import { SnackService } from '../../core/snack/snack.service'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { VectorClockService } from './vector-clock.service'; import { OperationApplierService } from '../apply/operation-applier.service'; +import { OperationLogEffects } from '../capture/operation-log.effects'; import { ConflictResolutionService } from './conflict-resolution.service'; import { ValidateStateService } from '../validation/validate-state.service'; import { SyncSessionValidationService } from './sync-session-validation.service'; @@ -58,6 +59,7 @@ describe('OperationLogSyncService', () => { let validateStateServiceSpy: jasmine.SpyObj; let lockServiceSpy: jasmine.SpyObj; let operationApplierSpy: jasmine.SpyObj; + let operationLogEffectsSpy: jasmine.SpyObj; beforeEach(() => { snackServiceSpy = jasmine.createSpyObj('SnackService', [ @@ -67,6 +69,8 @@ describe('OperationLogSyncService', () => { snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); opLogStoreSpy = jasmine.createSpyObj('OperationLogStoreService', [ 'getUnsynced', + 'getPendingRemoteOps', + 'getFailedRemoteOps', 'loadStateCache', 'getLastSeq', 'getOpById', @@ -85,6 +89,8 @@ describe('OperationLogSyncService', () => { ]); opLogStoreSpy.hasSyncedOps.and.resolveTo(true); opLogStoreSpy.getUnsynced.and.resolveTo([]); + opLogStoreSpy.getPendingRemoteOps.and.resolveTo([]); + opLogStoreSpy.getFailedRemoteOps.and.resolveTo([]); opLogStoreSpy.markSynced.and.resolveTo(); opLogStoreSpy.setVectorClock.and.resolveTo(); opLogStoreSpy.clearFullStateOps.and.resolveTo(); @@ -188,6 +194,10 @@ describe('OperationLogSyncService', () => { 'applyOperations', ]); operationApplierSpy.applyOperations.and.resolveTo({ appliedOps: [] }); + operationLogEffectsSpy = jasmine.createSpyObj('OperationLogEffects', [ + 'processDeferredActions', + ]); + operationLogEffectsSpy.processDeferredActions.and.resolveTo(); TestBed.configureTestingModule({ providers: [ @@ -209,6 +219,7 @@ describe('OperationLogSyncService', () => { provide: OperationApplierService, useValue: operationApplierSpy, }, + { provide: OperationLogEffects, useValue: operationLogEffectsSpy }, { provide: ConflictResolutionService, useValue: jasmine.createSpyObj('ConflictResolutionService', [ @@ -315,6 +326,41 @@ describe('OperationLogSyncService', () => { }); describe('uploadPendingOps', () => { + it('should drain deferred local actions before selecting pending uploads', async () => { + const callOrder: string[] = []; + writeFlushServiceSpy.flushPendingWrites.and.callFake(async () => { + callOrder.push('flush'); + }); + operationLogEffectsSpy.processDeferredActions.and.callFake(async () => { + callOrder.push('deferred'); + }); + uploadServiceSpy.uploadPendingOps.and.callFake(async () => { + callOrder.push('upload'); + return { + uploadedCount: 0, + piggybackedOps: [], + rejectedCount: 0, + rejectedOps: [], + }; + }); + + await service.uploadPendingOps({} as OperationSyncCapable); + + expect(callOrder).toEqual(['flush', 'deferred', 'flush', 'upload']); + }); + + it('should block upload while a remote operation is incompletely applied', async () => { + opLogStoreSpy.getPendingRemoteOps.and.resolveTo([ + { applicationStatus: 'pending' } as OperationLogEntry, + ]); + + await expectAsync( + service.uploadPendingOps({} as OperationSyncCapable), + ).toBeRejected(); + + expect(uploadServiceSpy.uploadPendingOps).not.toHaveBeenCalled(); + }); + it('should return localWinOpsCreated: 0 when no piggybacked ops', async () => { opLogStoreSpy.getUnsynced.and.returnValue(Promise.resolve([])); uploadServiceSpy.uploadPendingOps.and.returnValue( @@ -908,6 +954,18 @@ describe('OperationLogSyncService', () => { }); describe('downloadRemoteOps', () => { + it('should block download while a prior remote operation is incompletely applied', async () => { + opLogStoreSpy.getFailedRemoteOps.and.resolveTo([ + { applicationStatus: 'archive_pending' } as OperationLogEntry, + ]); + + await expectAsync( + service.downloadRemoteOps({} as OperationSyncCapable), + ).toBeRejected(); + + expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); + }); + it('should redo the raw rebuild when a prior USE_REMOTE replay was interrupted', async () => { // The normal download path excludes this client's own ops server-side, // so resuming an interrupted rebuild through it would silently lose them. @@ -3962,7 +4020,7 @@ describe('OperationLogSyncService', () => { const result = await service.downloadRemoteOps(mockProvider); - expect(events.slice(0, 2)).toEqual(['flush', 'getUnsynced']); + expect(events.slice(0, 4)).toEqual(['flush', 'flush', 'flush', 'getUnsynced']); expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith([ incomingSyncImport, ]); @@ -4296,7 +4354,7 @@ describe('OperationLogSyncService', () => { const result = await service.uploadPendingOps(mockProvider); - expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalledTimes(2); + expect(writeFlushServiceSpy.flushPendingWrites).toHaveBeenCalledTimes(3); expect(opLogStoreSpy.getUnsynced).toHaveBeenCalled(); expect(result.kind).toBe('completed'); }); diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 0aff4ddf64..f6d453f425 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -1,4 +1,4 @@ -import { inject, Injectable } from '@angular/core'; +import { inject, Injectable, Injector } from '@angular/core'; import { Store } from '@ngrx/store'; import { planSnapshotHydration } from '@sp/sync-core'; import { @@ -61,6 +61,7 @@ import { } from '../../features/config/local-only-sync-settings.util'; import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; import { OperationApplierService } from '../apply/operation-applier.service'; +import { processDeferredActions } from './process-deferred-actions-flush.util'; /** * Orchestrates synchronization of the Operation Log with remote storage. @@ -151,6 +152,7 @@ export class OperationLogSyncService { private syncImportConflictCoordinator = inject(SyncImportConflictCoordinatorService); private providerManager = inject(SyncProviderManager); private operationApplier = inject(OperationApplierService); + private injector = inject(Injector); /** * Checks if this client is "wholly fresh" - meaning it has never synced before @@ -206,7 +208,8 @@ export class OperationLogSyncService { // The effect that writes operations uses concatMap for sequential processing, // but if sync is triggered before all operations are written to IndexedDB, // we would upload an incomplete set. This flush waits for all queued writes. - await this.writeFlushService.flushPendingWrites(); + await this._flushLocalWritesIncludingDeferredActions(); + await this._assertNoIncompleteRemoteOperations(); // Capture never-synced status before the upload runs. The orchestrator passes a // value captured even earlier (pre-download, since download persists synced ops); @@ -470,6 +473,9 @@ export class OperationLogSyncService { return { kind: 'snapshot_hydrated' }; } + await this._flushLocalWritesIncludingDeferredActions(); + await this._assertNoIncompleteRemoteOperations(); + const result = await this.downloadService.downloadRemoteOps(syncProvider, options); // FIX #6571: Check download success before processing results. @@ -1524,6 +1530,26 @@ export class OperationLogSyncService { return merged; } + private async _flushLocalWritesIncludingDeferredActions(): Promise { + await this.writeFlushService.flushPendingWrites(); + await processDeferredActions(this.injector, false); + // Deferred writes are awaited directly, but this second barrier also + // catches ordinary actions dispatched while their drain was running. + await this.writeFlushService.flushPendingWrites(); + } + + private async _assertNoIncompleteRemoteOperations(): Promise { + const [pendingRemoteOps, failedRemoteOps] = await Promise.all([ + this.opLogStore.getPendingRemoteOps(), + this.opLogStore.getFailedRemoteOps(), + ]); + if (pendingRemoteOps.length > 0 || failedRemoteOps.length > 0) { + throw new Error( + 'Sync blocked because previously downloaded operations are not fully applied. Restart the app to retry recovery.', + ); + } + } + /** * Replays edits captured after an interrupted rebuild on top of the complete * authoritative history. They stay local/unsynced so the next upload carries diff --git a/src/app/op-log/sync/process-deferred-actions-flush.util.ts b/src/app/op-log/sync/process-deferred-actions-flush.util.ts index 326b8cba8c..d1b763d5f9 100644 --- a/src/app/op-log/sync/process-deferred-actions-flush.util.ts +++ b/src/app/op-log/sync/process-deferred-actions-flush.util.ts @@ -19,7 +19,7 @@ import { OperationLogEffects } from '../capture/operation-log.effects'; * the sp_op_log lock; the flush then runs inline rather * than re-acquiring (non-reentrant lock would deadlock). */ -export const processDeferredActionsAfterRemoteApply = async ( +export const processDeferredActions = async ( injector: Injector, callerHoldsOperationLogLock: boolean, ): Promise => { @@ -27,3 +27,5 @@ export const processDeferredActionsAfterRemoteApply = async ( callerHoldsOperationLogLock, }); }; + +export const processDeferredActionsAfterRemoteApply = processDeferredActions; diff --git a/src/assets/i18n/en.json b/src/assets/i18n/en.json index c51dd49868..6ced86a079 100644 --- a/src/assets/i18n/en.json +++ b/src/assets/i18n/en.json @@ -1549,7 +1549,7 @@ "CONFLICT_RESOLUTION_FAILED": "Sync conflict resolution failed. Please reload.", "DATA_REPAIRED": "Sync cleanup: {{count}} reference(s) resolved", "DECRYPTION_FAILED": "Failed to decrypt synced data. Please check your encryption password.", - "DEFERRED_ACTION_FAILED": "Some changes made during sync could not be saved. Please reload to recover.", + "DEFERRED_ACTION_FAILED": "Some changes made during sync could not be saved yet. Sync will retry automatically; keep the app open.", "DISABLE_ENCRYPTION_FAILED": "Failed to disable encryption: {{message}}", "ENABLE_ENCRYPTION_FAILED": "Failed to enable encryption: {{message}}", "ENCRYPTION_DISABLED_ON_OTHER_DEVICE": "Encryption was disabled on another device. Your sync settings have been updated.", From 0939f5af69a859884f5c7dbf7ddbc3d9ceaa126f Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:35:37 +0200 Subject: [PATCH 21/47] fix(sync): enforce upload completion contracts --- .../src/supersync-http-contract.ts | 10 +++- .../tests/supersync-http-contract.spec.ts | 15 +++-- .../tests/sync-compressed-body.routes.spec.ts | 43 ++++++++++++++ packages/sync-core/src/remote-apply.ts | 56 ++++++++++++++++--- packages/sync-core/tests/remote-apply.spec.ts | 51 +++++++++++++++++ .../sync/immediate-upload.service.spec.ts | 43 ++++++++++++++ .../op-log/sync/immediate-upload.service.ts | 41 ++++++++++++-- 7 files changed, 239 insertions(+), 20 deletions(-) diff --git a/packages/shared-schema/src/supersync-http-contract.ts b/packages/shared-schema/src/supersync-http-contract.ts index b128400cbf..f510c270f7 100644 --- a/packages/shared-schema/src/supersync-http-contract.ts +++ b/packages/shared-schema/src/supersync-http-contract.ts @@ -74,8 +74,16 @@ export const SuperSyncOperationSchema = z.object({ syncImportReason: z.enum(SUPER_SYNC_IMPORT_REASONS).optional(), }); +// Upload requests are envelopes for independently validated operations. Keep +// the transport shape numeric, but let the server return a per-op +// INVALID_SCHEMA_VERSION result so one malformed op cannot reject and stall +// every valid sibling in the batch. Download/response schemas remain strict. +const SuperSyncUploadOperationSchema = SuperSyncOperationSchema.extend({ + schemaVersion: z.number(), +}); + export const SuperSyncUploadOpsRequestSchema = z.object({ - ops: z.array(SuperSyncOperationSchema).min(1).max(SUPER_SYNC_MAX_OPS_PER_UPLOAD), + ops: z.array(SuperSyncUploadOperationSchema).min(1).max(SUPER_SYNC_MAX_OPS_PER_UPLOAD), clientId: SuperSyncClientIdSchema, lastKnownServerSeq: z.number().optional(), requestId: SuperSyncRequestIdSchema.optional(), diff --git a/packages/shared-schema/tests/supersync-http-contract.spec.ts b/packages/shared-schema/tests/supersync-http-contract.spec.ts index c0f725540b..10e8d30f4c 100644 --- a/packages/shared-schema/tests/supersync-http-contract.spec.ts +++ b/packages/shared-schema/tests/supersync-http-contract.spec.ts @@ -4,6 +4,7 @@ import { SUPER_SYNC_MAX_OPS_PER_UPLOAD, SuperSyncDownloadOpsQuerySchema, SuperSyncDownloadOpsResponseSchema, + SuperSyncOperationSchema, SuperSyncUploadOpsRequestSchema, SuperSyncUploadSnapshotRequestSchema, } from '../src/supersync-http-contract'; @@ -73,12 +74,18 @@ describe('SuperSync HTTP contract schemas', () => { }); it.each([0, 1.5, -1, 101])( - 'rejects malformed operation schema version %s', + 'passes operation schema version %s through the upload transport schema', (schemaVersion) => { + const parsed = SuperSyncUploadOpsRequestSchema.parse({ + ops: [{ ...createValidOperation(), schemaVersion }], + clientId: 'client_1', + }); + + expect(parsed.ops[0].schemaVersion).toBe(schemaVersion); expect(() => - SuperSyncUploadOpsRequestSchema.parse({ - ops: [{ ...createValidOperation(), schemaVersion }], - clientId: 'client_1', + SuperSyncOperationSchema.parse({ + ...createValidOperation(), + schemaVersion, }), ).toThrow(); }, diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index 0acbb475f0..d9cd53a1c7 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -212,6 +212,49 @@ describe('Sync compressed body routes', () => { expect(quotaCall[1]).toBeLessThan(payloadSize); }); + it('should pass mixed schema versions to per-operation validation', async () => { + const clientId = 'mixed-schema-client'; + const validOp = createOp(clientId); + const invalidOp = { + ...createOp(clientId), + id: 'invalid-schema-op', + schemaVersion: 101, + }; + mocks.syncService.uploadOps.mockResolvedValueOnce([ + { opId: validOp.id, accepted: true, serverSeq: 1 }, + { + opId: invalidOp.id, + accepted: false, + error: 'Invalid schema version: 101', + errorCode: SYNC_ERROR_CODES.INVALID_SCHEMA_VERSION, + }, + ]); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/ops', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + }, + payload: { ops: [validOp, invalidOp], clientId }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json().results).toEqual([ + expect.objectContaining({ opId: validOp.id, accepted: true }), + expect.objectContaining({ + opId: invalidOp.id, + accepted: false, + errorCode: SYNC_ERROR_CODES.INVALID_SCHEMA_VERSION, + }), + ]); + expect(mocks.syncService.uploadOps).toHaveBeenCalledWith(1, clientId, [ + validOp, + invalidOp, + ]); + }); + it('should subtract exact already-stored duplicate ops from the ops quota gate', async () => { const clientId = 'known-duplicate-quota-client'; const duplicateOp = { diff --git a/packages/sync-core/src/remote-apply.ts b/packages/sync-core/src/remote-apply.ts index 9676c3a311..70544f8d81 100644 --- a/packages/sync-core/src/remote-apply.ts +++ b/packages/sync-core/src/remote-apply.ts @@ -113,20 +113,58 @@ export const applyRemoteOperations = async < } }); + let reducerCommitCallbackCount = 0; + let reducerCommitPromise: Promise | undefined; const applyResult = await applier.applyOperations(appendResult.writtenOps, { - onReducersCommitted: async (reducerCommittedOps) => { - const reducerCommittedSeqs = reducerCommittedOps - .map((op) => opIdToSeq.get(op.id)) - .filter((seq): seq is number => seq !== undefined); - if (reducerCommittedSeqs.length !== reducerCommittedOps.length) { - throw new Error( - 'applyRemoteOperations: reducer commit contained an operation outside the appended batch.', + onReducersCommitted: (reducerCommittedOps) => { + reducerCommitCallbackCount++; + if (reducerCommitCallbackCount !== 1) { + reducerCommitPromise = Promise.reject( + new Error( + 'applyRemoteOperations: reducer-commit callback must be invoked exactly once.', + ), ); + return reducerCommitPromise; } - await store.markArchivePending(reducerCommittedSeqs); - await store.mergeRemoteOpClocks(reducerCommittedOps); + + const isEntireAppendedBatch = + reducerCommittedOps.length === appendResult.writtenOps.length && + reducerCommittedOps.every( + (op, index) => op.id === appendResult.writtenOps[index]?.id, + ); + if (!isEntireAppendedBatch) { + reducerCommitPromise = Promise.reject( + new Error( + 'applyRemoteOperations: reducer-commit callback must contain the entire appended batch in order.', + ), + ); + return reducerCommitPromise; + } + + reducerCommitPromise = (async () => { + const reducerCommittedSeqs = reducerCommittedOps + .map((op) => opIdToSeq.get(op.id)) + .filter((seq): seq is number => seq !== undefined); + if (reducerCommittedSeqs.length !== reducerCommittedOps.length) { + throw new Error( + 'applyRemoteOperations: reducer commit contained an operation outside the appended batch.', + ); + } + await store.markArchivePending(reducerCommittedSeqs); + await store.mergeRemoteOpClocks(reducerCommittedOps); + })(); + return reducerCommitPromise; }, }); + if (reducerCommitCallbackCount !== 1 || reducerCommitPromise === undefined) { + throw new Error( + 'applyRemoteOperations: applier did not invoke the reducer-commit callback.', + ); + } + // Also await host bookkeeping when an applier invoked the callback without + // awaiting its returned promise. Pending ops must never be marked applied + // before the reducer/archive checkpoint is durable. + await reducerCommitPromise; if (applyResult.failedOp && !opIdToSeq.has(applyResult.failedOp.op.id)) { throw new Error( `applyRemoteOperations: applier reported unknown failed op ${applyResult.failedOp.op.id}.`, diff --git a/packages/sync-core/tests/remote-apply.spec.ts b/packages/sync-core/tests/remote-apply.spec.ts index 3c71023caa..bdfedb28db 100644 --- a/packages/sync-core/tests/remote-apply.spec.ts +++ b/packages/sync-core/tests/remote-apply.spec.ts @@ -187,6 +187,57 @@ describe('applyRemoteOperations', () => { ); expect(store.markFailed).not.toHaveBeenCalled(); }); + + it('fails closed when the applier ignores the reducer-commit callback', async () => { + const op = createOperation('op-1'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + const applier: OperationApplyPort> = { + applyOperations: vi.fn().mockResolvedValue({ appliedOps: [op] }), + }; + + await expect(applyRemoteOperations({ ops: [op], store, applier })).rejects.toThrow( + 'reducer-commit callback', + ); + expect(store.markApplied).not.toHaveBeenCalled(); + }); + + it('fails closed when the reducer-commit callback omits an appended op', async () => { + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const store = createStore({ + seqs: [1, 2], + writtenOps: [op1, op2], + skippedCount: 0, + }); + const applier: OperationApplyPort> = { + applyOperations: vi.fn(async (_ops, options) => { + await options?.onReducersCommitted?.([op1]); + return { appliedOps: [op1, op2] }; + }), + }; + + await expect( + applyRemoteOperations({ ops: [op1, op2], store, applier }), + ).rejects.toThrow('entire appended batch'); + expect(store.markApplied).not.toHaveBeenCalled(); + }); + + it('fails closed when the reducer-commit callback is invoked twice', async () => { + const op = createOperation('op-1'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + const applier: OperationApplyPort> = { + applyOperations: vi.fn(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }), + }; + + await expect(applyRemoteOperations({ ops: [op], store, applier })).rejects.toThrow( + 'exactly once', + ); + expect(store.markApplied).not.toHaveBeenCalled(); + }); }); const jasmineLikeObjectContainingFunction = (): object => diff --git a/src/app/op-log/sync/immediate-upload.service.spec.ts b/src/app/op-log/sync/immediate-upload.service.spec.ts index 6d18e1ce4a..15a4880743 100644 --- a/src/app/op-log/sync/immediate-upload.service.spec.ts +++ b/src/app/op-log/sync/immediate-upload.service.spec.ts @@ -205,6 +205,49 @@ describe('ImmediateUploadService', () => { // Piggybacked ops are processed internally, no checkmark shown expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalled(); })); + + it('should report ERROR when the local-win follow-up is blocked', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.returnValues( + Promise.resolve(completedResult({ uploadedCount: 1, localWinOpsCreated: 1 })), + Promise.resolve({ kind: 'blocked_incompatible' }), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalledWith('IN_SYNC'); + })); + + it('should not show a checkmark when the local-win follow-up is cancelled', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.returnValues( + Promise.resolve(completedResult({ uploadedCount: 1, localWinOpsCreated: 1 })), + Promise.resolve({ kind: 'cancelled' }), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalled(); + })); + + it('should defer the checkmark when the local-win follow-up receives piggybacked ops', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.returnValues( + Promise.resolve(completedResult({ uploadedCount: 1, localWinOpsCreated: 1 })), + Promise.resolve(completedResult({ uploadedCount: 1, piggybackedOpsCount: 1 })), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalled(); + })); }); // #8309: the immediate-upload side channel must not interleave with another diff --git a/src/app/op-log/sync/immediate-upload.service.ts b/src/app/op-log/sync/immediate-upload.service.ts index 22c0fc0c9b..5f1c4365ca 100644 --- a/src/app/op-log/sync/immediate-upload.service.ts +++ b/src/app/op-log/sync/immediate-upload.service.ts @@ -249,11 +249,31 @@ export class ImmediateUploadService implements OnDestroy { // If LWW local-wins created new update ops from piggybacked ops, // do a follow-up upload to push them to the server immediately + let finalResult = result; + let totalUploadedCount = result.uploadedCount; + let hasPermanentRejection = result.permanentRejectionCount > 0; if (result.localWinOpsCreated > 0) { OpLog.verbose( `ImmediateUploadService: LWW created ${result.localWinOpsCreated} local-win op(s), re-uploading`, ); - await this._syncService.uploadPendingOps(syncCapableProvider); + const followUpResult = + await this._syncService.uploadPendingOps(syncCapableProvider); + if (followUpResult.kind === 'blocked_incompatible') { + OpLog.warn( + 'ImmediateUploadService: Local-win follow-up blocked by an incompatible operation', + ); + this._providerManager.setSyncStatus('ERROR'); + return; + } + if ( + followUpResult.kind === 'cancelled' || + followUpResult.kind === 'blocked_fresh_client' + ) { + return; + } + finalResult = followUpResult; + totalUploadedCount += followUpResult.uploadedCount; + hasPermanentRejection ||= followUpResult.permanentRejectionCount > 0; } // Read the validation latch BEFORE any IN_SYNC / deferred-checkmark @@ -270,20 +290,29 @@ export class ImmediateUploadService implements OnDestroy { // Don't show checkmark when piggybacked ops exist - there may be more // remote ops pending. Let normal sync cycle confirm full sync state. - if (result.piggybackedOpsCount > 0) { + if (hasPermanentRejection) { + this._providerManager.setSyncStatus('ERROR'); + return; + } + + if ( + finalResult.piggybackedOpsCount > 0 || + finalResult.hasMorePiggyback || + finalResult.localWinOpsCreated > 0 + ) { OpLog.verbose( - `ImmediateUploadService: Uploaded ${result.uploadedCount} ops, ` + - `processed ${result.piggybackedOpsCount} piggybacked (checkmark deferred)`, + `ImmediateUploadService: Uploaded ${totalUploadedCount} ops, ` + + `processed ${finalResult.piggybackedOpsCount} piggybacked (checkmark deferred)`, ); return; } // Show checkmark ONLY when server confirms no pending remote ops // (empty piggybackedOps means we're confirmed in sync) - if (result.uploadedCount > 0 || result.localWinOpsCreated > 0) { + if (totalUploadedCount > 0) { this._providerManager.setSyncStatus('IN_SYNC'); OpLog.verbose( - `ImmediateUploadService: Uploaded ${result.uploadedCount} ops, confirmed in sync`, + `ImmediateUploadService: Uploaded ${totalUploadedCount} ops, confirmed in sync`, ); } } catch (e) { From 5d568dcf8cd064ad34eed04c42f60d3c688939bb Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:37:55 +0200 Subject: [PATCH 22/47] test(sync): verify non-task conflict convergence --- .../sync/supersync-conflict-gate-non-task-work.spec.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts index e2d8dd0f8c..d153ccdc16 100644 --- a/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts +++ b/e2e/tests/sync/supersync-conflict-gate-non-task-work.spec.ts @@ -217,7 +217,14 @@ test.describe('@supersync incoming SYNC_IMPORT preserves non-task local work', ( await clientB.sync.syncAndWait(); await expectClickCounterValue(clientB, counterTitle, 2); - console.log('[Gate] ✓ Pending counter value survived resolution and re-sync'); + + // Prove convergence, not just Client B's local durability: Client A must + // download the state B force-uploaded when it chose "Use My Data". + await clientA.sync.syncAndWait(); + await clientA.page.goto('/#/tag/TODAY/tasks'); + await clientA.page.waitForURL(/(active\/tasks|tag\/TODAY\/tasks)/); + await expectClickCounterValue(clientA, counterTitle, 2); + console.log('[Gate] ✓ Pending counter value converged on both clients'); } finally { if (clientA) await closeClient(clientA); if (clientB) await closeClient(clientB); From 7866e16b56d63327ed6ab35cc5bdf9494b73f7b0 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 19:38:03 +0200 Subject: [PATCH 23/47] docs(sync): document recovery checkpoints --- .../diagrams/06-archive-operations.md | 2 +- docs/sync-and-op-log/operation-log-architecture.md | 12 ++++++++++++ .../supersync-scenarios-flowchart.md | 2 +- docs/sync-and-op-log/supersync-scenarios.md | 14 +++++++------- docs/wiki/3.06-User-Data.md | 2 +- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/sync-and-op-log/diagrams/06-archive-operations.md b/docs/sync-and-op-log/diagrams/06-archive-operations.md index 96151a3099..a65944f37d 100644 --- a/docs/sync-and-op-log/diagrams/06-archive-operations.md +++ b/docs/sync-and-op-log/diagrams/06-archive-operations.md @@ -113,7 +113,7 @@ flowchart TD ## ArchiveOperationHandler Integration -The `OperationApplierService` applies operations in the order they arrive from the sync server, which preserves causal ordering (each client uploads its ops in causal order, and the server assigns sequence numbers in upload order). It converts each op to an action, applies the batch via a single bulk dispatch, then runs archive side effects. If an operation fails to apply, the applier returns it as a `failedOp`; the caller surfaces a partial-apply failure and re-validates state, and the op is retried on the next hydration. +The `OperationApplierService` applies operations in the order they arrive from the sync server, which preserves causal ordering (each client uploads its ops in causal order, and the server assigns sequence numbers in upload order). Incoming rows are first stored as `pending`. Immediately after the single bulk reducer dispatch, the reducer-commit callback checkpoints the entire batch as `archive_pending` and merges its vector clocks; only then do archive side effects run. Successful rows become `applied`. The attempted row becomes `failed` if an archive side effect throws, while unattempted successors remain `archive_pending`. On startup, hydration restores reducer state and retries `archive_pending`/`failed` rows with reducer dispatch disabled, so additive reducers are never applied twice. ```mermaid flowchart TD diff --git a/docs/sync-and-op-log/operation-log-architecture.md b/docs/sync-and-op-log/operation-log-architecture.md index 5a2cd5197d..8fa2f97c0f 100644 --- a/docs/sync-and-op-log/operation-log-architecture.md +++ b/docs/sync-and-op-log/operation-log-architecture.md @@ -170,6 +170,7 @@ interface OperationLogEntry { source: 'local' | 'remote'; syncedAt?: number; // For server sync (Part C) rejectedAt?: number; // When rejected during conflict resolution + applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed'; } // state_cache table - periodic snapshots @@ -205,6 +206,17 @@ interface StateCache { **Key insight:** All application data is persisted in the `SUP_OPS` database via the operation log system. +### Remote Apply Checkpoints + +Downloaded operations use a durable status transition so reducer state, archive IndexedDB side effects, vector clocks, and the server cursor cannot disagree after a crash: + +1. `pending` — the remote op is stored, but no reducer-commit checkpoint exists yet. +2. `archive_pending` — the complete batch was bulk-dispatched to reducers and its vector clocks were merged; archive side effects are still outstanding. +3. `applied` — reducer and archive work both completed. +4. `failed` — an attempted archive side effect failed. Later operations in the same batch remain `archive_pending` without consuming retry budget. + +Startup hydration replays the persisted state history, converts surviving `pending` rows to the archive checkpoint, and retries `archive_pending`/`failed` rows with reducer dispatch disabled. Ordinary sync refuses to download, upload, or advance its cursor while any of these incomplete rows remain. + ## A.2 Write Path ``` diff --git a/docs/sync-and-op-log/supersync-scenarios-flowchart.md b/docs/sync-and-op-log/supersync-scenarios-flowchart.md index 12408ba929..f997147213 100644 --- a/docs/sync-and-op-log/supersync-scenarios-flowchart.md +++ b/docs/sync-and-op-log/supersync-scenarios-flowchart.md @@ -105,6 +105,6 @@ flowchart TD **Notes:** - The `Enter Password` and `Decrypt Error` dialogs correspond to `DecryptNoPasswordError` and `DecryptError` respectively — they are distinct components with different options. -- `IMPORT_CONFLICT` gate uses pending ops only, not store contents (`_hasMeaningfulPendingOps()`). PASSWORD_CHANGED SYNC_IMPORTs without pending ops fall through this gate naturally to silent acceptance — the data is identical, only the encryption changed. "Meaningful" = TASK/PROJECT/TAG/NOTE create/update/delete or full-state ops — config-only ops don't count. Already-synced store data is not a conflict with the incoming SYNC_IMPORT — the user-facing warning happens on the originating device. Including store contents in the gate would let an old client pick `USE_LOCAL` and force-upload its stale pre-import state, rolling back the remote import for everyone. +- `IMPORT_CONFLICT` uses pending ops only, not already-synced store contents (`_hasMeaningfulPendingOps()`). Every pending op is protected except onboarding example-task creates and, on a never-synced client, the GLOBAL_CONFIG write that enables the `sync` section. This includes non-task entities, user configuration, and MIGRATION/RECOVERY genesis batches. PASSWORD_CHANGED SYNC_IMPORTs without pending ops still fall through to silent acceptance — the data is identical, only the encryption changed. Including already-synced store contents would let an old client pick `USE_LOCAL` and force-upload stale pre-import state, rolling back the remote import for everyone. - LWW tie-breaking: on equal timestamps, remote wins (server-authoritative). `moveToArchive` operations always win regardless of timestamp. - Re-download retry limit: max 3 resolution attempts per entity (`MAX_CONCURRENT_RESOLUTION_ATTEMPTS`); if exceeded, ops are permanently rejected. diff --git a/docs/sync-and-op-log/supersync-scenarios.md b/docs/sync-and-op-log/supersync-scenarios.md index 5154fb62b1..16a8d720d5 100644 --- a/docs/sync-and-op-log/supersync-scenarios.md +++ b/docs/sync-and-op-log/supersync-scenarios.md @@ -160,14 +160,14 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat ### C.3: Fresh Client — Has Pending Ops with Meaningful User Data (File-Based Sync Only) -**Trigger:** Client has unsynced ops containing task/project/tag/note create/update actions, receiving a snapshot from a file-based provider +**Trigger:** Client has unsynced local work and receives a snapshot from a file-based provider **Expected:** 1. Download detects remote snapshot (file-based sync path) -2. Check unsynced ops for meaningful user data: TASK/PROJECT/TAG/NOTE CREATE/UPDATE ops, or any full-state op (SYNC_IMPORT/BACKUP_IMPORT/REPAIR) +2. Treat every pending op as meaningful except onboarding example-task creates and, before the first completed sync only, the GLOBAL_CONFIG write that enables the `sync` section. This protects non-task entities, recovered MIGRATION/RECOVERY genesis batches, and user configuration. 3. If meaningful → throw `LocalDataConflictError` → full conflict dialog -4. If only config/system ops → proceed without dialog +4. If only the explicitly discardable startup ops remain → proceed without dialog **Note:** This op-content check only applies to the file-based snapshot path. For SuperSync (incremental ops path), the fresh client check uses `_hasMeaningfulStoreData()` (store-based check) instead. @@ -198,7 +198,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat **Expected:** 1. Download batch contains SYNC_IMPORT -2. Check pending local ops → N > 0 (condition satisfied regardless of meaningful data) +2. Check pending local ops. Any pending work triggers the dialog except onboarding example-task creates and the fresh-client bootstrap write for the GLOBAL_CONFIG `sync` section. 3. **Show conflict dialog BEFORE processing** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` 4. USE_LOCAL → `forceUploadLocalState()` (overrides remote with local data) 5. USE_REMOTE → `forceDownloadRemoteState()` (clears local ops, downloads from seq 0) @@ -254,7 +254,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat 1. Upload completes → server returns piggybacked ops containing SYNC_IMPORT 2. Check for SYNC_IMPORT in piggybacked ops BEFORE `processRemoteOps()` -3. If found AND `_hasMeaningfulPendingOps()` = true (unsynced TASK/PROJECT/TAG/NOTE C/U/D or full-state ops): +3. If found AND `_hasMeaningfulPendingOps()` = true (all pending work except onboarding example-task creates and the fresh-client GLOBAL_CONFIG `sync` bootstrap write): - **Show conflict dialog** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` from the piggybacked op - USE_LOCAL → `forceUploadLocalState()` (overrides remote) - USE_REMOTE → `forceDownloadRemoteState()` (clears local, downloads from seq 0) @@ -465,9 +465,9 @@ network changes while switching Wi-Fi) before surfacing the warning. **Expected:** Log warning ("Remote model version newer than local — app update may be required"). Returns `HANDLED_ERROR`. No alert shown to user. User needs to update app. -### G.8: Operation Migration Failure +### G.8: Operation Migration or Apply Failure -**Expected:** Failed ops skipped. Snackbar shown once per session. Other ops applied normally. +**Expected:** A schema migration block keeps the server cursor behind the incompatible op and reports an error instead of skipping it. A remote op whose reducer/archive checkpoint is incomplete stays `pending`, `archive_pending`, or `failed`; later same-session downloads and uploads are blocked so the cursor cannot advance past it. Startup hydration restores reducer state and retries only outstanding archive side effects before sync can continue. ### G.9: Concurrent Sync Attempts diff --git a/docs/wiki/3.06-User-Data.md b/docs/wiki/3.06-User-Data.md index a054ed6ed8..1baea56684 100644 --- a/docs/wiki/3.06-User-Data.md +++ b/docs/wiki/3.06-User-Data.md @@ -107,7 +107,7 @@ On mobile (Android/iOS), if the app launches with no local data but a usable on- **Manual backups:** (1) **Create manual backup** in Settings → Sync & Export creates a backup using the same mechanism as automatic backups (platform-dependent location). (2) **Safety backups** are created automatically before certain sync operations; the app keeps a limited number of slots (e.g. two most recent, one first from today, one first from day before). (3) **Export data** downloads a complete backup JSON file to a path you choose (or the browser download folder on web). -**What is included:** All application data is included in backups and exports; nothing is excluded. When archives are included (e.g. when using "Export data" with full backup), both `archive_young` and `archive_old` are part of the file. A separate **import backup** (pre-import state) is stored in IndexedDB (`import_backup` store) only for recovery if an import fails; it is not part of the user-visible backup set. +**What is included:** All application data is included in backups and exports; nothing is excluded. When archives are included (e.g. when using "Export data" with full backup), both `archive_young` and `archive_old` are part of the file. A separate, single-slot **import backup** is stored locally in IndexedDB (`import_backup`). The app captures it before replacing local state through a file import or the sync conflict action **Use Server Data**. A successful Use Server Data replacement shows a persistent **Undo** action that restores this backup; interrupted replacements offer the same recovery action when sync resumes. The slot is internal recovery state and is not included in exports or the user-visible automatic/manual backup lists. ## Import and Export From 1edcb1bfacb0f1c6438320ab7ab7e90d30a7f8f3 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:10:52 +0200 Subject: [PATCH 24/47] fix(supersync): isolate invalid upload operations --- .../src/supersync-http-contract.ts | 23 ++++- .../tests/supersync-http-contract.spec.ts | 79 ++++++++++++--- .../sync/services/operation-upload.service.ts | 12 ++- .../src/sync/sync.routes.ops-handler.ts | 14 +-- .../src/sync/sync.routes.quota.ts | 27 +++--- .../src/sync/sync.service.ts | 29 ++++++ ...quest-dedup-failure-caching.routes.spec.ts | 4 + .../tests/sync-compressed-body.routes.spec.ts | 97 ++++++++++++++++++- .../sync-upload-rate-limit.routes.spec.ts | 3 + .../tests/sync.service.spec.ts | 25 +++++ 10 files changed, 268 insertions(+), 45 deletions(-) diff --git a/packages/shared-schema/src/supersync-http-contract.ts b/packages/shared-schema/src/supersync-http-contract.ts index f510c270f7..5b7e393283 100644 --- a/packages/shared-schema/src/supersync-http-contract.ts +++ b/packages/shared-schema/src/supersync-http-contract.ts @@ -5,6 +5,12 @@ export const SUPER_SYNC_MAX_CLIENT_ID_LENGTH = 255; export const SUPER_SYNC_MAX_OPS_PER_UPLOAD = 100; export const SUPER_SYNC_MAX_ENTITY_IDS_PER_OP = 1000; +// Upload-only fields must be loose enough to reach per-operation validation, +// but still bounded so one invalid item cannot amplify logs/responses or make +// semantic validation walk an arbitrarily large identifier collection. +const SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH = 4096; +const SUPER_SYNC_MAX_INVALID_ENTITY_IDS_TRANSPORT = SUPER_SYNC_MAX_ENTITY_IDS_PER_OP * 2; + export const SUPER_SYNC_OP_TYPES = [ 'CRT', 'UPD', @@ -75,10 +81,21 @@ export const SuperSyncOperationSchema = z.object({ }); // Upload requests are envelopes for independently validated operations. Keep -// the transport shape numeric, but let the server return a per-op -// INVALID_SCHEMA_VERSION result so one malformed op cannot reject and stall -// every valid sibling in the batch. Download/response schemas remain strict. +// structural types and fields that ValidationService does not handle strict, +// but defer semantic operation validation to the server so one malformed op +// cannot reject and stall every valid sibling in the batch. Download/response +// schemas remain strict. const SuperSyncUploadOperationSchema = SuperSyncOperationSchema.extend({ + id: z.string().max(SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH), + clientId: z.string().max(SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH), + opType: z.string().max(SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH), + entityType: z.string().max(SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH), + entityId: z.string().max(SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH).optional(), + entityIds: z + .array(z.string().max(SUPER_SYNC_MAX_INVALID_FIELD_TRANSPORT_LENGTH)) + .max(SUPER_SYNC_MAX_INVALID_ENTITY_IDS_TRANSPORT) + .optional(), + vectorClock: z.record(z.string(), z.unknown()), schemaVersion: z.number(), }); diff --git a/packages/shared-schema/tests/supersync-http-contract.spec.ts b/packages/shared-schema/tests/supersync-http-contract.spec.ts index 10e8d30f4c..c8ff1a9582 100644 --- a/packages/shared-schema/tests/supersync-http-contract.spec.ts +++ b/packages/shared-schema/tests/supersync-http-contract.spec.ts @@ -47,21 +47,22 @@ describe('SuperSync HTTP contract schemas', () => { expect('extraOpField' in parsed.ops[0]).toBe(false); }); - it('caps entityIds per operation', () => { - expect(() => - SuperSyncUploadOpsRequestSchema.parse({ - ops: [ - { - ...createValidOperation(), - entityIds: Array.from( - { length: SUPER_SYNC_MAX_ENTITY_IDS_PER_OP + 1 }, - (_, i) => `task-${i}`, - ), - }, - ], - clientId: 'client_1', - }), - ).toThrow(); + it('passes oversized entityIds through for per-operation validation', () => { + const operation = { + ...createValidOperation(), + entityIds: Array.from( + { length: SUPER_SYNC_MAX_ENTITY_IDS_PER_OP + 1 }, + (_, i) => `task-${i}`, + ), + }; + + const parsed = SuperSyncUploadOpsRequestSchema.parse({ + ops: [operation], + clientId: 'client_1', + }); + + expect(parsed.ops[0].entityIds).toHaveLength(SUPER_SYNC_MAX_ENTITY_IDS_PER_OP + 1); + expect(() => SuperSyncOperationSchema.parse(operation)).toThrow(); }); it('rejects invalid client IDs in upload requests', () => { @@ -91,6 +92,54 @@ describe('SuperSync HTTP contract schemas', () => { }, ); + it.each([ + ['empty operation ID', { id: '' }], + ['overlong operation ID', { id: 'x'.repeat(256) }], + ['mismatched operation client ID', { clientId: 'other client' }], + ['unknown operation type', { opType: 'UNKNOWN' }], + ['unknown entity type', { entityType: 'UNKNOWN' }], + ['overlong entity ID', { entityId: 'x'.repeat(256) }], + ['invalid vector-clock entry', { vectorClock: { client_1: 'invalid' } }], + ])('passes semantic %s through for per-operation validation', (_label, override) => { + const operation = { ...createValidOperation(), ...override }; + const parsed = SuperSyncUploadOpsRequestSchema.parse({ + ops: [operation], + clientId: 'client_1', + }); + + expect(parsed.ops).toHaveLength(1); + }); + + it.each([ + ['non-string operation ID', { id: 123 }], + ['empty action type', { actionType: '' }], + ['non-string operation type', { opType: 123 }], + ['non-string entity type', { entityType: 123 }], + ['non-string entity ID', { entityId: 123 }], + ['non-string entityIds member', { entityIds: ['task-1', 123] }], + ['non-object vector clock', { vectorClock: [] }], + ['non-numeric timestamp', { timestamp: '123' }], + ['non-numeric schema version', { schemaVersion: '1' }], + ['non-boolean encryption flag', { isPayloadEncrypted: 'true' }], + ['unknown import reason', { syncImportReason: 'UNKNOWN' }], + ])('keeps the upload transport constraint for %s', (_label, override) => { + expect(() => + SuperSyncUploadOpsRequestSchema.parse({ + ops: [{ ...createValidOperation(), ...override }], + clientId: 'client_1', + }), + ).toThrow(); + }); + + it('rejects semantically invalid identifiers beyond the absolute transport cap', () => { + expect(() => + SuperSyncUploadOpsRequestSchema.parse({ + ops: [{ ...createValidOperation(), id: 'x'.repeat(4097) }], + clientId: 'client_1', + }), + ).toThrow(); + }); + it.each([0, 1.5, -1, 101])( 'rejects malformed snapshot schema version %s', (schemaVersion) => { diff --git a/packages/super-sync-server/src/sync/services/operation-upload.service.ts b/packages/super-sync-server/src/sync/services/operation-upload.service.ts index 550b4e4276..304997b93d 100644 --- a/packages/super-sync-server/src/sync/services/operation-upload.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-upload.service.ts @@ -29,7 +29,7 @@ import { pruneVectorClockForStorage, resolveConflictForExistingOp, } from '../conflict'; -import { ValidationService } from './validation.service'; +import { ValidationService, type ValidationResult } from './validation.service'; // Observability threshold: log a warning when the full-state op aggregate scan // exceeds this duration. Mirrors the threshold used by the legacy snapshot @@ -189,6 +189,7 @@ export class OperationUploadService { ops: Operation[], now: number, tx: Prisma.TransactionClient, + prevalidatedResults?: ReadonlyMap, ): Promise<{ results: UploadResult[]; acceptedDeltaBytes: number; @@ -209,6 +210,7 @@ export class OperationUploadService { ops, now, results, + prevalidatedResults, ); const uniqueCandidates = this.rejectIntraBatchDuplicates( @@ -286,12 +288,14 @@ export class OperationUploadService { ops: Operation[], now: number, results: UploadResult[], + prevalidatedResults?: ReadonlyMap, ): BatchUploadCandidate[] { const validatedCandidates: BatchUploadCandidate[] = []; for (let i = 0; i < ops.length; i++) { const op = ops[i]; const originalTimestamp = this.clampFutureTimestamp(userId, clientId, op, now); - const validation = this.validationService.validateOp(op, clientId); + const validation = + prevalidatedResults?.get(op) ?? this.validationService.validateOp(op, clientId); if (!validation.valid) { results[i] = this.rejectedUploadResult( @@ -613,6 +617,7 @@ export class OperationUploadService { op: Operation, now: number, tx: Prisma.TransactionClient, + prevalidatedResult?: ValidationResult, ): Promise { // Rejected ops have no storage cost; the caller only reads storageBytes when // result.accepted is true. @@ -627,7 +632,8 @@ export class OperationUploadService { const originalTimestamp = this.clampFutureTimestamp(userId, clientId, op, now); // Validate operation (including clientId match) - const validation = this.validationService.validateOp(op, clientId); + const validation = + prevalidatedResult ?? this.validationService.validateOp(op, clientId); if (!validation.valid) { Logger.audit({ event: 'OP_REJECTED', diff --git a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts index 56b260cdc5..ccef9f2961 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts @@ -25,7 +25,7 @@ import { sendCompressedBodyParseFailure, } from './sync.routes.payload'; import { - computeOpsStorageBytesExcludingKnownDuplicates, + computeOpsStorageBytesExcludingUnstorableIds, enforceStorageQuota, getRawOpsCount, sendOpsBatchTooLargeReply, @@ -194,15 +194,17 @@ export const uploadOpsHandler = async ( // duplicates are rejected by uploadOps and never written, so don't make // quota cleanup reserve space for them. const typedOpsForGate = ops as unknown as Operation[]; + const validOpsForQuota = syncService.filterValidOpsForQuota( + typedOpsForGate, + clientId, + ); const { bytes: estimatedDelta, fallback: gateFallback } = - await computeOpsStorageBytesExcludingKnownDuplicates( - userId, - typedOpsForGate, - syncService.getMaxClockDriftMs(), + await computeOpsStorageBytesExcludingUnstorableIds(validOpsForQuota, (op) => + syncService.getPrevalidatedPayloadBytes(op), ); if (gateFallback > 0) { Logger.warn( - `computeOpsStorageBytes: ${gateFallback}/${typedOpsForGate.length} unserializable op(s) ` + + `computeOpsStorageBytes: ${gateFallback}/${validOpsForQuota.length} unserializable op(s) ` + `charged at APPROX_BYTES_PER_OP for user=${userId} (gate)`, ); } diff --git a/packages/super-sync-server/src/sync/sync.routes.quota.ts b/packages/super-sync-server/src/sync/sync.routes.quota.ts index 3e850361fc..ecf3361619 100644 --- a/packages/super-sync-server/src/sync/sync.routes.quota.ts +++ b/packages/super-sync-server/src/sync/sync.routes.quota.ts @@ -3,13 +3,7 @@ import { prisma } from '../db'; import { Logger } from '../logger'; import { getSyncService } from './sync.service'; import { computeOpStorageBytes } from './sync.const'; -import { - DUPLICATE_OP_SELECT, - Operation, - SYNC_ERROR_CODES, - type DuplicateOperationCandidate, -} from './sync.types'; -import { isSameDuplicateOperation } from './conflict'; +import { DUPLICATE_OP_SELECT, Operation, SYNC_ERROR_CODES } from './sync.types'; import { errorMessage, MAX_OPS_PER_BATCH } from './sync.routes.payload'; /** @@ -37,10 +31,9 @@ export const computeOpsStorageBytes = ( return { bytes, fallback }; }; -export const computeOpsStorageBytesExcludingKnownDuplicates = async ( - userId: number, +export const computeOpsStorageBytesExcludingUnstorableIds = async ( ops: Operation[], - maxClockDriftMs: number, + getCachedPayloadBytes?: (op: Operation) => number | undefined, ): Promise<{ bytes: number; fallback: number }> => { if (ops.length === 0) return { bytes: 0, fallback: 0 }; @@ -48,19 +41,21 @@ export const computeOpsStorageBytesExcludingKnownDuplicates = async ( where: { id: { in: Array.from(new Set(ops.map((op) => op.id))) } }, select: DUPLICATE_OP_SELECT, }); - const existingOpById = new Map( - existingOps.map((existingOp) => [existingOp.id, existingOp]), - ); + const occupiedIds = new Set(existingOps.map((existingOp) => existingOp.id)); let bytes = 0; let fallback = 0; + const seenIncomingIds = new Set(); for (const op of ops) { - const existingOp = existingOpById.get(op.id); - if (existingOp && isSameDuplicateOperation(existingOp, userId, op, maxClockDriftMs)) { + // uploadOps stores at most the first new operation for an ID. Any occupied + // ID (exact retry or collision) and every repeated incoming ID is terminally + // rejected, so charging it here can only poison the pre-write quota gate. + if (occupiedIds.has(op.id) || seenIncomingIds.has(op.id)) { continue; } + seenIncomingIds.add(op.id); - const sized = computeOpStorageBytes(op); + const sized = computeOpStorageBytes(op, getCachedPayloadBytes?.(op)); bytes += sized.bytes; if (sized.fallback) fallback += 1; } diff --git a/packages/super-sync-server/src/sync/sync.service.ts b/packages/super-sync-server/src/sync/sync.service.ts index 113781f822..557b2adb48 100644 --- a/packages/super-sync-server/src/sync/sync.service.ts +++ b/packages/super-sync-server/src/sync/sync.service.ts @@ -23,6 +23,7 @@ import { type CacheSnapshotResult, type SnapshotDedupResponse, } from './services'; +import type { ValidationResult } from './services/validation.service'; const getPrismaP2002TargetTokens = ( err: Prisma.PrismaClientKnownRequestError, ): string[] => { @@ -82,6 +83,7 @@ export class SyncService { private storageQuotaService: StorageQuotaService; private snapshotService: SnapshotService; private operationUploadService: OperationUploadService; + private prevalidatedOps = new WeakMap(); constructor(config: Partial = {}) { this.config = { ...DEFAULT_SYNC_CONFIG, ...config }; @@ -102,6 +104,24 @@ export class SyncService { return this.config.maxClockDriftMs; } + /** + * Return only operations that can consume storage if this upload commits. + * Invalid siblings still reach uploadOps so the client receives a terminal + * per-operation rejection, but they must not inflate the pre-write quota gate + * and block otherwise valid operations in the same request. + */ + filterValidOpsForQuota(ops: Operation[], clientId: string): Operation[] { + return ops.filter((op) => { + const validation = this.validationService.validateOp(op, clientId); + this.prevalidatedOps.set(op, validation); + return validation.valid; + }); + } + + getPrevalidatedPayloadBytes(op: Operation): number | undefined { + return this.prevalidatedOps.get(op)?.payloadBytes; + } + // === Upload Operations === async uploadOps( @@ -114,6 +134,13 @@ export class SyncService { const now = Date.now(); const txStartedAt = Date.now(); let uploadDbRoundtrips = 0; + const prevalidatedResults = new Map(); + for (const op of ops) { + const validation = this.prevalidatedOps.get(op); + if (validation) { + prevalidatedResults.set(op, validation); + } + } try { // Use transaction to acquire write lock and ensure atomicity @@ -171,6 +198,7 @@ export class SyncService { ops, now, tx, + prevalidatedResults, ); results.push(...batchResult.results); acceptedDeltaBytes = batchResult.acceptedDeltaBytes; @@ -196,6 +224,7 @@ export class SyncService { op, now, tx, + prevalidatedResults.get(op), ); results.push(result); if (result.accepted) { diff --git a/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts b/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts index 862ad0ebe7..a930048875 100644 --- a/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts +++ b/packages/super-sync-server/tests/request-dedup-failure-caching.routes.spec.ts @@ -37,6 +37,8 @@ const mocks = vi.hoisted(() => { getStorageInfo: vi.fn(), getCachedSnapshotBytes: vi.fn(), getMaxClockDriftMs: vi.fn(), + filterValidOpsForQuota: vi.fn(), + getPrevalidatedPayloadBytes: vi.fn(), }; const prisma = { operation: { @@ -119,6 +121,7 @@ describe('Request dedup — transaction-failure results are not cached (#8332)', }); mocks.syncService.getCachedSnapshotBytes.mockResolvedValue(0); mocks.syncService.getMaxClockDriftMs.mockReturnValue(60_000); + mocks.syncService.filterValidOpsForQuota.mockImplementation((ops: unknown[]) => ops); mocks.syncService.cacheSnapshotIfReplayable.mockResolvedValue({ deltaBytes: 0 }); mocks.syncService.prepareSnapshotCache.mockResolvedValue({ stateBytes: 0, @@ -285,6 +288,7 @@ describe('Request dedup round-trip with the real cache (#8332)', () => { mocks.syncService.getLatestSeq.mockResolvedValue(1); mocks.syncService.getOpsSinceWithSeq.mockResolvedValue({ ops: [], latestSeq: 1 }); mocks.syncService.getMaxClockDriftMs.mockReturnValue(60_000); + mocks.syncService.filterValidOpsForQuota.mockImplementation((ops: unknown[]) => ops); mocks.prisma.operation.findMany.mockResolvedValue([]); app = Fastify(); diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index d9cd53a1c7..b867f5a5a6 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -27,6 +27,8 @@ const mocks = vi.hoisted(() => { getCachedSnapshotBytes: vi.fn(), markStorageNeedsReconcile: vi.fn(), getMaxClockDriftMs: vi.fn(), + filterValidOpsForQuota: vi.fn(), + getPrevalidatedPayloadBytes: vi.fn(), }; const prisma = { operation: { @@ -115,6 +117,7 @@ describe('Sync compressed body routes', () => { mocks.syncService.checkOpsRequestDedup.mockReturnValue(null); mocks.syncService.checkSnapshotRequestDedup.mockReturnValue(null); mocks.syncService.getMaxClockDriftMs.mockReturnValue(60_000); + mocks.syncService.filterValidOpsForQuota.mockImplementation((ops: unknown[]) => ops); mocks.syncService.checkStorageQuota.mockResolvedValue({ allowed: true, currentUsage: 0, @@ -255,6 +258,68 @@ describe('Sync compressed body routes', () => { ]); }); + it('should not let an invalid large sibling poison a near-quota upload', async () => { + const clientId = 'invalid-sibling-quota-client'; + const validOp = { + ...createOp(clientId), + id: 'valid-near-quota-op', + payload: { title: 'Valid task' }, + }; + const invalidLargeOp = { + ...createOp(clientId), + id: 'invalid-large-op', + opType: 'UNKNOWN', + payload: { data: 'x'.repeat(10_000) }, + }; + const validBytes = computeOpStorageBytes(validOp).bytes; + mocks.syncService.filterValidOpsForQuota.mockReturnValueOnce([validOp]); + mocks.syncService.checkStorageQuota.mockImplementation( + async (_userId: number, additionalBytes: number) => ({ + allowed: additionalBytes <= validBytes, + currentUsage: 1_000_000 - validBytes, + quota: 1_000_000, + }), + ); + mocks.syncService.uploadOps.mockResolvedValueOnce([ + { opId: validOp.id, accepted: true, serverSeq: 1 }, + { + opId: invalidLargeOp.id, + accepted: false, + error: 'Invalid opType', + errorCode: SYNC_ERROR_CODES.INVALID_OP_TYPE, + }, + ]); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/ops', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + }, + payload: { ops: [validOp, invalidLargeOp], clientId }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json().results).toEqual([ + expect.objectContaining({ opId: validOp.id, accepted: true }), + expect.objectContaining({ + opId: invalidLargeOp.id, + accepted: false, + errorCode: SYNC_ERROR_CODES.INVALID_OP_TYPE, + }), + ]); + expect(mocks.syncService.filterValidOpsForQuota).toHaveBeenCalledWith( + [validOp, invalidLargeOp], + clientId, + ); + expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith(1, validBytes); + expect(mocks.syncService.uploadOps).toHaveBeenCalledWith(1, clientId, [ + validOp, + invalidLargeOp, + ]); + }); + it('should subtract exact already-stored duplicate ops from the ops quota gate', async () => { const clientId = 'known-duplicate-quota-client'; const duplicateOp = { @@ -307,7 +372,7 @@ describe('Sync compressed body routes', () => { ]); }); - it('should keep same-id different-content ops charged in the ops quota gate', async () => { + it('should not charge same-id different-content ops in the quota gate', async () => { const clientId = 'id-collision-quota-client'; const incomingOp = { ...createOp(clientId), @@ -342,10 +407,38 @@ describe('Sync compressed body routes', () => { }, }); + expect(response.statusCode).toBe(200); + expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith(1, 0); + }); + + it('should charge only the first occurrence of a repeated new ID', async () => { + const clientId = 'intra-batch-duplicate-quota-client'; + const firstOp = { + ...createOp(clientId), + id: 'repeated-new-id', + entityId: 'task-first', + payload: { title: 'First content' }, + }; + const repeatedOp = { + ...firstOp, + entityId: 'task-second', + payload: { title: 'Different repeated content' }, + }; + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/ops', + headers: { + authorization: `Bearer ${authToken}`, + 'content-type': 'application/json', + }, + payload: { ops: [firstOp, repeatedOp], clientId }, + }); + expect(response.statusCode).toBe(200); expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith( 1, - computeOpStorageBytes(incomingOp).bytes, + computeOpStorageBytes(firstOp).bytes, ); }); diff --git a/packages/super-sync-server/tests/sync-upload-rate-limit.routes.spec.ts b/packages/super-sync-server/tests/sync-upload-rate-limit.routes.spec.ts index 187a75f068..8171ceed8d 100644 --- a/packages/super-sync-server/tests/sync-upload-rate-limit.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-upload-rate-limit.routes.spec.ts @@ -14,6 +14,8 @@ const mocks = vi.hoisted(() => { getLatestSeq: vi.fn(), getOpsSinceWithSeq: vi.fn(), getMaxClockDriftMs: vi.fn(), + filterValidOpsForQuota: vi.fn(), + getPrevalidatedPayloadBytes: vi.fn(), }; return { @@ -106,6 +108,7 @@ describe('Sync upload route rate limiting', () => { latestSeq: 1, }); mocks.syncService.getMaxClockDriftMs.mockReturnValue(60_000); + mocks.syncService.filterValidOpsForQuota.mockImplementation((ops: unknown[]) => ops); mocks.prisma.operation.findMany.mockResolvedValue([]); app = Fastify(); diff --git a/packages/super-sync-server/tests/sync.service.spec.ts b/packages/super-sync-server/tests/sync.service.spec.ts index bcb192276a..f7ad17641c 100644 --- a/packages/super-sync-server/tests/sync.service.spec.ts +++ b/packages/super-sync-server/tests/sync.service.spec.ts @@ -653,6 +653,31 @@ describe('SyncService', () => { delete process.env.OLD_OPS_CLEANUP_MAX_DELETED_PER_RUN; }); + describe('filterValidOpsForQuota', () => { + it('excludes invalid schema and oversized payload siblings from quota sizing', () => { + const service = new SyncService({ maxPayloadSizeBytes: 100 }); + const validOp = makeOp({ + id: 'valid-op', + payload: { title: 'Fits quota' }, + }); + const invalidSchemaOp = makeOp({ + id: 'invalid-schema-op', + schemaVersion: 101, + }); + const oversizedInvalidOp = makeOp({ + id: 'oversized-invalid-op', + payload: { data: 'x'.repeat(200) }, + }); + + const result = service.filterValidOpsForQuota( + [validOp, invalidSchemaOp, oversizedInvalidOp], + clientId, + ); + + expect(result).toEqual([validOp]); + }); + }); + describe('uploadOps', () => { it('should correctly upload operations', async () => { const service = getSyncService(); From e3093a416cfe806a615e9a0f45c2218c181fa15f Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:11:00 +0200 Subject: [PATCH 25/47] fix(sync-core): validate remote apply results --- packages/sync-core/src/index.ts | 1 + packages/sync-core/src/ports.ts | 18 +++ packages/sync-core/src/remote-apply.ts | 148 +++++++++++------- packages/sync-core/tests/remote-apply.spec.ts | 147 ++++++++++++++++- 4 files changed, 259 insertions(+), 55 deletions(-) diff --git a/packages/sync-core/src/index.ts b/packages/sync-core/src/index.ts index 3f6ed1ab7b..835011137b 100644 --- a/packages/sync-core/src/index.ts +++ b/packages/sync-core/src/index.ts @@ -109,6 +109,7 @@ export type { ConflictUiPort, DeferredLocalActionsPort, OperationApplyPort, + ReducerCommitAwareOperationApplyPort, RemoteApplyWindowPort, SyncActionLike, } from './ports'; diff --git a/packages/sync-core/src/ports.ts b/packages/sync-core/src/ports.ts index 76396607a4..da02c6fece 100644 --- a/packages/sync-core/src/ports.ts +++ b/packages/sync-core/src/ports.ts @@ -27,6 +27,24 @@ export interface OperationApplyPort = Opera ): Promise>; } +/** + * Operation applier contract for crash-safe remote application. + * + * Unlike the general-purpose {@link OperationApplyPort}, remote application + * requires the reducer-commit callback: the coordinator must durably checkpoint + * the whole reducer batch before archive side effects can begin. + */ +export interface ReducerCommitAwareOperationApplyPort< + TOperation extends Operation = Operation, +> { + applyOperations( + ops: TOperation[], + options: ApplyOperationsOptions & { + onReducersCommitted: (ops: TOperation[]) => Promise; + }, + ): Promise>; +} + /** * Port for dispatching host actions. * diff --git a/packages/sync-core/src/remote-apply.ts b/packages/sync-core/src/remote-apply.ts index 70544f8d81..1c5dc1cf26 100644 --- a/packages/sync-core/src/remote-apply.ts +++ b/packages/sync-core/src/remote-apply.ts @@ -1,6 +1,6 @@ import type { ApplyOperationsResult } from './apply.types'; import type { Operation } from './operation.types'; -import type { OperationApplyPort } from './ports'; +import type { ReducerCommitAwareOperationApplyPort } from './ports'; export interface RemoteOperationAppendResult< TOperation extends Operation = Operation, @@ -36,7 +36,7 @@ export interface ApplyRemoteOperationsOptions< > { ops: TOperation[]; store: RemoteOperationApplyStorePort; - applier: OperationApplyPort; + applier: ReducerCommitAwareOperationApplyPort; isFullStateOperation?: (op: TOperation) => boolean; } @@ -71,14 +71,14 @@ const emptyRemoteApplyResult = < * the generic ordering: * * 1. append incoming remote ops as pending, skipping duplicates atomically; - * 2. apply only the newly appended ops through the host applier; - * 3. mark successfully applied seqs; - * 4. merge applied remote vector clocks; + * 2. bulk-apply only the newly appended ops through the host applier; + * 3. at reducer commit, mark the whole batch archive_pending and merge all + * reducer-committed vector clocks before archive side effects begin; + * 4. mark archive-complete seqs applied; * 5. after applying a full-state op, clear older full-state ops while * retaining every full-state op of this batch (incl. archive_pending); - * 6. mark the failed op and remaining ops as failed on partial error — their - * reducer effects committed with the bulk dispatch; only their archive side - * effects are outstanding (see ApplyOperationsResult.failedOp). + * 6. mark only the attempted archive failure as failed; unattempted successors + * remain archive_pending for ordered startup recovery. */ export const applyRemoteOperations = async < TOperation extends Operation = Operation, @@ -114,48 +114,62 @@ export const applyRemoteOperations = async < }); let reducerCommitCallbackCount = 0; + let reducerCommitCallbackError: Error | undefined; let reducerCommitPromise: Promise | undefined; - const applyResult = await applier.applyOperations(appendResult.writtenOps, { - onReducersCommitted: (reducerCommittedOps) => { - reducerCommitCallbackCount++; - if (reducerCommitCallbackCount !== 1) { - reducerCommitPromise = Promise.reject( - new Error( + let applyResult: ApplyOperationsResult; + try { + applyResult = await applier.applyOperations(appendResult.writtenOps, { + onReducersCommitted: (reducerCommittedOps) => { + reducerCommitCallbackCount++; + if (reducerCommitCallbackCount !== 1) { + reducerCommitCallbackError = new Error( 'applyRemoteOperations: reducer-commit callback must be invoked exactly once.', - ), - ); - return reducerCommitPromise; - } - - const isEntireAppendedBatch = - reducerCommittedOps.length === appendResult.writtenOps.length && - reducerCommittedOps.every( - (op, index) => op.id === appendResult.writtenOps[index]?.id, - ); - if (!isEntireAppendedBatch) { - reducerCommitPromise = Promise.reject( - new Error( - 'applyRemoteOperations: reducer-commit callback must contain the entire appended batch in order.', - ), - ); - return reducerCommitPromise; - } - - reducerCommitPromise = (async () => { - const reducerCommittedSeqs = reducerCommittedOps - .map((op) => opIdToSeq.get(op.id)) - .filter((seq): seq is number => seq !== undefined); - if (reducerCommittedSeqs.length !== reducerCommittedOps.length) { - throw new Error( - 'applyRemoteOperations: reducer commit contained an operation outside the appended batch.', ); + throw reducerCommitCallbackError; } - await store.markArchivePending(reducerCommittedSeqs); - await store.mergeRemoteOpClocks(reducerCommittedOps); - })(); - return reducerCommitPromise; - }, - }); + + const isEntireAppendedBatch = + reducerCommittedOps.length === appendResult.writtenOps.length && + reducerCommittedOps.every( + (op, index) => op.id === appendResult.writtenOps[index]?.id, + ); + if (!isEntireAppendedBatch) { + reducerCommitCallbackError = new Error( + 'applyRemoteOperations: reducer-commit callback must contain the entire appended batch in order.', + ); + throw reducerCommitCallbackError; + } + + reducerCommitPromise = (async () => { + const reducerCommittedSeqs = appendResult.writtenOps + .map((op) => opIdToSeq.get(op.id)) + .filter((seq): seq is number => seq !== undefined); + if (reducerCommittedSeqs.length !== appendResult.writtenOps.length) { + throw new Error( + 'applyRemoteOperations: reducer commit contained an operation outside the appended batch.', + ); + } + await store.markArchivePending(reducerCommittedSeqs); + await store.mergeRemoteOpClocks(appendResult.writtenOps); + })(); + return reducerCommitPromise; + }, + }); + } catch (applyError) { + // A valid first callback may already have started durable bookkeeping before + // a duplicate callback (or another applier error) throws. Observe that + // promise before propagating so it cannot become an unhandled rejection. + if (reducerCommitPromise) { + await reducerCommitPromise; + } + throw applyError; + } + if (reducerCommitCallbackError) { + if (reducerCommitPromise) { + await reducerCommitPromise; + } + throw reducerCommitCallbackError; + } if (reducerCommitCallbackCount !== 1 || reducerCommitPromise === undefined) { throw new Error( 'applyRemoteOperations: applier did not invoke the reducer-commit callback.', @@ -165,12 +179,40 @@ export const applyRemoteOperations = async < // awaiting its returned promise. Pending ops must never be marked applied // before the reducer/archive checkpoint is durable. await reducerCommitPromise; - if (applyResult.failedOp && !opIdToSeq.has(applyResult.failedOp.op.id)) { + + const appliedOpCount = applyResult.appliedOps.length; + const hasExactAppliedPrefix = + appliedOpCount <= appendResult.writtenOps.length && + applyResult.appliedOps.every( + (op, index) => op.id === appendResult.writtenOps[index]?.id, + ); + if (!hasExactAppliedPrefix) { throw new Error( - `applyRemoteOperations: applier reported unknown failed op ${applyResult.failedOp.op.id}.`, + 'applyRemoteOperations: applied operations must be the exact ordered prefix of the appended batch.', ); } - const appliedSeqs = applyResult.appliedOps + + const expectedFailedOp = appendResult.writtenOps[appliedOpCount]; + if ( + (applyResult.failedOp && applyResult.failedOp.op.id !== expectedFailedOp?.id) || + (!applyResult.failedOp && expectedFailedOp !== undefined) + ) { + const reportedFailedOpId = applyResult.failedOp?.op.id ?? 'none'; + const expectedFailedOpId = expectedFailedOp?.id ?? 'none'; + throw new Error( + 'applyRemoteOperations: applier must report the failed operation immediately after the applied prefix. ' + + `Reported ${reportedFailedOpId}; expected ${expectedFailedOpId}.`, + ); + } + + const authoritativeAppliedOps = appendResult.writtenOps.slice(0, appliedOpCount); + const authoritativeFailedOp = applyResult.failedOp + ? { + op: expectedFailedOp!, + error: applyResult.failedOp.error, + } + : undefined; + const appliedSeqs = authoritativeAppliedOps .map((op) => opIdToSeq.get(op.id)) .filter((seq): seq is number => seq !== undefined); @@ -179,7 +221,7 @@ export const applyRemoteOperations = async < if (appliedSeqs.length > 0) { await store.markApplied(appliedSeqs); - const appliedFullStateOpIds = applyResult.appliedOps + const appliedFullStateOpIds = authoritativeAppliedOps .filter(isFullStateOperation) .map((op) => op.id); @@ -197,7 +239,7 @@ export const applyRemoteOperations = async < } } - const failedOpIds = applyResult.failedOp ? [applyResult.failedOp.op.id] : []; + const failedOpIds = authoritativeFailedOp ? [authoritativeFailedOp.op.id] : []; if (failedOpIds.length > 0) { await store.markFailed(failedOpIds); @@ -206,10 +248,10 @@ export const applyRemoteOperations = async < return { appendedOps: appendResult.writtenOps, skippedCount: appendResult.skippedCount, - appliedOps: applyResult.appliedOps, + appliedOps: authoritativeAppliedOps, appliedSeqs, clearedFullStateOpCount, - failedOp: applyResult.failedOp, + failedOp: authoritativeFailedOp, failedOpIds, }; }; diff --git a/packages/sync-core/tests/remote-apply.spec.ts b/packages/sync-core/tests/remote-apply.spec.ts index bdfedb28db..478d9bedb3 100644 --- a/packages/sync-core/tests/remote-apply.spec.ts +++ b/packages/sync-core/tests/remote-apply.spec.ts @@ -1,7 +1,11 @@ import { describe, expect, it, vi } from 'vitest'; import { applyRemoteOperations } from '../src/remote-apply'; import type { RemoteOperationApplyStorePort } from '../src/remote-apply'; -import type { Operation, OperationApplyPort } from '../src'; +import type { + Operation, + OperationApplyPort, + ReducerCommitAwareOperationApplyPort, +} from '../src'; const createOperation = (id: string, opType = 'UPD'): Operation => ({ id, @@ -31,7 +35,7 @@ const createStore = (appendResult: { const createApplier = ( result: Awaited>['applyOperations']>>, -): OperationApplyPort> => ({ +): ReducerCommitAwareOperationApplyPort> => ({ applyOperations: vi.fn(async (ops, options) => { await options?.onReducersCommitted?.(ops); return result; @@ -238,6 +242,145 @@ describe('applyRemoteOperations', () => { ); expect(store.markApplied).not.toHaveBeenCalled(); }); + + it('uses authoritative written operations for clocks, cleanup, and results', async () => { + const writtenImport = createOperation('sync-import-1', 'SYNC_IMPORT'); + writtenImport.vectorClock = { authoritative: 7 }; + const callbackClone = { + ...writtenImport, + vectorClock: { forgedCallback: 99 }, + }; + const appliedClone = { + ...writtenImport, + opType: 'UPD', + vectorClock: { forgedResult: 100 }, + }; + const store = createStore({ + seqs: [11], + writtenOps: [writtenImport], + skippedCount: 0, + }); + const applier: OperationApplyPort> = { + applyOperations: vi.fn(async (_ops, options) => { + await options?.onReducersCommitted?.([callbackClone]); + return { appliedOps: [appliedClone] }; + }), + }; + + const result = await applyRemoteOperations({ + ops: [writtenImport], + store, + applier, + isFullStateOperation: (op) => op.opType === 'SYNC_IMPORT', + }); + + expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([writtenImport]); + expect(store.clearFullStateOpsExcept).toHaveBeenCalledWith(['sync-import-1']); + expect(result.appendedOps[0]).toBe(writtenImport); + expect(result.appliedOps[0]).toBe(writtenImport); + }); + + it('rejects applied operations that are not the exact ordered written prefix', async () => { + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const store = createStore({ + seqs: [1, 2], + writtenOps: [op1, op2], + skippedCount: 0, + }); + const applier = createApplier({ appliedOps: [op2, op1] }); + + await expect( + applyRemoteOperations({ ops: [op1, op2], store, applier }), + ).rejects.toThrow('exact ordered prefix'); + expect(store.markApplied).not.toHaveBeenCalled(); + }); + + it('rejects a partial applied prefix without the next failed operation', async () => { + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const store = createStore({ + seqs: [1, 2], + writtenOps: [op1, op2], + skippedCount: 0, + }); + const applier = createApplier({ appliedOps: [op1] }); + + await expect( + applyRemoteOperations({ ops: [op1, op2], store, applier }), + ).rejects.toThrow('failed operation immediately after the applied prefix'); + expect(store.markApplied).not.toHaveBeenCalled(); + }); + + it('rejects a failed operation that is not immediately after the applied prefix', async () => { + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const op3 = createOperation('op-3'); + const store = createStore({ + seqs: [1, 2, 3], + writtenOps: [op1, op2, op3], + skippedCount: 0, + }); + const applier = createApplier({ + appliedOps: [op1], + failedOp: { op: op3, error: new Error('wrong failure') }, + }); + + await expect( + applyRemoteOperations({ ops: [op1, op2, op3], store, applier }), + ).rejects.toThrow('failed operation immediately after the applied prefix'); + expect(store.markFailed).not.toHaveBeenCalled(); + }); + + it('throws synchronously from a malformed reducer-commit callback', async () => { + const op1 = createOperation('op-1'); + const op2 = createOperation('op-2'); + const store = createStore({ + seqs: [1, 2], + writtenOps: [op1, op2], + skippedCount: 0, + }); + let callbackThrewSynchronously = false; + const applier: OperationApplyPort> = { + applyOperations: vi.fn(async (_ops, options) => { + try { + void options?.onReducersCommitted?.([op2, op1]); + } catch (error) { + callbackThrewSynchronously = true; + throw error; + } + return { appliedOps: [op1, op2] }; + }), + }; + + await expect( + applyRemoteOperations({ ops: [op1, op2], store, applier }), + ).rejects.toThrow('entire appended batch in order'); + expect(callbackThrewSynchronously).toBe(true); + }); + + it('throws synchronously from a duplicate reducer-commit callback', async () => { + const op = createOperation('op-1'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + let duplicateCallbackThrewSynchronously = false; + const applier: OperationApplyPort> = { + applyOperations: vi.fn(async (ops, options) => { + void options?.onReducersCommitted?.(ops); + try { + void options?.onReducersCommitted?.(ops); + } catch (error) { + duplicateCallbackThrewSynchronously = true; + throw error; + } + return { appliedOps: ops }; + }), + }; + + await expect(applyRemoteOperations({ ops: [op], store, applier })).rejects.toThrow( + 'exactly once', + ); + expect(duplicateCallbackThrewSynchronously).toBe(true); + }); }); const jasmineLikeObjectContainingFunction = (): object => From 62e9e34b296ce1c952fed310dd4847c95e195516 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:11:11 +0200 Subject: [PATCH 26/47] fix(sync): preserve incomplete recovery work --- .../diagrams/06-archive-operations.md | 18 +++-- .../operation-log-architecture.md | 6 +- .../supersync-scenarios-flowchart.md | 2 +- docs/sync-and-op-log/supersync-scenarios.md | 4 +- .../imex/sync/sync-wrapper.service.spec.ts | 17 +++++ src/app/imex/sync/sync-wrapper.service.ts | 9 +++ .../capture/operation-log.effects.spec.ts | 20 +++--- .../op-log/capture/operation-log.effects.ts | 42 ++++++----- src/app/op-log/core/errors/sync-errors.ts | 18 ++++- src/app/op-log/core/operation-log.const.ts | 11 +-- ...ion-log-hydrator.retry.integration.spec.ts | 31 ++++---- .../operation-log-hydrator.service.spec.ts | 22 +++--- .../operation-log-hydrator.service.ts | 11 +-- .../operation-log-recovery.service.spec.ts | 20 +++--- .../operation-log-recovery.service.ts | 21 ++++-- .../operation-log-store.service.spec.ts | 36 +++++++--- .../operation-log-store.service.ts | 51 ++++++++----- .../sync/conflict-resolution.service.ts | 6 +- .../sync/immediate-upload.service.spec.ts | 43 +++++++++++ .../op-log/sync/immediate-upload.service.ts | 8 +++ .../sync/operation-log-sync.service.spec.ts | 64 +++++++++++++++-- .../op-log/sync/operation-log-sync.service.ts | 53 +++++++++----- ...g-upload-piggyback-seq.integration.spec.ts | 11 +++ .../operation-write-flush.service.spec.ts | 7 ++ .../sync/operation-write-flush.service.ts | 5 ++ .../remote-ops-processing.service.spec.ts | 72 +++++++++---------- .../sync/remote-ops-processing.service.ts | 3 +- .../sync-import-conflict-gate.service.spec.ts | 15 ++-- .../sync/sync-import-conflict-gate.service.ts | 25 +------ .../ws-triggered-download.service.spec.ts | 24 +++++++ .../sync/ws-triggered-download.service.ts | 17 +++++ .../migration-handling.integration.spec.ts | 22 +++--- .../service-logic.integration.spec.ts | 5 +- src/app/t.const.ts | 2 + src/assets/i18n/en.json | 4 +- 35 files changed, 491 insertions(+), 234 deletions(-) diff --git a/docs/sync-and-op-log/diagrams/06-archive-operations.md b/docs/sync-and-op-log/diagrams/06-archive-operations.md index a65944f37d..d6b25e71ec 100644 --- a/docs/sync-and-op-log/diagrams/06-archive-operations.md +++ b/docs/sync-and-op-log/diagrams/06-archive-operations.md @@ -86,10 +86,13 @@ flowchart TD end subgraph RemoteOp["REMOTE Operation (Sync)"] - R1[Download operation
from sync] --> R2[OperationApplierService
dispatches action] - R2 --> R3[Meta-reducers update NgRx state] - R3 --> R4["ArchiveOperationHandler
.handleOperation"] - R4 --> R5["Write to IndexedDB
(archiveYoung/archiveOld)"] + R1[Download operation
from sync] --> R2["Append remote row
status: pending"] + R2 --> R3["Bulk reducer dispatch
meta.isRemote=true"] + R3 --> R4["Reducer-commit checkpoint:
status archive_pending
merge vector clocks"] + R4 --> R5["ArchiveOperationHandler
.handleOperation"] + R5 --> R6{"Archive side effect
succeeded?"} + R6 -->|Yes| R7["Mark applied"] + R6 -->|No| R8["Mark attempted row failed;
successors stay archive_pending"] NoEffect["❌ Regular effects DON'T run
(action has meta.isRemote=true)"] end @@ -118,9 +121,10 @@ The `OperationApplierService` applies operations in the order they arrive from t ```mermaid flowchart TD subgraph OperationApplierService["OperationApplierService (Bulk Dispatch)"] - OA1[Receive operations] --> OA3[convertOpToAction] - OA3 --> OA4["store.dispatch(action)
with meta.isRemote=true"] - OA4 --> OA5["archiveOperationHandler
.handleOperation(action)"] + OA1["Receive rows already stored
as pending"] --> OA3[convertOpToAction] + OA3 --> OA4["Single bulk dispatch
with meta.isRemote=true"] + OA4 --> OA4b["Reducer-commit callback:
mark archive_pending
merge vector clocks"] + OA4b --> OA5["archiveOperationHandler
.handleOperation(action)"] end subgraph Handler["ArchiveOperationHandler"] diff --git a/docs/sync-and-op-log/operation-log-architecture.md b/docs/sync-and-op-log/operation-log-architecture.md index 8fa2f97c0f..fd663c96bb 100644 --- a/docs/sync-and-op-log/operation-log-architecture.md +++ b/docs/sync-and-op-log/operation-log-architecture.md @@ -217,6 +217,8 @@ Downloaded operations use a durable status transition so reducer state, archive Startup hydration replays the persisted state history, converts surviving `pending` rows to the archive checkpoint, and retries `archive_pending`/`failed` rows with reducer dispatch disabled. Ordinary sync refuses to download, upload, or advance its cursor while any of these incomplete rows remain. +Local actions buffered during a remote-apply window stay ordered until each operation is durable. Transient persistence failures keep the failed suffix queued and block the current sync so a later sync can retry. A deterministically invalid buffered action also remains queued, but requires reload: its reducer already changed live state, so discarding it would let live state diverge from the durable operation log. + ## A.2 Write Path ``` @@ -440,7 +442,7 @@ async compact(): Promise { | Max download ops in memory | 50,000 | Bounds memory during API download | | Remote file retention | 14 days | Server-side operation file retention | | Max remote files to keep | 100 | Minimum recent files on server | -| Max conflict retry attempts | 5 | Retries before rejecting failed ops | +| Remote archive retries | ∞ | Stay quarantined until side effects complete | | Max rejected ops before warning | 10 | Threshold for user notification | | Lock timeout | 30 sec | localStorage fallback lock timeout | | Lock acquire timeout | 60 sec | Max wait to acquire a lock | @@ -2322,7 +2324,7 @@ When adding new entities or relationships: > - **Archive validation**: archiveOld tasks now validated for project/tag references, null-safety added > - **Lock service robustness**: Handle NaN timestamps and invalid lock formats in fallback lock > - **Array payload rejection**: Explicit check to reject arrays (which bypass `typeof === 'object'`) -> - **Pending operation expiry**: Operations pending >24h are rejected instead of replayed (PENDING_OPERATION_EXPIRY_MS) +> - **Pending operation expiry**: Operations pending >24h are quarantined as failed; reducer replay is skipped but archive recovery and the sync gate remain active (`PENDING_OPERATION_EXPIRY_MS`). --- diff --git a/docs/sync-and-op-log/supersync-scenarios-flowchart.md b/docs/sync-and-op-log/supersync-scenarios-flowchart.md index f997147213..0ce06eaa22 100644 --- a/docs/sync-and-op-log/supersync-scenarios-flowchart.md +++ b/docs/sync-and-op-log/supersync-scenarios-flowchart.md @@ -105,6 +105,6 @@ flowchart TD **Notes:** - The `Enter Password` and `Decrypt Error` dialogs correspond to `DecryptNoPasswordError` and `DecryptError` respectively — they are distinct components with different options. -- `IMPORT_CONFLICT` uses pending ops only, not already-synced store contents (`_hasMeaningfulPendingOps()`). Every pending op is protected except onboarding example-task creates and, on a never-synced client, the GLOBAL_CONFIG write that enables the `sync` section. This includes non-task entities, user configuration, and MIGRATION/RECOVERY genesis batches. PASSWORD_CHANGED SYNC_IMPORTs without pending ops still fall through to silent acceptance — the data is identical, only the encryption changed. Including already-synced store contents would let an old client pick `USE_LOCAL` and force-upload stale pre-import state, rolling back the remote import for everyone. +- `IMPORT_CONFLICT` uses pending ops only, not already-synced store contents (`_hasMeaningfulPendingOps()`). Every pending op is protected except onboarding example-task creates. This includes GLOBAL_CONFIG changes before first sync, non-task entities, and MIGRATION/RECOVERY genesis batches. PASSWORD_CHANGED SYNC_IMPORTs without pending ops still fall through to silent acceptance — the data is identical, only the encryption changed. Including already-synced store contents would let an old client pick `USE_LOCAL` and force-upload stale pre-import state, rolling back the remote import for everyone. - LWW tie-breaking: on equal timestamps, remote wins (server-authoritative). `moveToArchive` operations always win regardless of timestamp. - Re-download retry limit: max 3 resolution attempts per entity (`MAX_CONCURRENT_RESOLUTION_ATTEMPTS`); if exceeded, ops are permanently rejected. diff --git a/docs/sync-and-op-log/supersync-scenarios.md b/docs/sync-and-op-log/supersync-scenarios.md index 16a8d720d5..3d3a315a40 100644 --- a/docs/sync-and-op-log/supersync-scenarios.md +++ b/docs/sync-and-op-log/supersync-scenarios.md @@ -198,7 +198,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat **Expected:** 1. Download batch contains SYNC_IMPORT -2. Check pending local ops. Any pending work triggers the dialog except onboarding example-task creates and the fresh-client bootstrap write for the GLOBAL_CONFIG `sync` section. +2. Check pending local ops. Any pending work triggers the dialog except onboarding example-task creates. GLOBAL_CONFIG changes are protected even before the first completed sync because the sync-section payload can carry user preferences. 3. **Show conflict dialog BEFORE processing** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` 4. USE_LOCAL → `forceUploadLocalState()` (overrides remote with local data) 5. USE_REMOTE → `forceDownloadRemoteState()` (clears local ops, downloads from seq 0) @@ -254,7 +254,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat 1. Upload completes → server returns piggybacked ops containing SYNC_IMPORT 2. Check for SYNC_IMPORT in piggybacked ops BEFORE `processRemoteOps()` -3. If found AND `_hasMeaningfulPendingOps()` = true (all pending work except onboarding example-task creates and the fresh-client GLOBAL_CONFIG `sync` bootstrap write): +3. If found AND `_hasMeaningfulPendingOps()` = true (all pending work except onboarding example-task creates): - **Show conflict dialog** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` from the piggybacked op - USE_LOCAL → `forceUploadLocalState()` (overrides remote) - USE_REMOTE → `forceDownloadRemoteState()` (clears local, downloads from seq 0) diff --git a/src/app/imex/sync/sync-wrapper.service.spec.ts b/src/app/imex/sync/sync-wrapper.service.spec.ts index a4d2112e7e..4e3d3e380a 100644 --- a/src/app/imex/sync/sync-wrapper.service.spec.ts +++ b/src/app/imex/sync/sync-wrapper.service.spec.ts @@ -37,6 +37,7 @@ import { UploadRevToMatchMismatchAPIError, WebDavNativeRequestError, EncryptNoPasswordError, + IncompleteRemoteOperationsError, } from '../../op-log/core/errors/sync-errors'; import { DialogEnterEncryptionPasswordComponent } from './dialog-enter-encryption-password/dialog-enter-encryption-password.component'; import { MAX_LWW_REUPLOAD_RETRIES } from '../../op-log/core/operation-log.const'; @@ -1049,6 +1050,22 @@ describe('SyncWrapperService', () => { }); describe('Error handling', () => { + it('should surface incomplete remote application as a sticky translated error', async () => { + mockSyncService.downloadRemoteOps.and.rejectWith( + new IncompleteRemoteOperationsError(), + ); + + const result = await service.sync(); + + expect(result).toBe('HANDLED_ERROR'); + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).toHaveBeenCalledWith({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + }); + it('should handle PotentialCorsError with snack message', async () => { mockSyncService.downloadRemoteOps.and.returnValue( Promise.reject(new PotentialCorsError('https://example.com')), diff --git a/src/app/imex/sync/sync-wrapper.service.ts b/src/app/imex/sync/sync-wrapper.service.ts index 0c9280756b..17c8870b1d 100644 --- a/src/app/imex/sync/sync-wrapper.service.ts +++ b/src/app/imex/sync/sync-wrapper.service.ts @@ -21,6 +21,7 @@ import { EmptyRemoteBodySPError, JsonParseError, LegacySyncFormatDetectedError, + IncompleteRemoteOperationsError, SyncDataCorruptedError, UploadRevToMatchMismatchAPIError, } from '../../op-log/core/errors/sync-errors'; @@ -707,6 +708,14 @@ export class SyncWrapperService { config: { duration: 12000 }, }); return 'HANDLED_ERROR'; + } else if (error instanceof IncompleteRemoteOperationsError) { + this._providerManager.setSyncStatus('ERROR'); + this._snackService.open({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + return 'HANDLED_ERROR'; } else if ( error instanceof AuthFailSPError || error instanceof MissingRefreshTokenAPIError || diff --git a/src/app/op-log/capture/operation-log.effects.spec.ts b/src/app/op-log/capture/operation-log.effects.spec.ts index b28d9c05c4..a330c80f8a 100644 --- a/src/app/op-log/capture/operation-log.effects.spec.ts +++ b/src/app/op-log/capture/operation-log.effects.spec.ts @@ -918,7 +918,7 @@ describe('OperationLogEffects', () => { expect(getDeferredActions()).toEqual([]); }); - it('should abandon a permanently invalid deferred action and continue with its successors', async () => { + it('should block and keep a permanently invalid deferred action with its successors', async () => { // Invalid entity identifiers are deterministic: retrying every sync // window forever (with a sticky error snack each time) can never succeed. const invalidAction = createPersistentAction(ActionType.TASK_SHARED_ADD); @@ -927,19 +927,17 @@ describe('OperationLogEffects', () => { bufferDeferredAction(invalidAction); bufferDeferredAction(validAction); - await effects.processDeferredActions(); + await expectAsync(effects.processDeferredActions()).toBeRejected(); - // Invalid action: single attempt, no retries, abandoned. Valid successor - // persisted (its relative order w.r.t. an unpersistable action is moot). - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledTimes(1); - expect( - mockOpLogStore.appendWithVectorClockUpdate.calls.mostRecent().args[0].actionType, - ).toBe(ActionType.TASK_SHARED_UPDATE); - expect(getDeferredActions()).toEqual([]); + // The invalid reducer action already changed live state. Neither it nor + // its successor may be discarded or persisted out of order. + expect(mockOpLogStore.appendWithVectorClockUpdate).not.toHaveBeenCalled(); + expect(getDeferredActions()).toEqual([invalidAction, validAction]); expect(mockSnackService.open).toHaveBeenCalledWith( jasmine.objectContaining({ - msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED, - actionStr: T.G.DISMISS, + msg: T.F.SYNC.S.DEFERRED_ACTION_PERMANENT_FAILED, + actionStr: T.PS.RELOAD, + actionFn: jasmine.any(Function), }), ); }); diff --git a/src/app/op-log/capture/operation-log.effects.ts b/src/app/op-log/capture/operation-log.effects.ts index 76edf6a342..fa2c83248b 100644 --- a/src/app/op-log/capture/operation-log.effects.ts +++ b/src/app/op-log/capture/operation-log.effects.ts @@ -692,8 +692,7 @@ export class OperationLogEffects implements DeferredLocalActionsPort { const MAX_RETRIES = 3; const BASE_DELAY_MS = 100; - let failedCount = 0; - let transientFailure: unknown; + let failure: unknown; for (const action of deferredActions) { let lastError: unknown; @@ -730,17 +729,16 @@ export class OperationLogEffects implements DeferredLocalActionsPort { continue; } - failedCount++; - if (lastError instanceof PermanentDeferredWriteError) { - // Terminal: the action can never be persisted (invalid identifiers or - // payload). Abandon it — keeping it queued would retry it with backoff - // and re-raise the failure snack on every future sync window. - acknowledgeDeferredAction(action); - OpLog.err('OperationLogEffects: Abandoning permanently invalid deferred action', { + // The reducer already committed this action. Dropping it would leave + // live state ahead of durable state and let sync claim convergence. + // Keep the full suffix buffered and require reload to roll live state + // back to the last durable snapshot. + OpLog.err('OperationLogEffects: Permanently invalid deferred action', { actionType: action.type, }); - continue; + failure = lastError; + break; } // Transient failure: STOP the drain here instead of persisting later @@ -755,26 +753,32 @@ export class OperationLogEffects implements DeferredLocalActionsPort { errorName: (lastError as Error | undefined)?.name, }, ); - transientFailure = lastError; + failure = lastError; break; } - // Show notification if any actions failed - if (failedCount > 0) { + if (failure) { + const isPermanent = failure instanceof PermanentDeferredWriteError; this.snackService.open({ type: 'ERROR', - msg: T.F.SYNC.S.DEFERRED_ACTION_FAILED, - actionStr: T.G.DISMISS, + msg: isPermanent + ? T.F.SYNC.S.DEFERRED_ACTION_PERMANENT_FAILED + : T.F.SYNC.S.DEFERRED_ACTION_FAILED, + actionStr: isPermanent ? T.PS.RELOAD : T.G.DISMISS, + ...(isPermanent + ? { + actionFn: (): void => { + window.location.reload(); + }, + } + : {}), config: { duration: 0, // Sticky - don't auto-dismiss critical errors }, }); - } - - if (transientFailure !== undefined) { throw new Error( 'Deferred local actions remain unpersisted after retry exhaustion.', - { cause: transientFailure }, + { cause: failure }, ); } } diff --git a/src/app/op-log/core/errors/sync-errors.ts b/src/app/op-log/core/errors/sync-errors.ts index f6f33e9002..e145490b17 100644 --- a/src/app/op-log/core/errors/sync-errors.ts +++ b/src/app/op-log/core/errors/sync-errors.ts @@ -115,13 +115,27 @@ export class UnknownSyncStateError extends Error { /** * A deferred action can never be persisted (invalid entity identifiers or an * invalid operation payload) — a deterministic condition, not a transient - * I/O failure. processDeferredActions abandons such actions instead of - * retrying them on every sync window forever. + * I/O failure. The reducer already committed, so the action stays buffered and + * sync remains blocked until reload restores the last durable state. */ export class PermanentDeferredWriteError extends Error { override name = 'PermanentDeferredWriteError'; } +/** Previously downloaded operations have not completed reducer/archive recovery. */ +export class IncompleteRemoteOperationsError extends Error { + override name = 'IncompleteRemoteOperationsError'; + + constructor(cause?: unknown) { + super( + cause instanceof Error + ? cause.message + : 'Downloaded operations are not fully applied.', + cause === undefined ? undefined : { cause }, + ); + } +} + // -----ENCRYPTION & COMPRESSION---- export class DecryptNoPasswordError extends AdditionalLogErrorBase { override name = 'DecryptNoPasswordError'; diff --git a/src/app/op-log/core/operation-log.const.ts b/src/app/op-log/core/operation-log.const.ts index 46cf13f3ac..f19d0c3fa1 100644 --- a/src/app/op-log/core/operation-log.const.ts +++ b/src/app/op-log/core/operation-log.const.ts @@ -21,7 +21,6 @@ import { InjectionToken } from '@angular/core'; * | Retention window | 7 days | COMPACTION_RETENTION_MS | Normal compaction | * | Emergency retention | 1 day | EMERGENCY_COMPACTION_RETENTION_MS | When quota exceeded | * | Clock drift warning | 5 min | CLOCK_DRIFT_THRESHOLD_MS | Warns user once per session | - * | Max conflict retries | 5 | MAX_CONFLICT_RETRY_ATTEMPTS | Then marked as rejected | * | Max concurrent resolution | 3 | MAX_CONCURRENT_RESOLUTION_ATTEMPTS | Prevents infinite merge loop | * | Post-sync cooldown | 2s | POST_SYNC_COOLDOWN_MS | Suppresses selector effects | * @@ -138,12 +137,6 @@ export const MAX_DOWNLOAD_OPS_IN_MEMORY = 50000; */ export const MAX_DOWNLOAD_ITERATIONS = 1000; -/** - * Maximum retry attempts for operations that fail during conflict resolution. - * After this many retries across restarts, the operation is marked as rejected. - */ -export const MAX_CONFLICT_RETRY_ATTEMPTS = 5; - /** * Maximum number of operations to be rejected before showing user warning. * Once this threshold is reached, user is notified about permanently failed ops. @@ -159,9 +152,9 @@ export const MAX_REJECTED_OPS_BEFORE_WARNING = 10; export const MAX_BATCH_OPERATIONS_SIZE = 50; /** - * Maximum age for pending operations before they expire and are rejected (milliseconds). + * Maximum age for pending operations before they are quarantined as failed (milliseconds). * If an operation has been pending for longer than this (e.g., due to data corruption - * or repeated crashes), it's marked as rejected instead of being replayed. + * or repeated crashes), reducer replay is skipped and archive recovery remains required. * Default: 24 hours - enough time for legitimate recovery scenarios */ export const PENDING_OPERATION_EXPIRY_MS = 24 * 60 * 60 * 1000; diff --git a/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts index aea87db639..1b42d0afd5 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts @@ -15,7 +15,6 @@ import { OperationApplierService } from '../apply/operation-applier.service'; import { HydrationStateService } from '../apply/hydration-state.service'; import { VectorClockService } from '../sync/vector-clock.service'; import { CLIENT_ID_PROVIDER, ClientIdProvider } from '../util/client-id.provider'; -import { MAX_CONFLICT_RETRY_ATTEMPTS } from '../core/operation-log.const'; import { ActionType, EntityType, Operation, OpType } from '../core/operation.types'; import { ApplyOperationsResult } from '../core/types/apply.types'; import { uuidv7 } from '../../util/uuid-v7'; @@ -25,10 +24,9 @@ import { uuidv7 } from '../../util/uuid-v7'; * OperationLogStoreService (real IndexedDB), with only the applier controlled. * * The co-located unit spec mocks the store, so it can't prove the actual status - * transitions the retry depends on: failed -> applied on success, the - * slice-from-failure re-mark, and the retry-count -> reject lifecycle that ends - * with getFailedRemoteOps filtering the op out. These tests exercise exactly - * that, end to end, across simulated reboots. + * transitions the retry depends on: failed -> applied on success and durable + * failed quarantine across repeated startup retries. These tests exercise that + * end to end, across simulated reboots. */ describe('OperationLogHydratorService retryFailedRemoteOps (integration, real store)', () => { let hydrator: OperationLogHydratorService; @@ -159,7 +157,7 @@ describe('OperationLogHydratorService retryFailedRemoteOps (integration, real st expect(stillFailed).toEqual([b.id, c.id].sort()); }); - it('eventually rejects a permanently-failing op across reboots, then stops returning it', async () => { + it('keeps a permanently-failing archive op quarantined across retries', async () => { const op = createOp(); await seedFailedRemoteOps([op]); // retryCount starts at 1 after the seed markFailed applier.applyOperations.and.callFake((toApply: Operation[]) => @@ -169,19 +167,18 @@ describe('OperationLogHydratorService retryFailedRemoteOps (integration, real st } as ApplyOperationsResult), ); - // Each call simulates one boot's retry. The seed left retryCount at 1, so - // it takes MAX_CONFLICT_RETRY_ATTEMPTS - 1 more failed retries to hit the cap. - let guard = 0; - while ((await store.getFailedRemoteOps()).length > 0 && guard < 10) { + // Each call simulates one boot's retry. It must remain visible to both the + // next startup retry and the sync safety gate instead of turning rejected. + const repeatedStartupRetries = 6; + for (let retry = 0; retry < repeatedStartupRetries; retry++) { await hydrator.retryFailedRemoteOps(); - guard++; } - // Op is gone from the retry set and persisted as rejected, not failed. - expect(await store.getFailedRemoteOps()).toEqual([]); - expect(guard).toBe(MAX_CONFLICT_RETRY_ATTEMPTS - 1); - const rejected = await store.getOpById(op.id); - expect(rejected?.rejectedAt).toBeTruthy(); - expect(rejected?.applicationStatus).toBeUndefined(); + expect((await store.getFailedRemoteOps()).map((entry) => entry.op.id)).toEqual([ + op.id, + ]); + const quarantined = await store.getOpById(op.id); + expect(quarantined?.rejectedAt).toBeUndefined(); + expect(quarantined?.applicationStatus).toBe('failed'); }); }); diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts index 815ae29a50..a207c9d916 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts @@ -27,10 +27,7 @@ import { import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { bulkApplyHydrationOperations } from '../apply/bulk-hydration.action'; import { CLIENT_ID_PROVIDER, ClientIdProvider } from '../util/client-id.provider'; -import { - MAX_CONFLICT_RETRY_ATTEMPTS, - MAX_VECTOR_CLOCK_SIZE, -} from '../core/operation-log.const'; +import { MAX_VECTOR_CLOCK_SIZE } from '../core/operation-log.const'; import { IndexedDBOpenError } from '../core/errors/indexed-db-open.error'; import { IDB_OPEN_ERROR_RELOAD_KEY } from './operation-log-hydrator.service'; import { SyncProviderId } from '../sync-providers/provider.const'; @@ -1503,10 +1500,19 @@ describe('OperationLogHydratorService', () => { expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([40]); // op-c remains archive-pending and has not consumed a retry attempt. - expect(mockOpLogStore.markFailed).toHaveBeenCalledWith( - ['op-b'], - MAX_CONFLICT_RETRY_ATTEMPTS, - ); + expect(mockOpLogStore.markFailed).toHaveBeenCalledOnceWith(['op-b']); + }); + + it('should merge clocks before marking archive retries applied', async () => { + mockOpLogStore.getFailedRemoteOps.and.resolveTo([failedEntry(40, 'op-a')]); + mockOperationApplierService.applyOperations.and.resolveTo({ + appliedOps: [failedEntry(40, 'op-a').op], + }); + mockOpLogStore.mergeRemoteOpClocks.and.rejectWith(new Error('clock write failed')); + + await expectAsync(service.retryFailedRemoteOps()).toBeRejected(); + + expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); }); it('should do nothing when there are no failed ops', async () => { diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.ts b/src/app/op-log/persistence/operation-log-hydrator.service.ts index 05b86adfe2..108d0f05a6 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.ts @@ -24,7 +24,6 @@ import { OperationApplierService } from '../apply/operation-applier.service'; import { HydrationStateService } from '../apply/hydration-state.service'; import { bulkApplyOperations } from '../apply/bulk-hydration.action'; import { VectorClockService } from '../sync/vector-clock.service'; -import { MAX_CONFLICT_RETRY_ATTEMPTS } from '../core/operation-log.const'; import { AppDataComplete } from '../model/model-config'; import { CLIENT_ID_PROVIDER, ClientIdProvider } from '../util/client-id.provider'; import { limitVectorClockSize } from '../../core/util/vector-clock'; @@ -638,7 +637,6 @@ export class OperationLogHydratorService { .map((op) => opIdToSeq.get(op.id)) .filter((seq): seq is number => seq !== undefined); if (appliedSeqs.length > 0) { - await this.opLogStore.markApplied(appliedSeqs); // The primary remote-apply path (applyRemoteOperations) merges clocks at // reducer commit for the WHOLE batch, including ops whose archive // handling later fails — so these clocks were usually merged already. @@ -646,6 +644,7 @@ export class OperationLogHydratorService { // that reached `failed`/`archive_pending` via crash recovery, where the // reducer-commit callback (and its clock merge) may never have run. await this.opLogStore.mergeRemoteOpClocks(result.appliedOps); + await this.opLogStore.markApplied(appliedSeqs); OpLog.normal( `OperationLogHydratorService: Successfully retried ${appliedSeqs.length} failed ops`, ); @@ -653,8 +652,8 @@ export class OperationLogHydratorService { // On a partial failure the batch applier stops at the first archive error. // Charge only that attempted operation: successors remain archive-pending - // without consuming retry budget and will run after the blocker succeeds or - // becomes terminally rejected. + // without consuming retry budget and will run after the blocker succeeds. + // A persistent blocker stays failed so ordinary sync remains safely paused. if (result.failedOp) { const failedOpIds = [result.failedOp.op.id]; @@ -662,7 +661,9 @@ export class OperationLogHydratorService { `OperationLogHydratorService: Failed to retry op ${result.failedOp.op.id}`, result.failedOp.error, ); - await this.opLogStore.markFailed(failedOpIds, MAX_CONFLICT_RETRY_ATTEMPTS); + // Keep archive failure visible to the sync safety gate. A retry cap that + // rejects it would hide incomplete downloaded work and allow false IN_SYNC. + await this.opLogStore.markFailed(failedOpIds); OpLog.warn( 'OperationLogHydratorService: Archive operation still failing after retry', ); diff --git a/src/app/op-log/persistence/operation-log-recovery.service.spec.ts b/src/app/op-log/persistence/operation-log-recovery.service.spec.ts index 76a9bc50fa..746c03e854 100644 --- a/src/app/op-log/persistence/operation-log-recovery.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-recovery.service.spec.ts @@ -24,12 +24,15 @@ describe('OperationLogRecoveryService', () => { 'saveStateCache', 'setVectorClock', 'getPendingRemoteOps', + 'recoverLegacyTerminalRemoteFailures', 'markRejected', + 'markFailed', 'markApplied', 'markArchivePending', 'getUnsynced', ]); mockOpLogStore.setVectorClock.and.resolveTo(undefined); + mockOpLogStore.recoverLegacyTerminalRemoteFailures.and.resolveTo(0); mockLegacyPfDb = jasmine.createSpyObj('LegacyPfDbService', [ 'hasUsableEntityData', 'loadAllEntityData', @@ -186,6 +189,7 @@ describe('OperationLogRecoveryService', () => { expect(mockOpLogStore.markArchivePending).not.toHaveBeenCalled(); expect(mockOpLogStore.markRejected).not.toHaveBeenCalled(); + expect(mockOpLogStore.markFailed).not.toHaveBeenCalled(); }); it('should mark valid crash-interrupted ops as archive-pending', async () => { @@ -203,7 +207,7 @@ describe('OperationLogRecoveryService', () => { expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); }); - it('should reject ops that exceed PENDING_OPERATION_EXPIRY_MS', async () => { + it('should quarantine ops that exceed PENDING_OPERATION_EXPIRY_MS', async () => { const now = Date.now(); const pendingOps = [ { seq: 1, op: { id: 'valid' }, appliedAt: now - 1000, source: 'remote' }, // Valid @@ -216,15 +220,15 @@ describe('OperationLogRecoveryService', () => { ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); mockOpLogStore.markArchivePending.and.resolveTo(undefined); - mockOpLogStore.markRejected.and.resolveTo(undefined); + mockOpLogStore.markFailed.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1]); - expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['expired']); + expect(mockOpLogStore.markFailed).toHaveBeenCalledWith(['expired']); }); - it('should reject all expired ops when all are superseded', async () => { + it('should quarantine all expired ops', async () => { const now = Date.now(); const expiredTime = now - PENDING_OPERATION_EXPIRY_MS - 100000; const pendingOps = [ @@ -232,11 +236,11 @@ describe('OperationLogRecoveryService', () => { { seq: 2, op: { id: 'old2' }, appliedAt: expiredTime - 1000, source: 'remote' }, ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markRejected.and.resolveTo(undefined); + mockOpLogStore.markFailed.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['old1', 'old2']); + expect(mockOpLogStore.markFailed).toHaveBeenCalledWith(['old1', 'old2']); expect(mockOpLogStore.markArchivePending).not.toHaveBeenCalled(); }); @@ -260,12 +264,12 @@ describe('OperationLogRecoveryService', () => { ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); mockOpLogStore.markArchivePending.and.resolveTo(undefined); - mockOpLogStore.markRejected.and.resolveTo(undefined); + mockOpLogStore.markFailed.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1, 3]); - expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['expired1', 'expired2']); + expect(mockOpLogStore.markFailed).toHaveBeenCalledWith(['expired1', 'expired2']); }); }); diff --git a/src/app/op-log/persistence/operation-log-recovery.service.ts b/src/app/op-log/persistence/operation-log-recovery.service.ts index 47d40c8020..b690f99373 100644 --- a/src/app/op-log/persistence/operation-log-recovery.service.ts +++ b/src/app/op-log/persistence/operation-log-recovery.service.ts @@ -141,11 +141,18 @@ export class OperationLogRecoveryService { * archive-pending checkpoint so hydration retries archive work without double-applying * reducers. * - * Operations pending for longer than PENDING_OPERATION_EXPIRY_MS are considered - * superseded (likely due to data corruption or repeated failures) and are rejected - * instead of replayed. + * Operations pending for longer than PENDING_OPERATION_EXPIRY_MS are quarantined + * as failed so sync stays blocked until archive recovery succeeds. */ async recoverPendingRemoteOps(): Promise { + const recoveredLegacyFailures = + await this.opLogStore.recoverLegacyTerminalRemoteFailures(); + if (recoveredLegacyFailures > 0) { + OpLog.warn( + `OperationLogRecoveryService: Re-quarantined ${recoveredLegacyFailures} legacy terminal remote failure(s).`, + ); + } + const pendingOps = await this.opLogStore.getPendingRemoteOps(); if (pendingOps.length === 0) { @@ -160,12 +167,12 @@ export class OperationLogRecoveryService { (e) => now - e.appliedAt >= PENDING_OPERATION_EXPIRY_MS, ); - // Reject expired ops - they've been pending too long + // Keep expired ops visible to retryFailedRemoteOps and the sync safety gate. if (expiredOps.length > 0) { const expiredIds = expiredOps.map((e) => e.op.id); - await this.opLogStore.markRejected(expiredIds); + await this.opLogStore.markFailed(expiredIds); OpLog.warn( - `OperationLogRecoveryService: Rejected ${expiredOps.length} expired pending remote ops ` + + `OperationLogRecoveryService: Quarantined ${expiredOps.length} expired pending remote ops ` + `(pending > ${PENDING_OPERATION_EXPIRY_MS / (60 * 60 * 1000)}h). ` + `Oldest was ${Math.round((now - Math.min(...expiredOps.map((e) => e.appliedAt))) / (60 * 60 * 1000))}h old.`, ); @@ -183,7 +190,7 @@ export class OperationLogRecoveryService { OpLog.normal( `OperationLogRecoveryService: Recovered ${validOps.length} pending remote ops, ` + - `rejected ${expiredOps.length} expired ops.`, + `quarantined ${expiredOps.length} expired ops.`, ); } diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index ee4b544471..40e099fb63 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -1578,17 +1578,34 @@ describe('OperationLogStoreService', () => { expect(ops[0].retryCount).toBe(3); }); - it('should mark as rejected when max retries reached', async () => { + it('should keep failed operations quarantined after repeated failures', async () => { const op = createTestOperation(); await service.append(op, 'remote', { pendingApply: true }); - await service.markFailed([op.id], 3); // maxRetries = 3 - await service.markFailed([op.id], 3); - await service.markFailed([op.id], 3); // 3rd failure = rejected + await service.markFailed([op.id]); + await service.markFailed([op.id]); + await service.markFailed([op.id]); const ops = await service.getOpsAfterSeq(0); - expect(ops[0].rejectedAt).toBeDefined(); - expect(ops[0].applicationStatus).toBeUndefined(); + expect(ops[0].rejectedAt).toBeUndefined(); + expect(ops[0].applicationStatus).toBe('failed'); + expect(ops[0].retryCount).toBe(3); + }); + + it('should re-quarantine legacy terminal remote failures', async () => { + const op = createTestOperation(); + await service.append(op, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([op.id]); + } + await service.markRejected([op.id]); + + expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(1); + + const [recovered] = await service.getFailedRemoteOps(); + expect(recovered.op.id).toBe(op.id); + expect(recovered.rejectedAt).toBeUndefined(); + expect(recovered.applicationStatus).toBe('failed'); }); it('should handle empty array', async () => { @@ -1801,7 +1818,7 @@ describe('OperationLogStoreService', () => { expect(unsynced2[0].op.id).toBe(op2.id); }); - it('should invalidate cache when markFailed terminally rejects an op', async () => { + it('should keep local failed ops unsynced after repeated failures', async () => { const op1 = createTestOperation({ entityId: 'task1' }); const op2 = createTestOperation({ entityId: 'task2' }); await service.append(op1); @@ -1810,11 +1827,10 @@ describe('OperationLogStoreService', () => { const unsynced1 = await service.getUnsynced(); expect(unsynced1.length).toBe(2); - await service.markFailed([op1.id], 1); + await service.markFailed([op1.id]); const unsynced2 = await service.getUnsynced(); - expect(unsynced2.length).toBe(1); - expect(unsynced2[0].op.id).toBe(op2.id); + expect(unsynced2.map((entry) => entry.op.id)).toEqual([op1.id, op2.id]); }); it('should not include already synced ops when incrementally updating', async () => { diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index 5bb985048c..af7ce7ef65 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -1089,14 +1089,12 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + async markFailed(opIds: string[]): Promise { await this._ensureInit(); - const now = Date.now(); - let terminallyRejected = false; await this._adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { for (const opId of opIds) { const entry = await tx.getFromIndex( @@ -1107,22 +1105,39 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort= maxRetries) { - entry.rejectedAt = now; - entry.applicationStatus = undefined; - terminallyRejected = true; - } else { - entry.applicationStatus = 'failed'; - entry.retryCount = newRetryCount; - } + entry.applicationStatus = 'failed'; + entry.retryCount = newRetryCount; await tx.put(STORE_NAMES.OPS, entry); } } }); - if (terminallyRejected) { - this._invalidateUnsyncedCache(); - } + } + + /** + * Upgrade repair for versions that terminally rejected remote archive work + * after five attempts. Those rows retained retryCount=4 while rejectedAt was + * set and applicationStatus cleared. Re-quarantine them so startup archive + * retry and the incomplete-remote sync gate can see them again. + */ + async recoverLegacyTerminalRemoteFailures(): Promise { + await this._ensureInit(); + let recoveredCount = 0; + await this._adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { + const entries = await tx.getAll(STORE_NAMES.OPS); + for (const entry of entries) { + if ( + entry.source === 'remote' && + entry.rejectedAt !== undefined && + (entry.retryCount ?? 0) >= 4 + ) { + entry.rejectedAt = undefined; + entry.applicationStatus = 'failed'; + await tx.put(STORE_NAMES.OPS, entry); + recoveredCount++; + } + } + }); + return recoveredCount; } /** diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index 1d7e93a747..f414e0d036 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -48,7 +48,6 @@ import { TranslateService } from '@ngx-translate/core'; import { T } from '../../t.const'; import { ValidateStateService } from '../validation/validate-state.service'; import { SyncSessionValidationService } from './sync-session-validation.service'; -import { MAX_CONFLICT_RETRY_ATTEMPTS } from '../core/operation-log.const'; import { compareVectorClocks, incrementVectorClock, @@ -62,6 +61,7 @@ import { uuidv7 } from '../../util/uuid-v7'; import { CURRENT_SCHEMA_VERSION } from '../persistence/schema-migration.service'; import { SYNC_LOGGER } from '../core/sync-logger.adapter'; import { processDeferredActionsAfterRemoteApply } from './process-deferred-actions-flush.util'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; /** * Represents the result of LWW (Last-Write-Wins) conflict resolution. @@ -463,7 +463,7 @@ export class ConflictResolutionService { 'Marking the attempted archive operation as failed.', applyResult.failedOp.error, ); - await this.opLogStore.markFailed(failedOpIds, MAX_CONFLICT_RETRY_ATTEMPTS); + await this.opLogStore.markFailed(failedOpIds); this.snackService.open({ type: 'ERROR', @@ -479,7 +479,7 @@ export class ConflictResolutionService { // thrown, causing sync to report IN_SYNC despite lost operations. // Deferred-actions flush runs in the finally below before the throw // propagates. - throw applyResult.failedOp.error; + throw new IncompleteRemoteOperationsError(applyResult.failedOp.error); } } diff --git a/src/app/op-log/sync/immediate-upload.service.spec.ts b/src/app/op-log/sync/immediate-upload.service.spec.ts index 15a4880743..3ac7dde023 100644 --- a/src/app/op-log/sync/immediate-upload.service.spec.ts +++ b/src/app/op-log/sync/immediate-upload.service.spec.ts @@ -27,6 +27,7 @@ describe('ImmediateUploadService', () => { permanentRejectionCount: number; hasMorePiggyback: boolean; rejectedOps: RejectedOpInfo[]; + encryptionRequiredKeyMissing: boolean; }> = {}, ): { kind: 'completed'; @@ -36,6 +37,7 @@ describe('ImmediateUploadService', () => { permanentRejectionCount: number; hasMorePiggyback: boolean; rejectedOps: RejectedOpInfo[]; + encryptionRequiredKeyMissing?: boolean; } => ({ kind: 'completed', uploadedCount: 0, @@ -136,6 +138,47 @@ describe('ImmediateUploadService', () => { expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('IN_SYNC'); })); + it('should report UNKNOWN_OR_CHANGED when the initial upload lacks a mandatory encryption key', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.resolveTo( + completedResult({ + uploadedCount: 1, + encryptionRequiredKeyMissing: true, + }), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith( + 'UNKNOWN_OR_CHANGED', + ); + expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalledWith('IN_SYNC'); + })); + + it('should report UNKNOWN_OR_CHANGED when a local-win follow-up lacks a mandatory encryption key', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.returnValues( + Promise.resolve(completedResult({ uploadedCount: 1, localWinOpsCreated: 1 })), + Promise.resolve( + completedResult({ + uploadedCount: 0, + encryptionRequiredKeyMissing: true, + }), + ), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith( + 'UNKNOWN_OR_CHANGED', + ); + expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalledWith('IN_SYNC'); + })); + it('should NOT show checkmark when piggybacked ops exist', fakeAsync(() => { mockSyncService.uploadPendingOps.and.returnValue( Promise.resolve(completedResult({ uploadedCount: 2, piggybackedOpsCount: 1 })), diff --git a/src/app/op-log/sync/immediate-upload.service.ts b/src/app/op-log/sync/immediate-upload.service.ts index 5f1c4365ca..52c7184a13 100644 --- a/src/app/op-log/sync/immediate-upload.service.ts +++ b/src/app/op-log/sync/immediate-upload.service.ts @@ -252,6 +252,7 @@ export class ImmediateUploadService implements OnDestroy { let finalResult = result; let totalUploadedCount = result.uploadedCount; let hasPermanentRejection = result.permanentRejectionCount > 0; + let encryptionRequiredKeyMissing = result.encryptionRequiredKeyMissing === true; if (result.localWinOpsCreated > 0) { OpLog.verbose( `ImmediateUploadService: LWW created ${result.localWinOpsCreated} local-win op(s), re-uploading`, @@ -274,6 +275,8 @@ export class ImmediateUploadService implements OnDestroy { finalResult = followUpResult; totalUploadedCount += followUpResult.uploadedCount; hasPermanentRejection ||= followUpResult.permanentRejectionCount > 0; + encryptionRequiredKeyMissing ||= + followUpResult.encryptionRequiredKeyMissing === true; } // Read the validation latch BEFORE any IN_SYNC / deferred-checkmark @@ -295,6 +298,11 @@ export class ImmediateUploadService implements OnDestroy { return; } + if (encryptionRequiredKeyMissing) { + this._providerManager.setSyncStatus('UNKNOWN_OR_CHANGED'); + return; + } + if ( finalResult.piggybackedOpsCount > 0 || finalResult.hasMorePiggyback || diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index ef3a282ebe..413c1d976b 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -171,8 +171,10 @@ describe('OperationLogSyncService', () => { writeFlushServiceSpy = jasmine.createSpyObj('OperationWriteFlushService', [ 'flushPendingWrites', 'flushThenRunExclusive', + 'hasPendingWrites', ]); writeFlushServiceSpy.flushPendingWrites.and.resolveTo(); + writeFlushServiceSpy.hasPendingWrites.and.returnValue(false); // Mirror the real barrier semantics: flush BEFORE the exclusive section runs. writeFlushServiceSpy.flushThenRunExclusive.and.callFake( async (fn: () => Promise) => { @@ -987,6 +989,25 @@ describe('OperationLogSyncService', () => { expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); }); + it('should flush deferred local work before entering crash-resume rebuild', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + operationLogEffectsSpy.processDeferredActions.and.rejectWith( + new Error('deferred write failed'), + ); + const forceDownloadSpy = spyOn(service, 'forceDownloadRemoteState'); + + await expectAsync( + service.downloadRemoteOps({} as OperationSyncCapable), + ).toBeRejectedWithError(/deferred write failed/); + + expect(forceDownloadSpy).not.toHaveBeenCalled(); + expect(opLogStoreSpy.isRawRebuildIncomplete).toHaveBeenCalled(); + expect(snackServiceSpy.open).toHaveBeenCalledWith( + jasmine.objectContaining({ msg: T.F.SYNC.S.LOCAL_DATA_REPLACE_UNDO }), + ); + }); + it('should offer the stranded pre-replace backup when an interrupted rebuild resume cannot finish', async () => { // Resume path: the prior attempt already committed the destructive // baseline, but this resume aborts in its download/validate phase (e.g. @@ -2806,6 +2827,7 @@ describe('OperationLogSyncService', () => { }); it('should preserve and replay local edits made after an interrupted rebuild', async () => { + const restoreOrder: string[] = []; const previouslyPreserved = makeRemoteOp('local-before-second-crash'); previouslyPreserved.clientId = 'local'; previouslyPreserved.vectorClock = { local: 2, remote: 1 }; @@ -2838,10 +2860,14 @@ describe('OperationLogSyncService', () => { writtenOps: source === 'local' ? ops : [], skippedCount: 0, })); - operationApplierSpy.applyOperations.and.callFake(async (ops) => ({ - appliedOps: ops, - })); + operationApplierSpy.applyOperations.and.callFake(async (ops) => { + restoreOrder.push('apply'); + return { appliedOps: ops }; + }); opLogStoreSpy.getVectorClock.and.resolveTo({ remote: 4 }); + opLogStoreSpy.setVectorClock.and.callFake(async () => { + restoreOrder.push('merge-clock'); + }); const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), @@ -2860,13 +2886,43 @@ describe('OperationLogSyncService', () => { ); expect(operationApplierSpy.applyOperations).toHaveBeenCalledWith( preservedLocalOps, - jasmine.objectContaining({ isLocalHydration: false }), + jasmine.objectContaining({ + isLocalHydration: false, + skipDeferredLocalActions: true, + }), ); expect(opLogStoreSpy.setVectorClock).toHaveBeenCalledWith({ remote: 4, local: 3, }); + expect(restoreOrder).toEqual(['merge-clock', 'apply']); expect(opLogStoreSpy.clearRawRebuildIncomplete).toHaveBeenCalled(); + expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ + callerHoldsOperationLogLock: true, + }); + }); + + it('should keep crash recovery armed when a local capture arrives during rebuild', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 50, + }); + writeFlushServiceSpy.hasPendingWrites.and.returnValue(true); + const setLastServerSeq = jasmine.createSpy('setLastServerSeq').and.resolveTo(); + + await expectAsync( + service.forceDownloadRemoteState({ + supportsOperationSync: true, + setLastServerSeq, + } as unknown as OperationSyncCapable), + ).toBeRejectedWithError(/local change arrived/); + + expect(setLastServerSeq).not.toHaveBeenCalledWith(50); + expect(opLogStoreSpy.clearRawRebuildIncomplete).not.toHaveBeenCalled(); }); it('should offer to restore the previous data after replacing (#8107)', async () => { diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index f6d453f425..026ee8a7f3 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -21,7 +21,10 @@ import { DownloadOutcome, UploadOutcome } from '../core/types/sync-results.types import { OperationLogDownloadService } from './operation-log-download.service'; import { SnackService } from '../../core/snack/snack.service'; import { T } from '../../t.const'; -import { LocalDataConflictError } from '../core/errors/sync-errors'; +import { + IncompleteRemoteOperationsError, + LocalDataConflictError, +} from '../core/errors/sync-errors'; import { SuperSyncStatusService } from './super-sync-status.service'; import { ServerMigrationService } from './server-migration.service'; import { OperationWriteFlushService } from './operation-write-flush.service'; @@ -457,6 +460,10 @@ export class OperationLogSyncService { 'OperationLogSyncService: Interrupted USE_REMOTE rebuild detected — redoing the raw rebuild.', ); try { + // Flush belongs inside the recovery boundary: if persistence is still + // unhealthy, the pre-replace backup is the only safe rollback and its + // Undo affordance must remain visible. + await this._flushLocalWritesIncludingDeferredActions(); await this.forceDownloadRemoteState(syncProvider, { isCrashResume: true }); } catch (e) { // The prior attempt already committed the destructive baseline (that is @@ -592,11 +599,9 @@ export class OperationLogSyncService { const exampleTaskOpIds = exampleTaskEntries.map((entry) => entry.op.id); // Nothing from this sync is persisted yet, so this live read reflects // whether the client completed a prior sync cycle. - const hasCompletedInitialSync = await this.opLogStore.hasSyncedOps(); const hasMeaningfulUserData = - this.syncImportConflictGateService.hasMeaningfulPendingOps(unsyncedOps, { - hasCompletedInitialSync, - }) || this.syncLocalStateService.hasMeaningfulStoreData(exampleTaskIds); + this.syncImportConflictGateService.hasMeaningfulPendingOps(unsyncedOps) || + this.syncLocalStateService.hasMeaningfulStoreData(exampleTaskIds); if (hasMeaningfulUserData) { // SPAP-9: before surfacing the binary USE_LOCAL/USE_REMOTE dialog, use @@ -1437,14 +1442,14 @@ export class OperationLogSyncService { await this._restorePreservedLocalOps(preservedLocalOps); + await this._assertNoCaptureRacedWithRebuild(); + // Update lastServerSeq after hydration if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); } - // Replay committed — the rebuild is complete and crash-resume is - // no longer needed. - await this.opLogStore.clearRawRebuildIncomplete(); + await this._completeRawRebuild(); OpLog.normal( 'OperationLogSyncService: Force download snapshot hydration complete.', @@ -1489,14 +1494,14 @@ export class OperationLogSyncService { await this._restorePreservedLocalOps(preservedLocalOps); + await this._assertNoCaptureRacedWithRebuild(); + // Update lastServerSeq if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); } - // Replay committed — the rebuild is complete and crash-resume is - // no longer needed. - await this.opLogStore.clearRawRebuildIncomplete(); + await this._completeRawRebuild(); OpLog.normal( `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, @@ -1544,12 +1549,23 @@ export class OperationLogSyncService { this.opLogStore.getFailedRemoteOps(), ]); if (pendingRemoteOps.length > 0 || failedRemoteOps.length > 0) { + throw new IncompleteRemoteOperationsError(); + } + } + + private _assertNoCaptureRacedWithRebuild(): void { + if (this.writeFlushService.hasPendingWrites()) { throw new Error( - 'Sync blocked because previously downloaded operations are not fully applied. Restart the app to retry recovery.', + 'USE_REMOTE incomplete: a local change arrived during the rebuild and will be restored on retry.', ); } } + private async _completeRawRebuild(): Promise { + this._assertNoCaptureRacedWithRebuild(); + await this.opLogStore.clearRawRebuildIncomplete(); + } + /** * Replays edits captured after an interrupted rebuild on top of the complete * authoritative history. They stay local/unsynced so the next upload carries @@ -1568,11 +1584,18 @@ export class OperationLogSyncService { return; } + let restoredClock = (await this.opLogStore.getVectorClock()) ?? {}; + for (const op of writtenOps) { + restoredClock = mergeVectorClocks(restoredClock, op.vectorClock); + } + await this.opLogStore.setVectorClock(restoredClock); + const applyResult = await this.operationApplier.applyOperations(writtenOps, { // The authoritative replacement also overwrote archive IndexedDB stores, // so archive-affecting post-crash edits must replay their side effects. // The entries themselves remain source=local and unsynced in the op-log. isLocalHydration: false, + skipDeferredLocalActions: true, }); if (applyResult.failedOp || applyResult.appliedOps.length !== writtenOps.length) { throw new Error( @@ -1580,11 +1603,7 @@ export class OperationLogSyncService { ); } - let restoredClock = (await this.opLogStore.getVectorClock()) ?? {}; - for (const op of writtenOps) { - restoredClock = mergeVectorClocks(restoredClock, op.vectorClock); - } - await this.opLogStore.setVectorClock(restoredClock); + await processDeferredActions(this.injector, true); } private _preflightRemoteOperations(remoteOps: Operation[]): Operation[] { diff --git a/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts b/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts index 11a1badebb..07159e4179 100644 --- a/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts +++ b/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts @@ -33,6 +33,7 @@ import type { OperationSyncCapable, SyncOperation, } from '../sync-providers/provider.interface'; +import { OperationLogEffects } from '../capture/operation-log.effects'; /** * #8304 cross-service integration test. @@ -147,6 +148,8 @@ describe('OperationLogSyncService + OperationLogUploadService — piggyback seq opLogStoreSpy = jasmine.createSpyObj('OperationLogStoreService', [ 'getUnsynced', + 'getPendingRemoteOps', + 'getFailedRemoteOps', 'markSynced', 'markRejected', 'loadStateCache', @@ -160,6 +163,8 @@ describe('OperationLogSyncService + OperationLogUploadService — piggyback seq 'deleteOpsWhere', ]); opLogStoreSpy.getUnsynced.and.resolveTo([localPendingEntry]); + opLogStoreSpy.getPendingRemoteOps.and.resolveTo([]); + opLogStoreSpy.getFailedRemoteOps.and.resolveTo([]); opLogStoreSpy.markSynced.and.resolveTo(undefined); opLogStoreSpy.markRejected.and.resolveTo(undefined); opLogStoreSpy.setVectorClock.and.resolveTo(); @@ -261,6 +266,12 @@ describe('OperationLogSyncService + OperationLogUploadService — piggyback seq { provide: StateSnapshotService, useValue: stateSnapshotServiceSpy }, { provide: RejectedOpsHandlerService, useValue: rejectedOpsHandlerServiceSpy }, { provide: OperationWriteFlushService, useValue: writeFlushServiceSpy }, + { + provide: OperationLogEffects, + useValue: jasmine.createSpyObj('OperationLogEffects', { + processDeferredActions: Promise.resolve(), + }), + }, { provide: SuperSyncStatusService, useValue: superSyncStatusServiceSpy }, { provide: SnackService, diff --git a/src/app/op-log/sync/operation-write-flush.service.spec.ts b/src/app/op-log/sync/operation-write-flush.service.spec.ts index a868d129a8..965863eb94 100644 --- a/src/app/op-log/sync/operation-write-flush.service.spec.ts +++ b/src/app/op-log/sync/operation-write-flush.service.spec.ts @@ -29,6 +29,13 @@ describe('OperationWriteFlushService', () => { }); describe('flushPendingWrites', () => { + it('should expose whether reducer captures are pending', () => { + captureServiceSpy.getPendingCount.and.returnValues(1, 0); + + expect(service.hasPendingWrites()).toBeTrue(); + expect(service.hasPendingWrites()).toBeFalse(); + }); + it('should acquire the sp_op_log lock', async () => { await service.flushPendingWrites(); diff --git a/src/app/op-log/sync/operation-write-flush.service.ts b/src/app/op-log/sync/operation-write-flush.service.ts index c3d6ab1bb4..14a8fdbde5 100644 --- a/src/app/op-log/sync/operation-write-flush.service.ts +++ b/src/app/op-log/sync/operation-write-flush.service.ts @@ -52,6 +52,11 @@ export class OperationWriteFlushService { */ private readonly POLL_INTERVAL = 10; + /** Whether a reducer action is still waiting to become durable. */ + hasPendingWrites(): boolean { + return this.captureService.getPendingCount() > 0; + } + /** * Waits for all pending operation writes to complete. * diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index 2c77b7527f..57c3f36ea4 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -51,6 +51,14 @@ describe('RemoteOpsProcessingService', () => { let syncImportFilterServiceSpy: jasmine.SpyObj; let operationLogEffectsSpy: jasmine.SpyObj; + const applyAllWithReducerCommit = async ( + ops: Operation[], + options?: Parameters[1], + ): ReturnType => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }; + beforeEach(() => { storeSpy = jasmine.createSpyObj('Store', ['select']); storeSpy.select.and.returnValue( @@ -286,9 +294,7 @@ describe('RemoteOpsProcessingService', () => { // Default: no local ops to replay after SYNC_IMPORT opLogStoreSpy.getOpsAfterSeq.and.returnValue(Promise.resolve([])); // Default: successful operation application - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); operationLogEffectsSpy.processDeferredActions.and.resolveTo(); }); @@ -331,9 +337,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); opLogStoreSpy.markApplied.and.returnValue(Promise.resolve()); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [remoteOps[0]] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); // Track call order const callOrder: string[] = []; @@ -707,9 +711,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [syncImportOp] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); // Spy on detectConflicts - it should NOT be called spyOn(service, 'detectConflicts').and.callThrough(); @@ -742,9 +744,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.getUnsynced.and.resolveTo([ { seq: 99, op: { opType: OpType.Update } as any, appliedAt: 0, source: 'local' }, ]); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [syncImportOp] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); const opLogSpy = spyOn(OpLog, 'log').and.callThrough(); await service.processRemoteOps([syncImportOp]); @@ -804,8 +804,9 @@ describe('RemoteOpsProcessingService', () => { let appliedOps: Operation[] = []; opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); - operationApplierServiceSpy.applyOperations.and.callFake(async (ops) => { + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { appliedOps = ops; + await options?.onReducersCommitted?.(ops); return { appliedOps: ops }; }); @@ -864,9 +865,7 @@ describe('RemoteOpsProcessingService', () => { }; opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); - operationApplierServiceSpy.applyOperations.and.callFake(async (ops) => ({ - appliedOps: ops, - })); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); await service.processRemoteOps([syncImportOp]); @@ -901,9 +900,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [backupImportOp] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); // Spy on detectConflicts - it should NOT be called spyOn(service, 'detectConflicts').and.callThrough(); @@ -938,9 +935,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); opLogStoreSpy.clearFullStateOpsExcept.and.returnValue(Promise.resolve(2)); // Had 2 old ops - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [syncImportOp] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); await service.processRemoteOps([syncImportOp]); @@ -971,9 +966,7 @@ describe('RemoteOpsProcessingService', () => { vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [regularOp] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); await service.processRemoteOps([regularOp]); @@ -1162,8 +1155,8 @@ describe('RemoteOpsProcessingService', () => { vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [validOp] }), + operationApplierServiceSpy.applyOperations.and.callFake( + applyAllWithReducerCommit, ); const remoteOps = [ @@ -1198,8 +1191,8 @@ describe('RemoteOpsProcessingService', () => { vectorClockServiceSpy.getSnapshotVectorClock.and.returnValue(Promise.resolve({})); opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [] }), + operationApplierServiceSpy.applyOperations.and.callFake( + applyAllWithReducerCommit, ); const result = await service.processRemoteOps([createFullOp({ id: 'op-1' })]); @@ -1511,9 +1504,12 @@ describe('RemoteOpsProcessingService', () => { ]; opLogStoreSpy.markFailed.and.resolveTo(); - operationApplierServiceSpy.applyOperations.and.resolveTo({ - appliedOps: [remoteOps[0]], - failedOp: { op: remoteOps[1], error: new Error('Test error') }, + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { + appliedOps: [remoteOps[0]], + failedOp: { op: remoteOps[1], error: new Error('Test error') }, + }; }); validateStateServiceSpy.validateAndRepairCurrentState.and.resolveTo(false); @@ -1554,8 +1550,8 @@ describe('RemoteOpsProcessingService', () => { skippedCount: 1, }), ); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [remoteOps[1]] }), + operationApplierServiceSpy.applyOperations.and.callFake( + applyAllWithReducerCommit, ); await service.applyNonConflictingOps(remoteOps); @@ -1676,9 +1672,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); opLogStoreSpy.markApplied.and.returnValue(Promise.resolve()); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [remoteOps[0]] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); spyOn(service, 'detectConflicts').and.resolveTo({ nonConflicting: remoteOps, conflicts: [], @@ -1707,9 +1701,7 @@ describe('RemoteOpsProcessingService', () => { opLogStoreSpy.hasOp.and.returnValue(Promise.resolve(false)); opLogStoreSpy.append.and.returnValue(Promise.resolve(1)); opLogStoreSpy.markApplied.and.returnValue(Promise.resolve()); - operationApplierServiceSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [remoteOps[0]] }), - ); + operationApplierServiceSpy.applyOperations.and.callFake(applyAllWithReducerCommit); spyOn(service, 'detectConflicts').and.resolveTo({ nonConflicting: remoteOps, conflicts: [], diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index 8d64f02ad2..5ee226c2bf 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -31,6 +31,7 @@ import { OperationLogCompactionService } from '../persistence/operation-log-comp import { SyncImportFilterService } from './sync-import-filter.service'; import { OperationWriteFlushService } from './operation-write-flush.service'; import { processDeferredActionsAfterRemoteApply } from './process-deferred-actions-flush.util'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; import { selectSyncConfig } from '../../features/config/store/global-config.reducer'; import { applyLocalOnlySyncSettingsToAppData, @@ -502,7 +503,7 @@ export class RemoteOpsProcessingService { // The deferred-actions flush in the finally below runs before the // throw propagates. - throw result.failedOp.error; + throw new IncompleteRemoteOperationsError(result.failedOp.error); } } finally { if (didApplyRemoteOps) { diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts index 860e258f09..30d1a2fdcc 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts @@ -118,14 +118,14 @@ describe('SyncImportConflictGateService', () => { expect(result.dialogData).toBeDefined(); }); - it('should NOT flag a pending config change on a never-synced client', async () => { + it('should protect a pending sync config change on a never-synced client', async () => { opLogStoreSpy.hasSyncedOps.and.resolveTo(false); opLogStoreSpy.getUnsynced.and.resolveTo([createPendingConfigEntry()]); const result = await service.checkIncomingFullStateConflict([createOperation()]); - expect(result.hasMeaningfulPending).toBeFalse(); - expect(result.dialogData).toBeUndefined(); + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); }); it('should flag a user config change on a never-synced client', async () => { @@ -442,13 +442,8 @@ describe('SyncImportConflictGateService', () => { { seq: 2 }, ); - const synced = { hasCompletedInitialSync: true }; - expect(service.hasMeaningfulPendingOps([pendingMigration], synced)).toBeTrue(); - expect(service.hasMeaningfulPendingOps([pendingRecovery], synced)).toBeTrue(); - - const neverSynced = { hasCompletedInitialSync: false }; - expect(service.hasMeaningfulPendingOps([pendingMigration], neverSynced)).toBeTrue(); - expect(service.hasMeaningfulPendingOps([pendingRecovery], neverSynced)).toBeTrue(); + expect(service.hasMeaningfulPendingOps([pendingMigration])).toBeTrue(); + expect(service.hasMeaningfulPendingOps([pendingRecovery])).toBeTrue(); }); }); diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index a0a1f920b2..1b31481a8a 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -2,7 +2,6 @@ import { inject, Injectable } from '@angular/core'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { FULL_STATE_OP_TYPES, - extractActionPayload, Operation, OperationLogEntry, OpType, @@ -35,20 +34,14 @@ export class SyncImportConflictGateService { private writeFlushService = inject(OperationWriteFlushService); /** - * Every pending op is user work unless it is an onboarding example-task create, - * or the sync-config write required to enable first sync on a fresh client. + * Every pending op is user work unless it is an onboarding example-task create. * Full-state ops are always meaningful because applying a newer full-state op * can invalidate their local import/repair semantics. * * The lifecycle default is deliberately conservative: a caller that does not * know whether initial sync completed must protect startup-entity changes. */ - hasMeaningfulPendingOps( - ops: OperationLogEntry[], - options: { hasCompletedInitialSync: boolean } = { - hasCompletedInitialSync: true, - }, - ): boolean { + hasMeaningfulPendingOps(ops: OperationLogEntry[]): boolean { return ops.some((entry) => { if (FULL_STATE_OP_TYPES.has(entry.op.opType as OpType)) { return true; @@ -56,13 +49,6 @@ export class SyncImportConflictGateService { if (isExampleTaskCreateOp(entry)) { return false; } - if ( - !options.hasCompletedInitialSync && - entry.op.entityType === 'GLOBAL_CONFIG' && - extractActionPayload(entry.op.payload).sectionKey === 'sync' - ) { - return false; - } return true; }); } @@ -123,14 +109,9 @@ export class SyncImportConflictGateService { }; } - // Capture lifecycle state before deciding whether startup writes are setup - // noise or post-sync divergence. Piggyback callers pass their pre-sync snapshot - // because a live read there already includes this sync cycle's writes. const isNeverSynced = options.isNeverSynced ?? !(await this.opLogStore.hasSyncedOps()); - const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps, { - hasCompletedInitialSync: !isNeverSynced, - }); + const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); const result = { fullStateOp, diff --git a/src/app/op-log/sync/ws-triggered-download.service.spec.ts b/src/app/op-log/sync/ws-triggered-download.service.spec.ts index df86aa1f47..7014500853 100644 --- a/src/app/op-log/sync/ws-triggered-download.service.spec.ts +++ b/src/app/op-log/sync/ws-triggered-download.service.spec.ts @@ -12,6 +12,9 @@ import { SyncSessionValidationService } from './sync-session-validation.service' import { SyncCycleGuardService } from './sync-cycle-guard.service'; import { SyncWrapperService } from '../../imex/sync/sync-wrapper.service'; import { AuthFailSPError, MissingCredentialsSPError } from '../sync-exports'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; +import { SnackService } from '../../core/snack/snack.service'; +import { T } from '../../t.const'; describe('WsTriggeredDownloadService', () => { let service: WsTriggeredDownloadService; @@ -21,6 +24,7 @@ describe('WsTriggeredDownloadService', () => { let mockProviderManager: jasmine.SpyObj; let mockWrappedProvider: jasmine.SpyObj; let mockSyncWrapper: { isEncryptionOperationInProgress: boolean }; + let mockSnackService: jasmine.SpyObj; let syncCapableProvider: any; beforeEach(() => { @@ -57,6 +61,7 @@ describe('WsTriggeredDownloadService', () => { // off it lazily (via Injector, to avoid a DI cycle). Provide a minimal mock so // the real (heavily-dependent) service is never constructed in the unit test. mockSyncWrapper = { isEncryptionOperationInProgress: false }; + mockSnackService = jasmine.createSpyObj('SnackService', ['open']); TestBed.configureTestingModule({ providers: [ @@ -66,6 +71,7 @@ describe('WsTriggeredDownloadService', () => { { provide: SyncProviderManager, useValue: mockProviderManager }, { provide: WrappedProviderService, useValue: mockWrappedProvider }, { provide: SyncWrapperService, useValue: mockSyncWrapper }, + { provide: SnackService, useValue: mockSnackService }, ], }); @@ -227,6 +233,24 @@ describe('WsTriggeredDownloadService', () => { expect(mockSyncService.downloadRemoteOps).toHaveBeenCalledTimes(2); })); + it('should report incomplete remote application as a sticky translated error', fakeAsync(() => { + mockSyncService.downloadRemoteOps.and.rejectWith( + new IncompleteRemoteOperationsError(new Error('archive failed')), + ); + + service.start(); + notification$.next({ latestSeq: 1 }); + tick(500); + flushMicrotasks(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).toHaveBeenCalledWith({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + })); + it('should be idempotent when start is called twice', fakeAsync(() => { service.start(); service.start(); diff --git a/src/app/op-log/sync/ws-triggered-download.service.ts b/src/app/op-log/sync/ws-triggered-download.service.ts index a2ecfed4a7..762cb7fd22 100644 --- a/src/app/op-log/sync/ws-triggered-download.service.ts +++ b/src/app/op-log/sync/ws-triggered-download.service.ts @@ -11,6 +11,9 @@ import { SyncWrapperService } from '../../imex/sync/sync-wrapper.service'; import { lazyInject } from '../../util/lazy-inject'; import { SyncLog } from '../../core/log'; import { AuthFailSPError, MissingCredentialsSPError } from '../sync-exports'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; +import { SnackService } from '../../core/snack/snack.service'; +import { T } from '../../t.const'; const WS_DOWNLOAD_DEBOUNCE_MS = 500; @@ -32,6 +35,7 @@ export class WsTriggeredDownloadService implements OnDestroy { private _wrappedProvider = inject(WrappedProviderService); private _sessionValidation = inject(SyncSessionValidationService); private _syncCycleGuard = inject(SyncCycleGuardService); + private _snackService = inject(SnackService); private _injector = inject(Injector); // Resolved lazily to break the DI cycle: SyncWrapperService injects this @@ -166,6 +170,19 @@ export class WsTriggeredDownloadService implements OnDestroy { this._providerManager.setSyncStatus('ERROR'); } } catch (err) { + if (err instanceof IncompleteRemoteOperationsError) { + SyncLog.err( + 'WsTriggeredDownloadService: Remote operation application is incomplete', + err, + ); + this._providerManager.setSyncStatus('ERROR'); + this._snackService.open({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + return; + } if (err instanceof AuthFailSPError || err instanceof MissingCredentialsSPError) { SyncLog.warn('WsTriggeredDownloadService: Auth failure during download', err); this.stop(); diff --git a/src/app/op-log/testing/integration/migration-handling.integration.spec.ts b/src/app/op-log/testing/integration/migration-handling.integration.spec.ts index f6d303695c..09067ad346 100644 --- a/src/app/op-log/testing/integration/migration-handling.integration.spec.ts +++ b/src/app/op-log/testing/integration/migration-handling.integration.spec.ts @@ -34,9 +34,10 @@ describe('Migration Handling Integration', () => { operationApplierSpy = jasmine.createSpyObj('OperationApplierService', [ 'applyOperations', ]); - operationApplierSpy.applyOperations.and.returnValue( - Promise.resolve({ appliedOps: [] }), - ); + operationApplierSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); TestBed.configureTestingModule({ providers: [ @@ -197,12 +198,15 @@ describe('Migration Handling Integration', () => { const op = createOp('op-fail'); // Make applier return failure result (new behavior with partial success support) - operationApplierSpy.applyOperations.and.resolveTo({ - appliedOps: [], - failedOp: { - op, - error: new Error('Simulated Apply Error'), - }, + operationApplierSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { + appliedOps: [], + failedOp: { + op, + error: new Error('Simulated Apply Error'), + }, + }; }); // Spy on store to verify markFailed is called diff --git a/src/app/op-log/testing/integration/service-logic.integration.spec.ts b/src/app/op-log/testing/integration/service-logic.integration.spec.ts index bfcf61cf9a..fea41b000e 100644 --- a/src/app/op-log/testing/integration/service-logic.integration.spec.ts +++ b/src/app/op-log/testing/integration/service-logic.integration.spec.ts @@ -316,7 +316,10 @@ describe('Service Logic Integration', () => { }, ); applierSpy = jasmine.createSpyObj('OperationApplierService', ['applyOperations']); - applierSpy.applyOperations.and.returnValue(Promise.resolve({ appliedOps: [] })); + applierSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); // Create spy properly before using in TestBed const waitServiceSpy = jasmine.createSpyObj('UserInputWaitStateService', [ diff --git a/src/app/t.const.ts b/src/app/t.const.ts index b795c8e45b..7701c9ebc3 100644 --- a/src/app/t.const.ts +++ b/src/app/t.const.ts @@ -1597,6 +1597,7 @@ const T = { DATA_REPAIRED: 'F.SYNC.S.DATA_REPAIRED', DECRYPTION_FAILED: 'F.SYNC.S.DECRYPTION_FAILED', DEFERRED_ACTION_FAILED: 'F.SYNC.S.DEFERRED_ACTION_FAILED', + DEFERRED_ACTION_PERMANENT_FAILED: 'F.SYNC.S.DEFERRED_ACTION_PERMANENT_FAILED', DISABLE_ENCRYPTION_FAILED: 'F.SYNC.S.DISABLE_ENCRYPTION_FAILED', ENABLE_ENCRYPTION_FAILED: 'F.SYNC.S.ENABLE_ENCRYPTION_FAILED', ENCRYPTION_DISABLED_ON_OTHER_DEVICE: @@ -1620,6 +1621,7 @@ const T = { IMPORTING: 'F.SYNC.S.IMPORTING', INCOMPLETE_CFG: 'F.SYNC.S.INCOMPLETE_CFG', INCOMPLETE_OP_SYNC: 'F.SYNC.S.INCOMPLETE_OP_SYNC', + INCOMPLETE_REMOTE_OPERATIONS: 'F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS', INITIAL_SYNC_ERROR: 'F.SYNC.S.INITIAL_SYNC_ERROR', INTEGRITY_CHECK_FAILED: 'F.SYNC.S.INTEGRITY_CHECK_FAILED', INVALID_AUTH_CODE: 'F.SYNC.S.INVALID_AUTH_CODE', diff --git a/src/assets/i18n/en.json b/src/assets/i18n/en.json index 6ced86a079..6e39ab4cee 100644 --- a/src/assets/i18n/en.json +++ b/src/assets/i18n/en.json @@ -1549,7 +1549,8 @@ "CONFLICT_RESOLUTION_FAILED": "Sync conflict resolution failed. Please reload.", "DATA_REPAIRED": "Sync cleanup: {{count}} reference(s) resolved", "DECRYPTION_FAILED": "Failed to decrypt synced data. Please check your encryption password.", - "DEFERRED_ACTION_FAILED": "Some changes made during sync could not be saved yet. Sync will retry automatically; keep the app open.", + "DEFERRED_ACTION_FAILED": "Some changes made during sync could not be saved yet. The next sync will retry; keep the app open.", + "DEFERRED_ACTION_PERMANENT_FAILED": "A change could not be saved and sync is paused. Reload to restore the last saved state.", "DISABLE_ENCRYPTION_FAILED": "Failed to disable encryption: {{message}}", "ENABLE_ENCRYPTION_FAILED": "Failed to enable encryption: {{message}}", "ENCRYPTION_DISABLED_ON_OTHER_DEVICE": "Encryption was disabled on another device. Your sync settings have been updated.", @@ -1572,6 +1573,7 @@ "IMPORTING": "Importing data", "INCOMPLETE_CFG": "Sync credentials are missing. Please configure sync.", "INCOMPLETE_OP_SYNC": "Sync incomplete: {{count}} operation file(s) failed to download", + "INCOMPLETE_REMOTE_OPERATIONS": "Downloaded changes could not finish applying. Sync is paused to protect your data. Restart the app to retry. If this keeps happening, restore a backup.", "INITIAL_SYNC_ERROR": "Initial Sync failed", "INTEGRITY_CHECK_FAILED": "Data integrity issue detected. Auto-repair attempted. If you experience issues, please reload the app.", "INVALID_AUTH_CODE": "The authorization code was rejected. Please try authenticating again and make sure to copy the code exactly.", From aa6e74bd347a274923cb4032fc5ee9d1d559bf72 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:35:05 +0200 Subject: [PATCH 27/47] fix(supersync): preserve occupied ids during cleanup --- .../sync/services/operation-upload.service.ts | 34 ++++++++++++-- .../src/sync/sync.routes.ops-handler.ts | 20 ++++++--- .../src/sync/sync.routes.quota.ts | 20 ++++++--- .../src/sync/sync.service.ts | 3 ++ .../tests/sync-compressed-body.routes.spec.ts | 44 ++++++++++++++----- .../tests/sync.service.spec.ts | 32 ++++++++++++++ 6 files changed, 124 insertions(+), 29 deletions(-) diff --git a/packages/super-sync-server/src/sync/services/operation-upload.service.ts b/packages/super-sync-server/src/sync/services/operation-upload.service.ts index 304997b93d..5b7750ae4c 100644 --- a/packages/super-sync-server/src/sync/services/operation-upload.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-upload.service.ts @@ -190,6 +190,7 @@ export class OperationUploadService { now: number, tx: Prisma.TransactionClient, prevalidatedResults?: ReadonlyMap, + requestStartOccupiedIds?: ReadonlySet, ): Promise<{ results: UploadResult[]; acceptedDeltaBytes: number; @@ -235,6 +236,7 @@ export class OperationUploadService { uniqueCandidates, tx, results, + requestStartOccupiedIds, ); dbRoundtrips += classified.dbRoundtrips; const duplicateFreeCandidates = classified.duplicateFreeCandidates; @@ -363,10 +365,12 @@ export class OperationUploadService { } /** - * Stage 3: prefetch any already-persisted ops sharing an incoming id (one + * Stage 3: prefetch any currently persisted ops sharing an incoming id (one * query) and classify each as an idempotent retry (DUPLICATE_OPERATION) or - * an id collision with different content (INVALID_OP_ID). Survivors are - * returned for conflict detection. + * an id collision with different content (INVALID_OP_ID). If quota cleanup + * removed a row after the route's occupancy check, the request-start set + * still rejects that ID rather than letting it consume unestimated storage. + * Survivors are returned for conflict detection. */ private async classifyExistingDuplicates( userId: number, @@ -374,6 +378,7 @@ export class OperationUploadService { uniqueCandidates: BatchUploadCandidate[], tx: Prisma.TransactionClient, results: UploadResult[], + requestStartOccupiedIds?: ReadonlySet, ): Promise<{ duplicateFreeCandidates: BatchUploadCandidate[]; dbRoundtrips: number; @@ -390,6 +395,16 @@ export class OperationUploadService { for (const candidate of uniqueCandidates) { const existingOp = existingOpById.get(candidate.op.id); if (!existingOp) { + if (requestStartOccupiedIds?.has(candidate.op.id)) { + results[candidate.resultIndex] = this.rejectedUploadResult( + userId, + clientId, + candidate.op, + 'Operation ID was already occupied before quota enforcement', + SYNC_ERROR_CODES.INVALID_OP_ID, + ); + continue; + } duplicateFreeCandidates.push(candidate); continue; } @@ -618,6 +633,7 @@ export class OperationUploadService { now: number, tx: Prisma.TransactionClient, prevalidatedResult?: ValidationResult, + wasOccupiedAtRequestStart?: boolean, ): Promise { // Rejected ops have no storage cost; the caller only reads storageBytes when // result.accepted is true. @@ -718,6 +734,18 @@ export class OperationUploadService { }); } + if (wasOccupiedAtRequestStart) { + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + 'Operation ID was already occupied before quota enforcement', + SYNC_ERROR_CODES.INVALID_OP_ID, + ), + ); + } + // Check for conflicts with existing operations const conflict = await detectConflict(userId, op, tx); if (conflict.hasConflict) { diff --git a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts index ccef9f2961..29f4e15c3a 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts @@ -190,18 +190,22 @@ export const uploadOpsHandler = async ( // Check storage quota before processing (after dedup to allow retries). // Account using the same per-op payload+vectorClock measure that the // post-accept counter increment uses, so the gate and the increment - // cannot disagree on what "size" means. Already-stored exact - // duplicates are rejected by uploadOps and never written, so don't make - // quota cleanup reserve space for them. + // cannot disagree on what "size" means. Already-occupied IDs are + // rejected by uploadOps and never written, so don't make quota cleanup + // reserve space for them. Carry the occupied-ID snapshot through upload + // so cleanup cannot make one of those IDs insertable mid-request. const typedOpsForGate = ops as unknown as Operation[]; const validOpsForQuota = syncService.filterValidOpsForQuota( typedOpsForGate, clientId, ); - const { bytes: estimatedDelta, fallback: gateFallback } = - await computeOpsStorageBytesExcludingUnstorableIds(validOpsForQuota, (op) => - syncService.getPrevalidatedPayloadBytes(op), - ); + const { + bytes: estimatedDelta, + fallback: gateFallback, + requestStartOccupiedIds, + } = await computeOpsStorageBytesExcludingUnstorableIds(validOpsForQuota, (op) => + syncService.getPrevalidatedPayloadBytes(op), + ); if (gateFallback > 0) { Logger.warn( `computeOpsStorageBytes: ${gateFallback}/${validOpsForQuota.length} unserializable op(s) ` + @@ -220,6 +224,8 @@ export const uploadOpsHandler = async ( userId, clientId, ops as unknown as Operation[], + undefined, + requestStartOccupiedIds, ); return uploadResults; diff --git a/packages/super-sync-server/src/sync/sync.routes.quota.ts b/packages/super-sync-server/src/sync/sync.routes.quota.ts index ecf3361619..ef0649b75d 100644 --- a/packages/super-sync-server/src/sync/sync.routes.quota.ts +++ b/packages/super-sync-server/src/sync/sync.routes.quota.ts @@ -3,7 +3,7 @@ import { prisma } from '../db'; import { Logger } from '../logger'; import { getSyncService } from './sync.service'; import { computeOpStorageBytes } from './sync.const'; -import { DUPLICATE_OP_SELECT, Operation, SYNC_ERROR_CODES } from './sync.types'; +import { Operation, SYNC_ERROR_CODES } from './sync.types'; import { errorMessage, MAX_OPS_PER_BATCH } from './sync.routes.payload'; /** @@ -34,14 +34,20 @@ export const computeOpsStorageBytes = ( export const computeOpsStorageBytesExcludingUnstorableIds = async ( ops: Operation[], getCachedPayloadBytes?: (op: Operation) => number | undefined, -): Promise<{ bytes: number; fallback: number }> => { - if (ops.length === 0) return { bytes: 0, fallback: 0 }; +): Promise<{ + bytes: number; + fallback: number; + requestStartOccupiedIds: ReadonlySet; +}> => { + if (ops.length === 0) { + return { bytes: 0, fallback: 0, requestStartOccupiedIds: new Set() }; + } const existingOps = await prisma.operation.findMany({ where: { id: { in: Array.from(new Set(ops.map((op) => op.id))) } }, - select: DUPLICATE_OP_SELECT, + select: { id: true }, }); - const occupiedIds = new Set(existingOps.map((existingOp) => existingOp.id)); + const requestStartOccupiedIds = new Set(existingOps.map((existingOp) => existingOp.id)); let bytes = 0; let fallback = 0; @@ -50,7 +56,7 @@ export const computeOpsStorageBytesExcludingUnstorableIds = async ( // uploadOps stores at most the first new operation for an ID. Any occupied // ID (exact retry or collision) and every repeated incoming ID is terminally // rejected, so charging it here can only poison the pre-write quota gate. - if (occupiedIds.has(op.id) || seenIncomingIds.has(op.id)) { + if (requestStartOccupiedIds.has(op.id) || seenIncomingIds.has(op.id)) { continue; } seenIncomingIds.add(op.id); @@ -59,7 +65,7 @@ export const computeOpsStorageBytesExcludingUnstorableIds = async ( bytes += sized.bytes; if (sized.fallback) fallback += 1; } - return { bytes, fallback }; + return { bytes, fallback, requestStartOccupiedIds }; }; export const computeJsonStorageBytes = (value: unknown, fallback: unknown): number => { diff --git a/packages/super-sync-server/src/sync/sync.service.ts b/packages/super-sync-server/src/sync/sync.service.ts index 557b2adb48..a1d3697e7c 100644 --- a/packages/super-sync-server/src/sync/sync.service.ts +++ b/packages/super-sync-server/src/sync/sync.service.ts @@ -129,6 +129,7 @@ export class SyncService { clientId: string, ops: Operation[], isCleanSlate?: boolean, + requestStartOccupiedIds?: ReadonlySet, ): Promise { const results: UploadResult[] = []; const now = Date.now(); @@ -199,6 +200,7 @@ export class SyncService { now, tx, prevalidatedResults, + requestStartOccupiedIds, ); results.push(...batchResult.results); acceptedDeltaBytes = batchResult.acceptedDeltaBytes; @@ -225,6 +227,7 @@ export class SyncService { now, tx, prevalidatedResults.get(op), + requestStartOccupiedIds?.has(op.id), ); results.push(result); if (result.accepted) { diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index b867f5a5a6..b4a112d193 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -252,10 +252,13 @@ describe('Sync compressed body routes', () => { errorCode: SYNC_ERROR_CODES.INVALID_SCHEMA_VERSION, }), ]); - expect(mocks.syncService.uploadOps).toHaveBeenCalledWith(1, clientId, [ - validOp, - invalidOp, - ]); + expect(mocks.syncService.uploadOps).toHaveBeenCalledWith( + 1, + clientId, + [validOp, invalidOp], + undefined, + new Set(), + ); }); it('should not let an invalid large sibling poison a near-quota upload', async () => { @@ -314,10 +317,13 @@ describe('Sync compressed body routes', () => { clientId, ); expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith(1, validBytes); - expect(mocks.syncService.uploadOps).toHaveBeenCalledWith(1, clientId, [ - validOp, - invalidLargeOp, - ]); + expect(mocks.syncService.uploadOps).toHaveBeenCalledWith( + 1, + clientId, + [validOp, invalidLargeOp], + undefined, + new Set(), + ); }); it('should subtract exact already-stored duplicate ops from the ops quota gate', async () => { @@ -366,10 +372,13 @@ describe('Sync compressed body routes', () => { 1, computeOpStorageBytes(newOp).bytes, ); - expect(mocks.syncService.uploadOps).toHaveBeenCalledWith(1, clientId, [ - duplicateOp, - newOp, - ]); + expect(mocks.syncService.uploadOps).toHaveBeenCalledWith( + 1, + clientId, + [duplicateOp, newOp], + undefined, + new Set([duplicateOp.id]), + ); }); it('should not charge same-id different-content ops in the quota gate', async () => { @@ -409,6 +418,17 @@ describe('Sync compressed body routes', () => { expect(response.statusCode).toBe(200); expect(mocks.syncService.checkStorageQuota).toHaveBeenCalledWith(1, 0); + expect(mocks.prisma.operation.findMany).toHaveBeenCalledWith({ + where: { id: { in: [incomingOp.id] } }, + select: { id: true }, + }); + expect(mocks.syncService.uploadOps).toHaveBeenCalledWith( + 1, + clientId, + [incomingOp], + undefined, + new Set([incomingOp.id]), + ); }); it('should charge only the first occurrence of a repeated new ID', async () => { diff --git a/packages/super-sync-server/tests/sync.service.spec.ts b/packages/super-sync-server/tests/sync.service.spec.ts index f7ad17641c..3dd887c608 100644 --- a/packages/super-sync-server/tests/sync.service.spec.ts +++ b/packages/super-sync-server/tests/sync.service.spec.ts @@ -733,6 +733,38 @@ describe('SyncService', () => { expect(testState.operations.size).toBe(25); }); + it.each([ + ['legacy serial', false], + ['batch', true], + ])( + 'rejects a request-start occupied ID in the %s path after its row disappears', + async (_label, batchUpload) => { + const service = new SyncService({ batchUpload }); + const op = makeOp({ + id: 'occupied-before-quota-cleanup', + entityId: 'new-entity-after-cleanup', + payload: { title: 'Must not consume unestimated storage' }, + }); + + const results = await service.uploadOps( + userId, + clientId, + [op], + undefined, + new Set([op.id]), + ); + + expect(results).toEqual([ + expect.objectContaining({ + opId: op.id, + accepted: false, + errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, + }), + ]); + expect(testState.operations.has(op.id)).toBe(false); + }, + ); + it('rejects an intra-batch same-id collision as INVALID_OP_ID', async () => { const service = new SyncService({ batchUpload: true }); const opId = uuidv7(); From 8aa0b8ca508bad4552b883ae072719333f01d7f6 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:35:17 +0200 Subject: [PATCH 28/47] fix(sync): gate incomplete rebuild recovery --- .../sync/conflict-resolution.service.spec.ts | 32 ++++++++ .../sync/conflict-resolution.service.ts | 24 +++++- .../sync/immediate-upload.service.spec.ts | 24 ++++++ .../op-log/sync/immediate-upload.service.ts | 14 ++++ .../sync/operation-log-sync.service.spec.ts | 38 +++++++++- .../op-log/sync/operation-log-sync.service.ts | 73 ++++++++++++------- ...g-upload-piggyback-seq.integration.spec.ts | 2 + .../remote-ops-processing.service.spec.ts | 28 +++++++ .../sync/remote-ops-processing.service.ts | 18 ++++- 9 files changed, 222 insertions(+), 31 deletions(-) diff --git a/src/app/op-log/sync/conflict-resolution.service.spec.ts b/src/app/op-log/sync/conflict-resolution.service.spec.ts index 64690bf83d..d618cfdeed 100644 --- a/src/app/op-log/sync/conflict-resolution.service.spec.ts +++ b/src/app/op-log/sync/conflict-resolution.service.spec.ts @@ -14,6 +14,7 @@ import { CLIENT_ID_PROVIDER } from '../util/client-id.provider'; import { MAX_VECTOR_CLOCK_SIZE } from '../core/operation-log.const'; import { buildEntityRegistry, ENTITY_REGISTRY } from '../core/entity-registry'; import { OperationLogEffects } from '../capture/operation-log.effects'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; describe('ConflictResolutionService', () => { let service: ConflictResolutionService; @@ -3981,6 +3982,37 @@ describe('ConflictResolutionService', () => { callOrder.indexOf('applyOperations'), ); }); + + it('should preserve the incomplete-remote error when the deferred drain also fails', async () => { + const localOp = createOpForBug('local-1', 'client-a', now - 1000); + const remoteOp = createOpForBug('remote-1', 'client-b', now); + const conflicts: EntityConflict[] = [ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ]; + mockOperationApplier.applyOperations.and.resolveTo({ + appliedOps: [], + failedOp: { op: remoteOp, error: new Error('archive failed') }, + }); + mockOperationLogEffects.processDeferredActions.and.rejectWith( + new Error('deferred drain failed'), + ); + + let thrown: unknown; + try { + await service.autoResolveConflictsLWW(conflicts); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(IncompleteRemoteOperationsError); + expect((thrown as Error).message).toBe('archive failed'); + }); }); describe('LWW Update payload always has top-level id (#7330)', () => { diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index f414e0d036..f8538fa034 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -419,6 +419,7 @@ export class ConflictResolutionService { // and guarantees buffered local actions don't leak to the next sync. (#7700) // ───────────────────────────────────────────────────────────────────────── let didApplyRemoteOps = false; + let primaryIncompleteError: IncompleteRemoteOperationsError | undefined; try { if (allOpsToApply.length > 0) { OpLog.normal( @@ -493,12 +494,27 @@ export class ConflictResolutionService { `ConflictResolutionService: Appended local-win update op ${op.id} for ${op.entityType}:${op.entityId}`, ); } + } catch (error) { + if (error instanceof IncompleteRemoteOperationsError) { + primaryIncompleteError = error; + } + throw error; } finally { if (didApplyRemoteOps) { - await processDeferredActionsAfterRemoteApply( - this.injector, - options.callerHoldsOperationLogLock ?? false, - ); + try { + await processDeferredActionsAfterRemoteApply( + this.injector, + options.callerHoldsOperationLogLock ?? false, + ); + } catch (deferredError) { + if (!primaryIncompleteError) { + throw deferredError; + } + OpLog.err( + 'ConflictResolutionService: Deferred-action drain also failed after incomplete remote application', + { name: (deferredError as Error | undefined)?.name }, + ); + } } } diff --git a/src/app/op-log/sync/immediate-upload.service.spec.ts b/src/app/op-log/sync/immediate-upload.service.spec.ts index 3ac7dde023..673b9c77f5 100644 --- a/src/app/op-log/sync/immediate-upload.service.spec.ts +++ b/src/app/op-log/sync/immediate-upload.service.spec.ts @@ -9,6 +9,9 @@ import { SyncSessionValidationService } from './sync-session-validation.service' import { SyncCycleGuardService } from './sync-cycle-guard.service'; import { BehaviorSubject } from 'rxjs'; import { RejectedOpInfo } from '../core/types/sync-results.types'; +import { SnackService } from '../../core/snack/snack.service'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; +import { T } from '../../t.const'; describe('ImmediateUploadService', () => { let service: ImmediateUploadService; @@ -16,6 +19,7 @@ describe('ImmediateUploadService', () => { let mockSyncService: jasmine.SpyObj; let mockDataInitStateService: { isAllDataLoadedInitially$: BehaviorSubject }; let mockSyncWrapperService: { isEncryptionOperationInProgress: boolean }; + let mockSnackService: jasmine.SpyObj; let mockProvider: any; let originalOnLineDescriptor: PropertyDescriptor | undefined; @@ -97,6 +101,7 @@ describe('ImmediateUploadService', () => { mockSyncWrapperService = { isEncryptionOperationInProgress: false, }; + mockSnackService = jasmine.createSpyObj('SnackService', ['open']); TestBed.configureTestingModule({ providers: [ @@ -105,6 +110,7 @@ describe('ImmediateUploadService', () => { { provide: OperationLogSyncService, useValue: mockSyncService }, { provide: DataInitStateService, useValue: mockDataInitStateService }, { provide: SyncWrapperService, useValue: mockSyncWrapperService }, + { provide: SnackService, useValue: mockSnackService }, ], }); @@ -157,6 +163,24 @@ describe('ImmediateUploadService', () => { expect(mockProviderManager.setSyncStatus).not.toHaveBeenCalledWith('IN_SYNC'); })); + it('should report incomplete remote application as a sticky translated error', fakeAsync(() => { + mockSyncService.uploadPendingOps.and.rejectWith( + new IncompleteRemoteOperationsError(new Error('archive failed')), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).toHaveBeenCalledWith({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + })); + it('should report UNKNOWN_OR_CHANGED when a local-win follow-up lacks a mandatory encryption key', fakeAsync(() => { mockSyncService.uploadPendingOps.and.returnValues( Promise.resolve(completedResult({ uploadedCount: 1, localWinOpsCreated: 1 })), diff --git a/src/app/op-log/sync/immediate-upload.service.ts b/src/app/op-log/sync/immediate-upload.service.ts index 52c7184a13..6ebed036b1 100644 --- a/src/app/op-log/sync/immediate-upload.service.ts +++ b/src/app/op-log/sync/immediate-upload.service.ts @@ -11,6 +11,9 @@ import { handleStorageQuotaError } from './sync-error-utils'; import { SyncWrapperService } from '../../imex/sync/sync-wrapper.service'; import { SyncSessionValidationService } from './sync-session-validation.service'; import { SyncCycleGuardService } from './sync-cycle-guard.service'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; +import { SnackService } from '../../core/snack/snack.service'; +import { T } from '../../t.const'; const IMMEDIATE_UPLOAD_DEBOUNCE_MS = 2000; @@ -55,6 +58,7 @@ export class ImmediateUploadService implements OnDestroy { private _syncWrapper = inject(SyncWrapperService); private _sessionValidation = inject(SyncSessionValidationService); private _syncCycleGuard = inject(SyncCycleGuardService); + private _snackService = inject(SnackService); private _uploadTrigger$ = new Subject(); private _subscription: Subscription | null = null; @@ -324,6 +328,16 @@ export class ImmediateUploadService implements OnDestroy { ); } } catch (e) { + if (e instanceof IncompleteRemoteOperationsError) { + this._providerManager.setSyncStatus('ERROR'); + this._snackService.open({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + return; + } + // Check for storage quota exceeded - this requires user action const message = e instanceof Error ? e.message : 'Unknown error'; handleStorageQuotaError(message); diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 413c1d976b..48508f8ab0 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -30,7 +30,10 @@ import { OpType, } from '../core/operation.types'; import { TranslateService } from '@ngx-translate/core'; -import { LocalDataConflictError } from '../core/errors/sync-errors'; +import { + IncompleteRemoteOperationsError, + LocalDataConflictError, +} from '../core/errors/sync-errors'; import { SyncHydrationService } from '../persistence/sync-hydration.service'; import { SyncImportConflictDialogService } from './sync-import-conflict-dialog.service'; import { StateSnapshotService } from '../backup/state-snapshot.service'; @@ -363,6 +366,16 @@ describe('OperationLogSyncService', () => { expect(uploadServiceSpy.uploadPendingOps).not.toHaveBeenCalled(); }); + it('should block upload while a raw rebuild remains incomplete', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + + await expectAsync( + service.uploadPendingOps({} as OperationSyncCapable), + ).toBeRejectedWithError(IncompleteRemoteOperationsError); + + expect(uploadServiceSpy.uploadPendingOps).not.toHaveBeenCalled(); + }); + it('should return localWinOpsCreated: 0 when no piggybacked ops', async () => { opLogStoreSpy.getUnsynced.and.returnValue(Promise.resolve([])); uploadServiceSpy.uploadPendingOps.and.returnValue( @@ -989,6 +1002,29 @@ describe('OperationLogSyncService', () => { expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); }); + it('should resume a raw rebuild whose marker appears while local writes flush', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.returnValues( + Promise.resolve(false), + Promise.resolve(true), + ); + const forceDownloadSpy = spyOn( + service, + 'forceDownloadRemoteState', + ).and.resolveTo(); + const mockProvider = { + isReady: () => Promise.resolve(true), + } as any; + + const result = await service.downloadRemoteOps(mockProvider); + + expect(opLogStoreSpy.isRawRebuildIncomplete).toHaveBeenCalledTimes(2); + expect(forceDownloadSpy).toHaveBeenCalledWith(mockProvider, { + isCrashResume: true, + }); + expect(result.kind).toBe('snapshot_hydrated'); + expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); + }); + it('should flush deferred local work before entering crash-resume rebuild', async () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 026ee8a7f3..c56bc70a77 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -456,31 +456,20 @@ export class OperationLogSyncService { // path excludes this client's own ops server-side, so resuming through it // would silently lose them — redo the raw rebuild instead. if (await this.opLogStore.isRawRebuildIncomplete()) { - OpLog.warn( - 'OperationLogSyncService: Interrupted USE_REMOTE rebuild detected — redoing the raw rebuild.', - ); - try { - // Flush belongs inside the recovery boundary: if persistence is still - // unhealthy, the pre-replace backup is the only safe rollback and its - // Undo affordance must remain visible. - await this._flushLocalWritesIncludingDeferredActions(); - await this.forceDownloadRemoteState(syncProvider, { isCrashResume: true }); - } catch (e) { - // The prior attempt already committed the destructive baseline (that is - // why we are resuming), so the user's original data now lives only in - // the pre-replace backup. If this resume cannot finish — empty/newer- - // schema remote, or a persistent download failure — forceDownloadRemoteState - // throws in its download/validate phase, before it can offer Undo, and - // the backup would otherwise stay stranded with no restore affordance - // (reads as total data loss). Surface the recovery Undo before rethrowing. - await this._offerStrandedRebuildBackup(); - throw e; - } + await this._resumeInterruptedRawRebuild(syncProvider, true); // State was replaced wholesale, exactly like a snapshot hydration. return { kind: 'snapshot_hydrated' }; } await this._flushLocalWritesIncludingDeferredActions(); + // Another tab can commit the destructive replacement while this caller is + // waiting for the operation-log flush barrier. Re-read the marker after the + // barrier and resume the raw rebuild instead of entering the normal download + // path with a partial baseline. + if (await this.opLogStore.isRawRebuildIncomplete()) { + await this._resumeInterruptedRawRebuild(syncProvider, false); + return { kind: 'snapshot_hydrated' }; + } await this._assertNoIncompleteRemoteOperations(); const result = await this.downloadService.downloadRemoteOps(syncProvider, options); @@ -1544,15 +1533,49 @@ export class OperationLogSyncService { } private async _assertNoIncompleteRemoteOperations(): Promise { - const [pendingRemoteOps, failedRemoteOps] = await Promise.all([ - this.opLogStore.getPendingRemoteOps(), - this.opLogStore.getFailedRemoteOps(), - ]); - if (pendingRemoteOps.length > 0 || failedRemoteOps.length > 0) { + const [isRawRebuildIncomplete, pendingRemoteOps, failedRemoteOps] = await Promise.all( + [ + this.opLogStore.isRawRebuildIncomplete(), + this.opLogStore.getPendingRemoteOps(), + this.opLogStore.getFailedRemoteOps(), + ], + ); + if ( + isRawRebuildIncomplete || + pendingRemoteOps.length > 0 || + failedRemoteOps.length > 0 + ) { throw new IncompleteRemoteOperationsError(); } } + private async _resumeInterruptedRawRebuild( + syncProvider: OperationSyncCapable, + flushLocalWrites: boolean, + ): Promise { + OpLog.warn( + 'OperationLogSyncService: Interrupted USE_REMOTE rebuild detected — redoing the raw rebuild.', + ); + try { + if (flushLocalWrites) { + // Flush belongs inside the recovery boundary: if persistence is still + // unhealthy, the pre-replace backup is the only safe rollback and its + // Undo affordance must remain visible. + await this._flushLocalWritesIncludingDeferredActions(); + } + await this.forceDownloadRemoteState(syncProvider, { isCrashResume: true }); + } catch (error) { + // The prior attempt already committed the destructive baseline (that is + // why we are resuming), so the user's original data now lives only in + // the pre-replace backup. If this resume cannot finish — empty/newer- + // schema remote, or a persistent download failure — forceDownloadRemoteState + // throws in its download/validate phase, before it can offer Undo, and + // the backup would otherwise stay stranded with no restore affordance. + await this._offerStrandedRebuildBackup(); + throw error; + } + } + private _assertNoCaptureRacedWithRebuild(): void { if (this.writeFlushService.hasPendingWrites()) { throw new Error( diff --git a/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts b/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts index 07159e4179..65c973f4f8 100644 --- a/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts +++ b/src/app/op-log/sync/operation-log-upload-piggyback-seq.integration.spec.ts @@ -161,10 +161,12 @@ describe('OperationLogSyncService + OperationLogUploadService — piggyback seq 'appendBatchSkipDuplicates', 'hasSyncedOps', 'deleteOpsWhere', + 'isRawRebuildIncomplete', ]); opLogStoreSpy.getUnsynced.and.resolveTo([localPendingEntry]); opLogStoreSpy.getPendingRemoteOps.and.resolveTo([]); opLogStoreSpy.getFailedRemoteOps.and.resolveTo([]); + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); opLogStoreSpy.markSynced.and.resolveTo(undefined); opLogStoreSpy.markRejected.and.resolveTo(undefined); opLogStoreSpy.setVectorClock.and.resolveTo(); diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index 57c3f36ea4..597bffee63 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -35,6 +35,7 @@ import { T } from '../../t.const'; import { OpLog } from '../../core/log'; import { SyncProviderId } from '../sync-providers/provider.const'; import { LOCAL_ONLY_SYNC_KEYS } from '../../features/config/local-only-sync-settings.util'; +import { IncompleteRemoteOperationsError } from '../core/errors/sync-errors'; describe('RemoteOpsProcessingService', () => { let service: RemoteOpsProcessingService; @@ -1497,6 +1498,33 @@ describe('RemoteOpsProcessingService', () => { ); }); + it('should preserve the incomplete-remote error when the deferred drain also fails', async () => { + const remoteOps: Operation[] = [ + createFullOp({ id: 'op-1' }), + createFullOp({ id: 'op-2' }), + ]; + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { + appliedOps: [remoteOps[0]], + failedOp: { op: remoteOps[1], error: new Error('archive failed') }, + }; + }); + operationLogEffectsSpy.processDeferredActions.and.rejectWith( + new Error('deferred drain failed'), + ); + + let thrown: unknown; + try { + await service.applyNonConflictingOps(remoteOps); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(IncompleteRemoteOperationsError); + expect((thrown as Error).message).toBe('archive failed'); + }); + it('should flip the session-validation latch when partial-failure validation fails', async () => { const remoteOps: Operation[] = [ createFullOp({ id: 'op-1' }), diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index 5ee226c2bf..69bd8f79f2 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -446,6 +446,7 @@ export class RemoteOpsProcessingService { // wrapped operationApplier.applyOperations) would leave buffered actions // to leak into the next sync window with stale clocks. (#7700) let didApplyRemoteOps = false; + let primaryIncompleteError: IncompleteRemoteOperationsError | undefined; try { // Core owns the generic crash-safety ordering. Angular diagnostics, // validation, and user notifications stay in this service. @@ -505,9 +506,24 @@ export class RemoteOpsProcessingService { // throw propagates. throw new IncompleteRemoteOperationsError(result.failedOp.error); } + } catch (error) { + if (error instanceof IncompleteRemoteOperationsError) { + primaryIncompleteError = error; + } + throw error; } finally { if (didApplyRemoteOps) { - await processDeferredActionsAfterRemoteApply(this.injector, callerHoldsLock); + try { + await processDeferredActionsAfterRemoteApply(this.injector, callerHoldsLock); + } catch (deferredError) { + if (!primaryIncompleteError) { + throw deferredError; + } + OpLog.err( + 'RemoteOpsProcessingService: Deferred-action drain also failed after incomplete remote application', + { name: (deferredError as Error | undefined)?.name }, + ); + } } } } From eabfd84436a217cc0bd4efc09cdc3495b620fb78 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:35:30 +0200 Subject: [PATCH 29/47] fix(sync): migrate legacy remote failures once --- .../operation-log-architecture.md | 2 +- docs/sync-and-op-log/operation-rules.md | 18 +++--- docs/sync-and-op-log/supersync-scenarios.md | 2 +- src/app/op-log/core/operation-log.const.ts | 3 +- src/app/op-log/persistence/db-keys.const.ts | 7 +++ .../operation-log-store.service.spec.ts | 21 +++++++ .../operation-log-store.service.ts | 59 +++++++++++++++---- 7 files changed, 87 insertions(+), 25 deletions(-) diff --git a/docs/sync-and-op-log/operation-log-architecture.md b/docs/sync-and-op-log/operation-log-architecture.md index fd663c96bb..57592e02b3 100644 --- a/docs/sync-and-op-log/operation-log-architecture.md +++ b/docs/sync-and-op-log/operation-log-architecture.md @@ -2324,7 +2324,7 @@ When adding new entities or relationships: > - **Archive validation**: archiveOld tasks now validated for project/tag references, null-safety added > - **Lock service robustness**: Handle NaN timestamps and invalid lock formats in fallback lock > - **Array payload rejection**: Explicit check to reject arrays (which bypass `typeof === 'object'`) -> - **Pending operation expiry**: Operations pending >24h are quarantined as failed; reducer replay is skipped but archive recovery and the sync gate remain active (`PENDING_OPERATION_EXPIRY_MS`). +> - **Pending operation expiry**: Operations pending >24h are quarantined as failed; hydration still restores their reducer history, while archive recovery and the sync gate remain active (`PENDING_OPERATION_EXPIRY_MS`). --- diff --git a/docs/sync-and-op-log/operation-rules.md b/docs/sync-and-op-log/operation-rules.md index ea837e8dce..455cf1d3c2 100644 --- a/docs/sync-and-op-log/operation-rules.md +++ b/docs/sync-and-op-log/operation-rules.md @@ -193,15 +193,15 @@ export class MyEffects { See `operation-log.const.ts` for all configurable values: -| Constant | Value | Description | -| ----------------------------------- | -------- | ----------------------------------------- | -| `COMPACTION_TRIGGER` | 500 ops | Operations before automatic compaction | -| `COMPACTION_RETENTION_MS` | 7 days | Synced ops older than this may be deleted | -| `EMERGENCY_COMPACTION_RETENTION_MS` | 1 day | Shorter retention for quota exceeded | -| `MAX_COMPACTION_FAILURES` | 3 | Failures before user notification | -| `MAX_DOWNLOAD_OPS_IN_MEMORY` | 50,000 | Bounds memory during API download | -| `REMOTE_OP_FILE_RETENTION_MS` | 14 days | Server-side operation file retention | -| `PENDING_OPERATION_EXPIRY_MS` | 24 hours | Pending ops older than this are rejected | +| Constant | Value | Description | +| ----------------------------------- | -------- | -------------------------------------------------------------------------- | +| `COMPACTION_TRIGGER` | 500 ops | Operations before automatic compaction | +| `COMPACTION_RETENTION_MS` | 7 days | Synced ops older than this may be deleted | +| `EMERGENCY_COMPACTION_RETENTION_MS` | 1 day | Shorter retention for quota exceeded | +| `MAX_COMPACTION_FAILURES` | 3 | Failures before user notification | +| `MAX_DOWNLOAD_OPS_IN_MEMORY` | 50,000 | Bounds memory during API download | +| `REMOTE_OP_FILE_RETENTION_MS` | 14 days | Server-side operation file retention | +| `PENDING_OPERATION_EXPIRY_MS` | 24 hours | Pending ops older than this are quarantined as failed for archive recovery | ## 7. Quick Reference Checklist diff --git a/docs/sync-and-op-log/supersync-scenarios.md b/docs/sync-and-op-log/supersync-scenarios.md index 3d3a315a40..0b70f61e78 100644 --- a/docs/sync-and-op-log/supersync-scenarios.md +++ b/docs/sync-and-op-log/supersync-scenarios.md @@ -165,7 +165,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat **Expected:** 1. Download detects remote snapshot (file-based sync path) -2. Treat every pending op as meaningful except onboarding example-task creates and, before the first completed sync only, the GLOBAL_CONFIG write that enables the `sync` section. This protects non-task entities, recovered MIGRATION/RECOVERY genesis batches, and user configuration. +2. Treat every pending op as meaningful except onboarding example-task creates. GLOBAL_CONFIG writes remain protected even before the first completed sync because the sync-section payload can also contain user preferences. This also protects non-task entities and recovered MIGRATION/RECOVERY genesis batches. 3. If meaningful → throw `LocalDataConflictError` → full conflict dialog 4. If only the explicitly discardable startup ops remain → proceed without dialog diff --git a/src/app/op-log/core/operation-log.const.ts b/src/app/op-log/core/operation-log.const.ts index f19d0c3fa1..5355851a0a 100644 --- a/src/app/op-log/core/operation-log.const.ts +++ b/src/app/op-log/core/operation-log.const.ts @@ -154,7 +154,8 @@ export const MAX_BATCH_OPERATIONS_SIZE = 50; /** * Maximum age for pending operations before they are quarantined as failed (milliseconds). * If an operation has been pending for longer than this (e.g., due to data corruption - * or repeated crashes), reducer replay is skipped and archive recovery remains required. + * or repeated crashes), hydration still restores its reducer history and archive recovery + * remains required. * Default: 24 hours - enough time for legitimate recovery scenarios */ export const PENDING_OPERATION_EXPIRY_MS = 24 * 60 * 60 * 1000; diff --git a/src/app/op-log/persistence/db-keys.const.ts b/src/app/op-log/persistence/db-keys.const.ts index 7d59f6319f..dcce2bd4f1 100644 --- a/src/app/op-log/persistence/db-keys.const.ts +++ b/src/app/op-log/persistence/db-keys.const.ts @@ -54,6 +54,13 @@ export const FULL_STATE_OPS_META_KEY = 'full_state_ops' as const; */ export const RAW_REBUILD_INCOMPLETE_META_KEY = 'raw_rebuild_incomplete' as const; +/** Versioned marker for the one-time legacy terminal remote failure repair. */ +export const LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY = + 'legacy_terminal_remote_failures_migration' as const; + +/** Increment when the legacy terminal remote failure repair needs to run again. */ +export const LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION = 1; + /** Index names for ops object store */ export const OPS_INDEXES = { BY_ID: 'byId' as const, diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 40e099fb63..c0ef012b90 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -1608,6 +1608,27 @@ describe('OperationLogStoreService', () => { expect(recovered.applicationStatus).toBe('failed'); }); + it('should run the legacy terminal remote failure repair only once', async () => { + const firstLegacyOp = createTestOperation({ entityId: 'first-legacy' }); + await service.append(firstLegacyOp, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([firstLegacyOp.id]); + } + await service.markRejected([firstLegacyOp.id]); + + expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(1); + + const laterLegacyOp = createTestOperation({ entityId: 'later-legacy' }); + await service.append(laterLegacyOp, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([laterLegacyOp.id]); + } + await service.markRejected([laterLegacyOp.id]); + + expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(0); + expect((await service.getOpById(laterLegacyOp.id))?.rejectedAt).toBeDefined(); + }); + it('should handle empty array', async () => { await service.markFailed([]); // Should not throw diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index af7ce7ef65..cc02a797a8 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -18,6 +18,8 @@ import { SINGLETON_KEY, BACKUP_KEY, FULL_STATE_OPS_META_KEY, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION, RAW_REBUILD_INCOMPLETE_META_KEY, OPS_INDEXES, ArchiveStoreEntry, @@ -86,6 +88,15 @@ export interface RawRebuildIncompleteEntry { preservedLocalOps: Operation[]; } +interface LegacyTerminalRemoteFailuresMigrationEntry { + version: number; +} + +type OpLogMetaEntry = + | FullStateOpsMetaEntry + | RawRebuildIncompleteEntry + | LegacyTerminalRemoteFailuresMigrationEntry; + /** * Stored operation log entry that can hold either compact or full operation format. * Used internally for backwards compatibility with existing data. @@ -211,7 +222,7 @@ interface OpLogDB extends DBSchema { */ [STORE_NAMES.META]: { key: string; - value: FullStateOpsMetaEntry; + value: OpLogMetaEntry; }; } @@ -1122,21 +1133,43 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { await this._ensureInit(); let recoveredCount = 0; - await this._adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { - const entries = await tx.getAll(STORE_NAMES.OPS); - for (const entry of entries) { + await this._adapter.transaction( + [STORE_NAMES.OPS, STORE_NAMES.META], + 'readwrite', + async (tx) => { + const migration = await tx.get( + STORE_NAMES.META, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, + ); if ( - entry.source === 'remote' && - entry.rejectedAt !== undefined && - (entry.retryCount ?? 0) >= 4 + (migration?.version ?? 0) >= LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION ) { - entry.rejectedAt = undefined; - entry.applicationStatus = 'failed'; - await tx.put(STORE_NAMES.OPS, entry); - recoveredCount++; + return; } - } - }); + + const entries = await tx.getAll(STORE_NAMES.OPS); + for (const entry of entries) { + if ( + entry.source === 'remote' && + entry.rejectedAt !== undefined && + (entry.retryCount ?? 0) >= 4 + ) { + entry.rejectedAt = undefined; + entry.applicationStatus = 'failed'; + await tx.put(STORE_NAMES.OPS, entry); + recoveredCount++; + } + } + + await tx.put( + STORE_NAMES.META, + { + version: LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION, + } satisfies LegacyTerminalRemoteFailuresMigrationEntry, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, + ); + }, + ); return recoveredCount; } From bfdc63e0c65e8562c101a00050b282390fedbddd Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Fri, 10 Jul 2026 22:50:33 +0200 Subject: [PATCH 30/47] fix(sync): preserve actionable recovery state --- .../imex/sync/sync-wrapper.service.spec.ts | 13 +++ src/app/imex/sync/sync-wrapper.service.ts | 12 ++- .../operation-log-store.service.spec.ts | 91 +++++++++++++++---- .../operation-log-store.service.ts | 6 ++ .../sync/immediate-upload.service.spec.ts | 21 ++++- .../op-log/sync/immediate-upload.service.ts | 12 ++- .../sync/operation-log-sync.service.spec.ts | 44 +++++++++ .../ws-triggered-download.service.spec.ts | 21 ++++- .../sync/ws-triggered-download.service.ts | 12 ++- 9 files changed, 199 insertions(+), 33 deletions(-) diff --git a/src/app/imex/sync/sync-wrapper.service.spec.ts b/src/app/imex/sync/sync-wrapper.service.spec.ts index 4e3d3e380a..c487876b9d 100644 --- a/src/app/imex/sync/sync-wrapper.service.spec.ts +++ b/src/app/imex/sync/sync-wrapper.service.spec.ts @@ -1066,6 +1066,19 @@ describe('SyncWrapperService', () => { }); }); + it('should preserve an existing persistent recovery action for incomplete remote work', async () => { + mockSnackService.hasPendingPersistentAction.and.returnValue(true); + mockSyncService.downloadRemoteOps.and.rejectWith( + new IncompleteRemoteOperationsError(new Error('archive failed')), + ); + + const result = await service.sync(); + + expect(result).toBe('HANDLED_ERROR'); + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).not.toHaveBeenCalled(); + }); + it('should handle PotentialCorsError with snack message', async () => { mockSyncService.downloadRemoteOps.and.returnValue( Promise.reject(new PotentialCorsError('https://example.com')), diff --git a/src/app/imex/sync/sync-wrapper.service.ts b/src/app/imex/sync/sync-wrapper.service.ts index 17c8870b1d..2cd42204f2 100644 --- a/src/app/imex/sync/sync-wrapper.service.ts +++ b/src/app/imex/sync/sync-wrapper.service.ts @@ -710,11 +710,13 @@ export class SyncWrapperService { return 'HANDLED_ERROR'; } else if (error instanceof IncompleteRemoteOperationsError) { this._providerManager.setSyncStatus('ERROR'); - this._snackService.open({ - msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, - type: 'ERROR', - config: { duration: 0 }, - }); + if (!this._snackService.hasPendingPersistentAction()) { + this._snackService.open({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + } return 'HANDLED_ERROR'; } else if ( error instanceof AuthFailSPError || diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index c0ef012b90..70280af355 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -6,6 +6,7 @@ import { VectorClockService } from '../sync/vector-clock.service'; import { ActionType, Operation, + OperationLogEntry, OpType, EntityType, VectorClock, @@ -29,7 +30,12 @@ import { MAX_VECTOR_CLOCK_SIZE, } from '../core/operation-log.const'; import { IndexedDBOpenError } from '../core/errors/indexed-db-open.error'; -import { FULL_STATE_OPS_META_KEY, SINGLETON_KEY, STORE_NAMES } from './db-keys.const'; +import { + FULL_STATE_OPS_META_KEY, + OPS_INDEXES, + SINGLETON_KEY, + STORE_NAMES, +} from './db-keys.const'; import { ArchiveModel } from '../../features/time-tracking/time-tracking.model'; describe('OperationLogStoreService', () => { @@ -1555,6 +1561,32 @@ describe('OperationLogStoreService', () => { }); describe('markFailed', () => { + const seedLegacyTerminalRemoteFailure = async (op: Operation): Promise => { + await service.append(op, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([op.id]); + } + await service.markRejected([op.id]); + + const adapter = ( + service as unknown as { + _adapter: OpLogDbAdapter; + } + )._adapter; + await adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { + const entry = await tx.getFromIndex( + STORE_NAMES.OPS, + OPS_INDEXES.BY_ID, + op.id, + ); + if (!entry) { + throw new Error('Expected seeded legacy operation'); + } + entry.applicationStatus = undefined; + await tx.put(STORE_NAMES.OPS, entry); + }); + }; + it('should increment retry count', async () => { const op = createTestOperation(); await service.append(op, 'remote', { pendingApply: true }); @@ -1594,11 +1626,7 @@ describe('OperationLogStoreService', () => { it('should re-quarantine legacy terminal remote failures', async () => { const op = createTestOperation(); - await service.append(op, 'remote', { pendingApply: true }); - for (let retry = 0; retry < 4; retry++) { - await service.markFailed([op.id]); - } - await service.markRejected([op.id]); + await seedLegacyTerminalRemoteFailure(op); expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(1); @@ -1610,25 +1638,32 @@ describe('OperationLogStoreService', () => { it('should run the legacy terminal remote failure repair only once', async () => { const firstLegacyOp = createTestOperation({ entityId: 'first-legacy' }); - await service.append(firstLegacyOp, 'remote', { pendingApply: true }); - for (let retry = 0; retry < 4; retry++) { - await service.markFailed([firstLegacyOp.id]); - } - await service.markRejected([firstLegacyOp.id]); + await seedLegacyTerminalRemoteFailure(firstLegacyOp); expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(1); const laterLegacyOp = createTestOperation({ entityId: 'later-legacy' }); - await service.append(laterLegacyOp, 'remote', { pendingApply: true }); - for (let retry = 0; retry < 4; retry++) { - await service.markFailed([laterLegacyOp.id]); - } - await service.markRejected([laterLegacyOp.id]); + await seedLegacyTerminalRemoteFailure(laterLegacyOp); expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(0); expect((await service.getOpById(laterLegacyOp.id))?.rejectedAt).toBeDefined(); }); + it('should leave legitimately rejected failed remote work untouched', async () => { + const op = createTestOperation({ entityId: 'legitimately-rejected' }); + await service.append(op, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([op.id]); + } + await service.markRejected([op.id]); + + expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(0); + + const stored = await service.getOpById(op.id); + expect(stored?.rejectedAt).toBeDefined(); + expect(stored?.applicationStatus).toBe('failed'); + }); + it('should handle empty array', async () => { await service.markFailed([]); // Should not throw @@ -2139,6 +2174,30 @@ describe('OperationLogStoreService', () => { lastTimeTrackingFlush: 0, }); + it('should atomically clear an interrupted raw-rebuild marker', async () => { + await service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('remote-young'), + archiveOld: createArchive('remote-old'), + }); + expect(await service.isRawRebuildIncomplete()).toBe(true); + + await service.runDestructiveStateReplacement({ + syncImportOp: createTestOperation({ + opType: OpType.BackupImport, + entityType: 'ALL' as EntityType, + entityId: 'restored-backup', + payload: { task: { ids: [], entities: {} } }, + }), + snapshotEntityKeys: [], + }); + + expect(await service.isRawRebuildIncomplete()).toBe(false); + }); + it('should write archives in the same destructive replacement', async () => { const archiveYoung = createArchive('young-task'); const archiveOld = createArchive('old-task'); diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index cc02a797a8..a94ed39154 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -1152,6 +1152,7 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort= 4 ) { entry.rejectedAt = undefined; @@ -2061,6 +2062,11 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { mockSyncWrapperService = { isEncryptionOperationInProgress: false, }; - mockSnackService = jasmine.createSpyObj('SnackService', ['open']); + mockSnackService = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + mockSnackService.hasPendingPersistentAction.and.returnValue(false); TestBed.configureTestingModule({ providers: [ @@ -181,6 +185,21 @@ describe('ImmediateUploadService', () => { }); })); + it('should preserve an existing persistent recovery action for incomplete remote work', fakeAsync(() => { + mockSnackService.hasPendingPersistentAction.and.returnValue(true); + mockSyncService.uploadPendingOps.and.rejectWith( + new IncompleteRemoteOperationsError(new Error('archive failed')), + ); + + service.initialize(); + service.trigger(); + tick(2000); + flush(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).not.toHaveBeenCalled(); + })); + it('should report UNKNOWN_OR_CHANGED when a local-win follow-up lacks a mandatory encryption key', fakeAsync(() => { mockSyncService.uploadPendingOps.and.returnValues( Promise.resolve(completedResult({ uploadedCount: 1, localWinOpsCreated: 1 })), diff --git a/src/app/op-log/sync/immediate-upload.service.ts b/src/app/op-log/sync/immediate-upload.service.ts index 6ebed036b1..d2440aa2da 100644 --- a/src/app/op-log/sync/immediate-upload.service.ts +++ b/src/app/op-log/sync/immediate-upload.service.ts @@ -330,11 +330,13 @@ export class ImmediateUploadService implements OnDestroy { } catch (e) { if (e instanceof IncompleteRemoteOperationsError) { this._providerManager.setSyncStatus('ERROR'); - this._snackService.open({ - msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, - type: 'ERROR', - config: { duration: 0 }, - }); + if (!this._snackService.hasPendingPersistentAction()) { + this._snackService.open({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + } return; } diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 48508f8ab0..d7bfd73e6b 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -1064,6 +1064,50 @@ describe('OperationLogSyncService', () => { ); }); + it('should allow uploads after stranded-rebuild Undo clears the marker', async () => { + let isRawRebuildIncomplete = true; + opLogStoreSpy.isRawRebuildIncomplete.and.callFake( + async () => isRawRebuildIncomplete, + ); + spyOn(service, 'forceDownloadRemoteState').and.rejectWith( + new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'), + ); + opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + backupServiceSpy.restoreImportBackup.and.callFake(async () => { + isRawRebuildIncomplete = false; + return true; + }); + uploadServiceSpy.uploadPendingOps.and.resolveTo({ + uploadedCount: 0, + piggybackedOps: [], + rejectedCount: 0, + rejectedOps: [], + }); + const mockProvider = { isReady: () => Promise.resolve(true) } as any; + + await expectAsync(service.downloadRemoteOps(mockProvider)).toBeRejected(); + + const undoSnack = snackServiceSpy.open.calls + .allArgs() + .map(([params]) => params) + .find( + (params) => + typeof params === 'object' && + params !== null && + params.msg === T.F.SYNC.S.LOCAL_DATA_REPLACE_UNDO, + ); + expect(undoSnack).toBeDefined(); + if (typeof undoSnack !== 'object' || undoSnack === null || !undoSnack.actionFn) { + throw new Error('Expected the stranded-rebuild Undo action'); + } + await undoSnack.actionFn(); + + await service.uploadPendingOps(mockProvider); + + expect(backupServiceSpy.restoreImportBackup).toHaveBeenCalledWith(4242); + expect(uploadServiceSpy.uploadPendingOps).toHaveBeenCalled(); + }); + it('should not respawn the recovery snack while one is already showing', async () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); spyOn(service, 'forceDownloadRemoteState').and.rejectWith( diff --git a/src/app/op-log/sync/ws-triggered-download.service.spec.ts b/src/app/op-log/sync/ws-triggered-download.service.spec.ts index 7014500853..1efd679ea9 100644 --- a/src/app/op-log/sync/ws-triggered-download.service.spec.ts +++ b/src/app/op-log/sync/ws-triggered-download.service.spec.ts @@ -61,7 +61,11 @@ describe('WsTriggeredDownloadService', () => { // off it lazily (via Injector, to avoid a DI cycle). Provide a minimal mock so // the real (heavily-dependent) service is never constructed in the unit test. mockSyncWrapper = { isEncryptionOperationInProgress: false }; - mockSnackService = jasmine.createSpyObj('SnackService', ['open']); + mockSnackService = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + mockSnackService.hasPendingPersistentAction.and.returnValue(false); TestBed.configureTestingModule({ providers: [ @@ -251,6 +255,21 @@ describe('WsTriggeredDownloadService', () => { }); })); + it('should preserve an existing persistent recovery action for incomplete remote work', fakeAsync(() => { + mockSnackService.hasPendingPersistentAction.and.returnValue(true); + mockSyncService.downloadRemoteOps.and.rejectWith( + new IncompleteRemoteOperationsError(new Error('archive failed')), + ); + + service.start(); + notification$.next({ latestSeq: 1 }); + tick(500); + flushMicrotasks(); + + expect(mockProviderManager.setSyncStatus).toHaveBeenCalledWith('ERROR'); + expect(mockSnackService.open).not.toHaveBeenCalled(); + })); + it('should be idempotent when start is called twice', fakeAsync(() => { service.start(); service.start(); diff --git a/src/app/op-log/sync/ws-triggered-download.service.ts b/src/app/op-log/sync/ws-triggered-download.service.ts index 762cb7fd22..3135c7e7ea 100644 --- a/src/app/op-log/sync/ws-triggered-download.service.ts +++ b/src/app/op-log/sync/ws-triggered-download.service.ts @@ -176,11 +176,13 @@ export class WsTriggeredDownloadService implements OnDestroy { err, ); this._providerManager.setSyncStatus('ERROR'); - this._snackService.open({ - msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, - type: 'ERROR', - config: { duration: 0 }, - }); + if (!this._snackService.hasPendingPersistentAction()) { + this._snackService.open({ + msg: T.F.SYNC.S.INCOMPLETE_REMOTE_OPERATIONS, + type: 'ERROR', + config: { duration: 0 }, + }); + } return; } if (err instanceof AuthFailSPError || err instanceof MissingCredentialsSPError) { From eaa15fe575f062da7352359cf71613b9e3b0d04e Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 11:25:57 +0200 Subject: [PATCH 31/47] fix(sync): handle initial provider setup safely --- .../supersync-scenarios-flowchart.md | 2 +- docs/sync-and-op-log/supersync-scenarios.md | 10 +- .../sync/operation-log-sync.service.spec.ts | 297 +++++++++++++++++- .../op-log/sync/operation-log-sync.service.ts | 151 +++++++-- .../remote-ops-processing.service.spec.ts | 34 ++ .../sync/remote-ops-processing.service.ts | 30 +- .../sync-import-conflict-gate.service.spec.ts | 27 +- .../sync/sync-import-conflict-gate.service.ts | 69 +++- ...ample-task-import-gate.integration.spec.ts | 37 ++- 9 files changed, 571 insertions(+), 86 deletions(-) diff --git a/docs/sync-and-op-log/supersync-scenarios-flowchart.md b/docs/sync-and-op-log/supersync-scenarios-flowchart.md index 0ce06eaa22..9412bf61af 100644 --- a/docs/sync-and-op-log/supersync-scenarios-flowchart.md +++ b/docs/sync-and-op-log/supersync-scenarios-flowchart.md @@ -105,6 +105,6 @@ flowchart TD **Notes:** - The `Enter Password` and `Decrypt Error` dialogs correspond to `DecryptNoPasswordError` and `DecryptError` respectively — they are distinct components with different options. -- `IMPORT_CONFLICT` uses pending ops only, not already-synced store contents (`_hasMeaningfulPendingOps()`). Every pending op is protected except onboarding example-task creates. This includes GLOBAL_CONFIG changes before first sync, non-task entities, and MIGRATION/RECOVERY genesis batches. PASSWORD_CHANGED SYNC_IMPORTs without pending ops still fall through to silent acceptance — the data is identical, only the encryption changed. Including already-synced store contents would let an old client pick `USE_LOCAL` and force-upload stale pre-import state, rolling back the remote import for everyone. +- `IMPORT_CONFLICT` uses pending ops only, not already-synced store contents (`_hasMeaningfulPendingOps()`). Every pending op is protected except onboarding example-task creates and the never-synced `GLOBAL_CONFIG:sync` provider-setup write. Other config sections, non-task entities, and MIGRATION/RECOVERY full-state batches remain protected. PASSWORD_CHANGED SYNC_IMPORTs without meaningful pending ops still fall through to silent acceptance — the data is identical, only the encryption changed. Including already-synced store contents would let an old client pick `USE_LOCAL` and force-upload stale pre-import state, rolling back the remote import for everyone. - LWW tie-breaking: on equal timestamps, remote wins (server-authoritative). `moveToArchive` operations always win regardless of timestamp. - Re-download retry limit: max 3 resolution attempts per entity (`MAX_CONCURRENT_RESOLUTION_ATTEMPTS`); if exceeded, ops are permanently rejected. diff --git a/docs/sync-and-op-log/supersync-scenarios.md b/docs/sync-and-op-log/supersync-scenarios.md index 0b70f61e78..5bb3d45ebf 100644 --- a/docs/sync-and-op-log/supersync-scenarios.md +++ b/docs/sync-and-op-log/supersync-scenarios.md @@ -165,9 +165,9 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat **Expected:** 1. Download detects remote snapshot (file-based sync path) -2. Treat every pending op as meaningful except onboarding example-task creates. GLOBAL_CONFIG writes remain protected even before the first completed sync because the sync-section payload can also contain user preferences. This also protects non-task entities and recovered MIGRATION/RECOVERY genesis batches. +2. Treat every pending op as meaningful except onboarding example-task creates and, before the first completed sync only, the `GLOBAL_CONFIG:sync` setup write needed to configure the provider. Other config sections, non-task entities, and MIGRATION/RECOVERY full-state batches remain protected. 3. If meaningful → throw `LocalDataConflictError` → full conflict dialog -4. If only the explicitly discardable startup ops remain → proceed without dialog +4. If only the explicitly discardable startup ops remain → proceed without dialog and reject those ops locally so they cannot replay after the imported state **Note:** This op-content check only applies to the file-based snapshot path. For SuperSync (incremental ops path), the fresh client check uses `_hasMeaningfulStoreData()` (store-based check) instead. @@ -198,7 +198,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat **Expected:** 1. Download batch contains SYNC_IMPORT -2. Check pending local ops. Any pending work triggers the dialog except onboarding example-task creates. GLOBAL_CONFIG changes are protected even before the first completed sync because the sync-section payload can carry user preferences. +2. Check pending local ops. Any pending work triggers the dialog except onboarding example-task creates and the never-synced `GLOBAL_CONFIG:sync` provider-setup write. Other GLOBAL_CONFIG sections remain protected. 3. **Show conflict dialog BEFORE processing** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` 4. USE_LOCAL → `forceUploadLocalState()` (overrides remote with local data) 5. USE_REMOTE → `forceDownloadRemoteState()` (clears local ops, downloads from seq 0) @@ -254,12 +254,12 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat 1. Upload completes → server returns piggybacked ops containing SYNC_IMPORT 2. Check for SYNC_IMPORT in piggybacked ops BEFORE `processRemoteOps()` -3. If found AND `_hasMeaningfulPendingOps()` = true (all pending work except onboarding example-task creates): +3. If found AND `_hasMeaningfulPendingOps()` = true (all pending work except onboarding example-task creates and the never-synced `GLOBAL_CONFIG:sync` provider-setup write): - **Show conflict dialog** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` from the piggybacked op - USE_LOCAL → `forceUploadLocalState()` (overrides remote) - USE_REMOTE → `forceDownloadRemoteState()` (clears local, downloads from seq 0) - CANCEL → return with `cancelled: true`, callers skip post-upload logic -4. If no meaningful pending ops → `processRemoteOps()` applies silently (no dialog) regardless of whether the NgRx store already has user data — that data was already synced and the SYNC_IMPORT is the new authoritative state. +4. If no meaningful pending ops → `processRemoteOps()` applies silently (no dialog), then reject live discardable startup ops only after processing succeeds. Existing NgRx store data does not trigger a dialog because it was already synced and the SYNC_IMPORT is the new authoritative state. **Mirrors the download path (D.1 / D.2):** the gate is unsynced pending changes, not store contents. Prompting on already-synced store data would let an old client roll back the remote import via USE_LOCAL. diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index d7bfd73e6b..0ac19981b6 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -12,7 +12,10 @@ import { ValidateStateService } from '../validation/validate-state.service'; import { SyncSessionValidationService } from './sync-session-validation.service'; import { RepairOperationService } from '../validation/repair-operation.service'; import { OperationLogUploadService } from './operation-log-upload.service'; -import { OperationLogDownloadService } from './operation-log-download.service'; +import { + DownloadResult, + OperationLogDownloadService, +} from './operation-log-download.service'; import { LockService } from './lock.service'; import { OperationLogCompactionService } from '../persistence/operation-log-compaction.service'; import { SyncImportFilterService } from './sync-import-filter.service'; @@ -45,6 +48,7 @@ import { OperationSyncCapable } from '../sync-providers/provider.interface'; import { selectSyncConfig } from '../../features/config/store/global-config.reducer'; import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; import { SyncProviderId } from '../sync-providers/provider.const'; +import { stripLocalOnlySyncSettingsFromAppData } from '../../features/config/local-only-sync-settings.util'; describe('OperationLogSyncService', () => { let service: OperationLogSyncService; @@ -64,6 +68,24 @@ describe('OperationLogSyncService', () => { let operationApplierSpy: jasmine.SpyObj; let operationLogEffectsSpy: jasmine.SpyObj; + const createProviderSetupEntry = (): OperationLogEntry => ({ + seq: 1, + op: { + id: 'sync-provider-setup', + clientId: 'client-A', + actionType: ActionType.GLOBAL_CONFIG_UPDATE_SECTION, + opType: OpType.Update, + entityType: 'GLOBAL_CONFIG', + entityId: 'sync', + payload: { sectionKey: 'sync' }, + vectorClock: { clientA: 1 }, + timestamp: Date.now(), + schemaVersion: 1, + }, + appliedAt: Date.now(), + source: 'local', + }); + beforeEach(() => { snackServiceSpy = jasmine.createSpyObj('SnackService', [ 'open', @@ -1669,10 +1691,8 @@ describe('OperationLogSyncService', () => { source: 'local', }); - const fileSnapshotDownloadResult = { + const fileSnapshotDownloadResult: DownloadResult = { newOps: [], - hasMore: false, - latestSeq: 0, needsFullStateUpload: false, success: true, providerMode: 'fileSnapshotOps', @@ -1682,6 +1702,32 @@ describe('OperationLogSyncService', () => { latestServerSeq: 1, }; + it('silently adopts a file snapshot and rejects the never-synced provider setup op', async () => { + const setupEntry = createProviderSetupEntry(); + opLogStoreSpy.hasSyncedOps.and.resolveTo(false); + opLogStoreSpy.getUnsynced.and.resolveTo([setupEntry]); + + const syncHydrationServiceSpy = TestBed.inject( + SyncHydrationService, + ) as jasmine.SpyObj; + syncHydrationServiceSpy.hydrateFromRemoteSync.and.resolveTo(); + downloadServiceSpy.downloadRemoteOps.and.resolveTo(fileSnapshotDownloadResult); + + const mockProvider = { + isReady: () => Promise.resolve(true), + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await expectAsync( + service.downloadRemoteOps(mockProvider, { isNeverSynced: true }), + ).toBeResolved(); + expect(syncHydrationServiceSpy.hydrateFromRemoteSync).toHaveBeenCalled(); + expect(opLogStoreSpy.markRejected).toHaveBeenCalledOnceWith([ + 'sync-provider-setup', + ]); + }); + it('does NOT throw LocalDataConflictError when store + pending ops contain only example tasks (#7985)', async () => { opLogStoreSpy.getUnsynced.and.returnValue( Promise.resolve([exampleCreateEntry('ex-task-1')]), @@ -1700,7 +1746,7 @@ describe('OperationLogSyncService', () => { syncHydrationServiceSpy.hydrateFromRemoteSync.and.resolveTo(); downloadServiceSpy.downloadRemoteOps.and.returnValue( - Promise.resolve(fileSnapshotDownloadResult as any), + Promise.resolve(fileSnapshotDownloadResult), ); const mockProvider = { @@ -1747,7 +1793,7 @@ describe('OperationLogSyncService', () => { ); downloadServiceSpy.downloadRemoteOps.and.returnValue( - Promise.resolve(fileSnapshotDownloadResult as any), + Promise.resolve(fileSnapshotDownloadResult), ); const mockProvider = { @@ -1774,7 +1820,7 @@ describe('OperationLogSyncService', () => { } as any); downloadServiceSpy.downloadRemoteOps.and.returnValue( - Promise.resolve(fileSnapshotDownloadResult as any), + Promise.resolve(fileSnapshotDownloadResult), ); const mockProvider = { @@ -2656,7 +2702,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), - } as any; + } as unknown as OperationSyncCapable; await service.forceUploadLocalState(mockProvider); @@ -3230,6 +3276,55 @@ describe('OperationLogSyncService', () => { expect(opLogStoreSpy.runRemoteStateReplacement).not.toHaveBeenCalled(); }); + it('should restore device-local sync settings before validating a file snapshot', async () => { + const mockStore = TestBed.inject(MockStore); + mockStore.overrideSelector(selectSyncConfig, { + ...DEFAULT_GLOBAL_CONFIG.sync, + syncProvider: SyncProviderId.WebDAV, + isEnabled: true, + isEncryptionEnabled: true, + syncInterval: 23, + isManualSyncOnly: true, + }); + mockStore.refreshState(); + const wireSnapshot = stripLocalOnlySyncSettingsFromAppData({ + globalConfig: DEFAULT_GLOBAL_CONFIG, + }); + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [], + needsFullStateUpload: false, + success: true, + providerMode: 'fileSnapshotOps', + failedFileCount: 0, + snapshotState: wireSnapshot, + latestServerSeq: 1, + }); + validateStateServiceSpy.validateAndRepair.and.resolveTo({ + isValid: true, + wasRepaired: false, + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + await service.forceDownloadRemoteState(mockProvider); + + expect(validateStateServiceSpy.validateAndRepair).toHaveBeenCalledOnceWith( + jasmine.objectContaining({ + globalConfig: jasmine.objectContaining({ + sync: jasmine.objectContaining({ + syncProvider: SyncProviderId.WebDAV, + isEnabled: true, + isEncryptionEnabled: true, + syncInterval: 23, + isManualSyncOnly: true, + }), + }), + }), + ); + }); + it('should NOT advance the cursor past 0 when replay blocks on a migration failure', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], @@ -4025,6 +4120,81 @@ describe('OperationLogSyncService', () => { expect(result.kind).toBe('ops_processed'); }); + it('should discard initial provider setup only after the incoming import commits', async () => { + const incomingSyncImport = createIncomingSyncImport(); + const setupEntry = createProviderSetupEntry(); + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [incomingSyncImport], + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 42, + }); + opLogStoreSpy.getUnsynced.and.resolveTo([setupEntry]); + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as unknown as OperationSyncCapable; + + const result = await service.downloadRemoteOps(mockProvider, { + isNeverSynced: true, + }); + + expect(result.kind).toBe('blocked_incompatible'); + expect(opLogStoreSpy.markRejected).not.toHaveBeenCalled(); + expect(mockProvider.setLastServerSeq).not.toHaveBeenCalled(); + + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + committedFullStateOpIds: [incomingSyncImport.id], + }); + + const committedPrefixResult = await service.downloadRemoteOps(mockProvider, { + isNeverSynced: true, + }); + + expect(committedPrefixResult.kind).toBe('blocked_incompatible'); + expect(opLogStoreSpy.markRejected).toHaveBeenCalledOnceWith([setupEntry.op.id]); + expect(mockProvider.setLastServerSeq).not.toHaveBeenCalled(); + + const processingError = new Error('deferred drain failed'); + opLogStoreSpy.markRejected.calls.reset(); + opLogStoreSpy.getOpById.and.resolveTo(undefined); + remoteOpsProcessingServiceSpy.processRemoteOps.and.rejectWith(processingError); + + await expectAsync( + service.downloadRemoteOps(mockProvider, { isNeverSynced: true }), + ).toBeRejectedWith(processingError); + expect(opLogStoreSpy.markRejected).not.toHaveBeenCalled(); + + for (const applicationStatus of ['applied', 'archive_pending', 'failed'] as const) { + opLogStoreSpy.markRejected.calls.reset(); + opLogStoreSpy.getOpById.and.resolveTo({ + seq: 2, + op: incomingSyncImport, + appliedAt: Date.now(), + source: 'remote', + applicationStatus, + }); + + await expectAsync( + service.downloadRemoteOps(mockProvider, { isNeverSynced: true }), + ).toBeRejectedWith(processingError); + expect(opLogStoreSpy.markRejected).toHaveBeenCalledOnceWith([setupEntry.op.id]); + } + }); + it('should show conflict dialog for incoming SYNC_IMPORT when client has pending meaningful ops', async () => { const incomingSyncImport = createIncomingSyncImport(); @@ -4345,7 +4515,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { isReady: () => Promise.resolve(true), - } as any; + } as unknown as OperationSyncCapable; const result = await service.uploadPendingOps(mockProvider); @@ -4405,7 +4575,7 @@ describe('OperationLogSyncService', () => { const mockProvider = { isReady: () => Promise.resolve(true), - } as any; + } as unknown as OperationSyncCapable; const result = await service.uploadPendingOps(mockProvider); @@ -4463,6 +4633,113 @@ describe('OperationLogSyncService', () => { expect(result.kind).toBe('completed'); }); + it('should reject a live provider setup op after silently applying a piggybacked import on initial sync', async () => { + const piggybackedSyncImport: Operation = { + id: 'import-1', + clientId: 'client-B', + actionType: ActionType.LOAD_ALL_DATA, + opType: OpType.SyncImport, + entityType: 'ALL', + payload: {}, + vectorClock: { clientB: 5 }, + timestamp: Date.now(), + schemaVersion: 1, + }; + const setupEntry = createProviderSetupEntry(); + + uploadServiceSpy.uploadPendingOps.and.resolveTo({ + uploadedCount: 1, + piggybackedOps: [piggybackedSyncImport], + rejectedCount: 0, + rejectedOps: [], + selectedPendingOps: [setupEntry], + }); + opLogStoreSpy.getUnsynced.and.resolveTo([setupEntry]); + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: false, + committedFullStateOpIds: [piggybackedSyncImport.id], + }); + + const mockProvider = { + isReady: () => Promise.resolve(true), + } as unknown as OperationSyncCapable; + + const result = await service.uploadPendingOps(mockProvider, { + isNeverSynced: true, + }); + + expect( + syncImportConflictDialogServiceSpy.showConflictDialog, + ).not.toHaveBeenCalled(); + expect(opLogStoreSpy.markRejected).toHaveBeenCalledOnceWith([ + 'sync-provider-setup', + ]); + expect(remoteOpsProcessingServiceSpy.processRemoteOps).toHaveBeenCalledWith([ + piggybackedSyncImport, + ]); + expect(result.kind).toBe('completed'); + }); + + it('should keep the initial provider setup op pending when piggybacked import processing is blocked', async () => { + const piggybackedSyncImport: Operation = { + id: 'import-1', + clientId: 'client-B', + actionType: ActionType.LOAD_ALL_DATA, + opType: OpType.SyncImport, + entityType: 'ALL', + payload: {}, + vectorClock: { clientB: 5 }, + timestamp: Date.now(), + schemaVersion: 1, + }; + const setupEntry = createProviderSetupEntry(); + uploadServiceSpy.uploadPendingOps.and.resolveTo({ + uploadedCount: 1, + piggybackedOps: [piggybackedSyncImport], + rejectedCount: 0, + rejectedOps: [], + selectedPendingOps: [setupEntry], + }); + opLogStoreSpy.getUnsynced.and.resolveTo([setupEntry]); + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + }); + const mockProvider = { + isReady: () => Promise.resolve(true), + } as unknown as OperationSyncCapable; + + const result = await service.uploadPendingOps(mockProvider, { + isNeverSynced: true, + }); + + expect(result.kind).toBe('blocked_incompatible'); + expect(opLogStoreSpy.markRejected).not.toHaveBeenCalled(); + + remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ + localWinOpsCreated: 0, + allOpsFilteredBySyncImport: false, + filteredOpCount: 0, + isLocalUnsyncedImport: false, + blockedByIncompatibleOp: true, + committedFullStateOpIds: [piggybackedSyncImport.id], + }); + + const committedPrefixResult = await service.uploadPendingOps(mockProvider, { + isNeverSynced: true, + }); + + expect(committedPrefixResult.kind).toBe('blocked_incompatible'); + expect(opLogStoreSpy.markRejected).toHaveBeenCalledOnceWith([setupEntry.op.id]); + }); + it('should flush again before checking piggybacked SYNC_IMPORT conflicts', async () => { const piggybackedSyncImport: Operation = { id: 'import-1', diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index c56bc70a77..8e1416735d 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -61,11 +61,16 @@ import { selectSyncConfig } from '../../features/config/store/global-config.redu import { applyLocalOnlySyncSettingsToAppData, LocalOnlySyncSettings, + stripLocalOnlySyncSettingsFromAppData, } from '../../features/config/local-only-sync-settings.util'; import { DEFAULT_GLOBAL_CONFIG } from '../../features/config/default-global-config.const'; import { OperationApplierService } from '../apply/operation-applier.service'; import { processDeferredActions } from './process-deferred-actions-flush.util'; +type RemoteOpsProcessingResult = Awaited< + ReturnType +>; + /** * Orchestrates synchronization of the Operation Log with remote storage. * @@ -268,6 +273,8 @@ export class OperationLogSyncService { }; if (result.piggybackedOps.length > 0) { + let startupOpIdsToDiscard: string[] = []; + let startupCleanupFullStateOpId: string | undefined; // Check for piggybacked SYNC_IMPORT — mirrors the download path check (lines 552-604). // Without this, a SYNC_IMPORT from another client arriving as a piggybacked op // would silently replace local state via processRemoteOps(). @@ -323,7 +330,8 @@ export class OperationLogSyncService { // against the import (SyncImportFilterService). Only reachable in the narrow window // where example tasks are created on a still-empty server and uploaded just as a // remote import arrives; afterInitialSyncDoneStrict$ shrinks it further. - await this._discardExampleTaskOps(piggybackedConflict.discardablePendingOpIds); + startupOpIdsToDiscard = piggybackedConflict.discardablePendingOpIds; + startupCleanupFullStateOpId = fullStateOp.id; OpLog.normal( `OperationLogSyncService: Accepting piggybacked ${fullStateOp.opType} from client ` + `${fullStateOp.clientId} without conflict dialog; ` + @@ -332,8 +340,10 @@ export class OperationLogSyncService { } } - const processResult = await this.remoteOpsProcessingService.processRemoteOps( + const processResult = await this._processRemoteOpsWithStartupCleanup( result.piggybackedOps, + startupCleanupFullStateOpId, + startupOpIdsToDiscard, ); localWinOpsCreated = processResult.localWinOpsCreated; // Validation failure (if any) is on the session-validation latch. @@ -559,9 +569,9 @@ export class OperationLogSyncService { const hasLocalChanges = unsyncedOps.length > 0; // Collected here, applied AFTER hydrateFromRemoteSync succeeds so a - // hydration failure doesn't permanently drop the pending example-create - // ops while leaving the user without the remote snapshot. - let exampleTaskOpIdsToDiscard: string[] = []; + // hydration failure doesn't permanently drop discardable startup ops + // while leaving the user without the remote snapshot. + let startupOpIdsToDiscard: string[] = []; if (hasLocalChanges) { // Throw LocalDataConflictError if unsynced ops contain meaningful user data @@ -585,12 +595,23 @@ export class OperationLogSyncService { .map((entry) => entry.op.entityId) .filter((id): id is string => id !== undefined), ); - const exampleTaskOpIds = exampleTaskEntries.map((entry) => entry.op.id); // Nothing from this sync is persisted yet, so this live read reflects // whether the client completed a prior sync cycle. + const isNeverSyncedAtSyncStart = + options?.isNeverSynced ?? !(await this.opLogStore.hasSyncedOps()); + const pendingOpClassification = { + hasCompletedInitialSync: !isNeverSyncedAtSyncStart, + }; + const discardableStartupOpIds = + this.syncImportConflictGateService.getDiscardablePendingOpIds( + unsyncedOps, + pendingOpClassification, + ); const hasMeaningfulUserData = - this.syncImportConflictGateService.hasMeaningfulPendingOps(unsyncedOps) || - this.syncLocalStateService.hasMeaningfulStoreData(exampleTaskIds); + this.syncImportConflictGateService.hasMeaningfulPendingOps( + unsyncedOps, + pendingOpClassification, + ) || this.syncLocalStateService.hasMeaningfulStoreData(exampleTaskIds); if (hasMeaningfulUserData) { // SPAP-9: before surfacing the binary USE_LOCAL/USE_REMOTE dialog, use @@ -674,15 +695,15 @@ export class OperationLogSyncService { // gate === 'apply-snapshot': the remote snapshot strictly dominates the // local clock, so local holds nothing the snapshot lacks. Adopt the // snapshot without a dialog by falling through to hydration below. - exampleTaskOpIdsToDiscard = exampleTaskOpIds; + startupOpIdsToDiscard = discardableStartupOpIds; OpLog.normal( 'OperationLogSyncService: Remote snapshot strictly ahead of local clock — ' + 'applying snapshot without conflict dialog.', ); } else { // Defer the markRejected call until hydration has succeeded — see - // the declaration of exampleTaskOpIdsToDiscard above for rationale. - exampleTaskOpIdsToDiscard = exampleTaskOpIds; + // the declaration of startupOpIdsToDiscard above for rationale. + startupOpIdsToDiscard = discardableStartupOpIds; // Only system/config ops AND no meaningful store data - proceed with download OpLog.normal( `OperationLogSyncService: Client has ${unsyncedOps.length} unsynced ops but no meaningful user data. ` + @@ -747,10 +768,10 @@ export class OperationLogSyncService { ); // Now that the remote snapshot is applied, it's safe to drop the - // example-create ops we previously decided were obsolete. Doing this + // startup ops we previously decided were obsolete. Doing this // after hydration ensures a hydration failure leaves the queue intact // so the next attempt can retry. - await this._discardExampleTaskOps(exampleTaskOpIdsToDiscard); + await this._discardStartupOps(startupOpIdsToDiscard); // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. // File-based providers return ALL recentOps on every download, relying on @@ -889,6 +910,8 @@ export class OperationLogSyncService { isNeverSynced: options?.isNeverSynced, }, ); + let startupOpIdsToDiscard: string[] = []; + let startupCleanupFullStateOpId: string | undefined; if (incomingConflict.fullStateOp) { const { fullStateOp, pendingOps, dialogData } = incomingConflict; // Existing synced store data is not a conflict here. Prompt only when @@ -915,7 +938,8 @@ export class OperationLogSyncService { // the session-validation latch — wrapper reads it. (#7330) return { kind: 'no_new_ops' }; } else { - await this._discardExampleTaskOps(incomingConflict.discardablePendingOpIds); + startupOpIdsToDiscard = incomingConflict.discardablePendingOpIds; + startupCleanupFullStateOpId = fullStateOp.id; OpLog.normal( `OperationLogSyncService: Accepting incoming ${fullStateOp.opType} from client ` + `${fullStateOp.clientId} without conflict dialog; ` + @@ -924,8 +948,10 @@ export class OperationLogSyncService { } } - const processResult = await this.remoteOpsProcessingService.processRemoteOps( + const processResult = await this._processRemoteOpsWithStartupCleanup( result.newOps, + startupCleanupFullStateOpId, + startupOpIdsToDiscard, ); if (processResult.blockedByIncompatibleOp) { @@ -1010,17 +1036,74 @@ export class OperationLogSyncService { }; } + private async _processRemoteOpsWithStartupCleanup( + remoteOps: Operation[], + fullStateOpId: string | undefined, + startupOpIds: string[], + ): Promise { + try { + const result = await this.remoteOpsProcessingService.processRemoteOps(remoteOps); + await this._discardStartupOpsIfFullStateCommitted( + fullStateOpId, + startupOpIds, + result.committedFullStateOpIds, + ); + return result; + } catch (error) { + try { + // The reducer/apply transaction can commit the full-state op before a + // later validation or deferred-action drain throws. Query persistence + // so obsolete startup ops cannot replay after an already-applied import. + await this._discardStartupOpsIfFullStateCommitted( + fullStateOpId, + startupOpIds, + [], + true, + ); + } catch (cleanupError) { + // Preserve the primary processing error. A later retry can re-check and + // clean up once persistence is available again. + OpLog.err( + 'OperationLogSyncService: Failed to verify startup-op cleanup after remote processing error.', + { name: (cleanupError as Error | undefined)?.name }, + ); + } + throw error; + } + } + + private async _discardStartupOpsIfFullStateCommitted( + fullStateOpId: string | undefined, + startupOpIds: string[], + committedFullStateOpIds: string[] = [], + acceptReducerCommittedFailureStatus: boolean = false, + ): Promise { + if (!fullStateOpId || startupOpIds.length === 0) { + return; + } + + const applicationStatus = (await this.opLogStore.getOpById(fullStateOpId)) + ?.applicationStatus; + const isCommitted = + committedFullStateOpIds.includes(fullStateOpId) || + applicationStatus === 'applied' || + (acceptReducerCommittedFailureStatus && + (applicationStatus === 'archive_pending' || applicationStatus === 'failed')); + if (isCommitted) { + await this._discardStartupOps(startupOpIds); + } + } + /** - * Rejects the auto-generated startup example-task ops so they are NOT uploaded - * after a SYNC_IMPORT is accepted silently. They were already excluded from the - * conflict gate's "meaningful work" check (see SyncImportConflictGateService); the - * import replaces local state, so rejecting them keeps the op-log consistent with - * the just-applied remote data instead of re-uploading throwaway onboarding tasks. + * Rejects startup-only ops so they are NOT uploaded after an authoritative remote + * state is accepted silently. They were already excluded from the conflict gate's + * "meaningful work" check (see SyncImportConflictGateService); rejecting them keeps + * the op-log consistent with the just-applied remote data. * * These ids always come from getUnsynced() (local pending ops, never remote ops), - * so a remote `isExampleTask` flag can never reach this path. + * so a remote startup marker can never reach this path. */ - private async _discardExampleTaskOps(opIds: string[]): Promise { + private async _discardStartupOps(opIds: string[]): Promise { if (opIds.length > 0) { await this.opLogStore.markRejected(opIds); } @@ -1260,8 +1343,24 @@ export class OperationLogSyncService { const migratedRemoteOps = this._preflightRemoteOperations(result.newOps); + const currentSyncConfig = await firstValueFrom(this.store.select(selectSyncConfig)); + const localOnlySyncSettings: LocalOnlySyncSettings = { + isEnabled: currentSyncConfig.isEnabled, + isEncryptionEnabled: currentSyncConfig.isEncryptionEnabled, + syncProvider: currentSyncConfig.syncProvider, + syncInterval: currentSyncConfig.syncInterval, + isManualSyncOnly: currentSyncConfig.isManualSyncOnly, + }; + let snapshotState = result.snapshotState as Record | undefined; if (hasSnapshotState && snapshotState) { + // File providers intentionally omit device-local schedule fields and null + // the provider on the wire. Restore this device's values before schema + // validation so a valid transport snapshot is locally replayable. + snapshotState = applyLocalOnlySyncSettingsToAppData( + stripLocalOnlySyncSettingsFromAppData(snapshotState), + localOnlySyncSettings, + ) as Record; const validation = await this.validateStateService.validateAndRepair(snapshotState); if (!validation.isValid) { throw new Error( @@ -1298,14 +1397,6 @@ export class OperationLogSyncService { baselineGlobalConfig['sync'] && typeof baselineGlobalConfig['sync'] === 'object' ? (baselineGlobalConfig['sync'] as Record) : {}; - const currentSyncConfig = await firstValueFrom(this.store.select(selectSyncConfig)); - const localOnlySyncSettings: LocalOnlySyncSettings = { - isEnabled: currentSyncConfig.isEnabled, - isEncryptionEnabled: currentSyncConfig.isEncryptionEnabled, - syncProvider: currentSyncConfig.syncProvider, - syncInterval: currentSyncConfig.syncInterval, - isManualSyncOnly: currentSyncConfig.isManualSyncOnly, - }; // getDefaultMainModelData intentionally excludes globalConfig. Add a // default config shell before applying the canonical device-local fields // so an interrupted rebuild can hydrate enough configuration to sync again. diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index 597bffee63..d427953960 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -611,6 +611,40 @@ describe('RemoteOpsProcessingService', () => { expect(result.blockedByIncompatibleOp).toBe(true); }); + it('should report a committed full-state prefix before a blocked suffix', async () => { + const repair: Operation = { + id: 'repair-prefix', + opType: OpType.Repair, + actionType: ActionType.REPAIR_AUTO, + entityType: 'ALL', + payload: {}, + clientId: 'client-1', + vectorClock: { client1: 1 }, + timestamp: Date.now(), + schemaVersion: 1, + }; + const futureOp: Operation = { + id: 'future-suffix', + opType: OpType.Update, + actionType: ActionType.TASK_SHARED_UPDATE, + entityType: 'TASK', + entityId: 'task-1', + payload: {}, + clientId: 'client-2', + vectorClock: { client2: 1 }, + timestamp: Date.now(), + schemaVersion: 2, + }; + opLogStoreSpy.hasOp.and.resolveTo(false); + opLogStoreSpy.append.and.resolveTo(1); + + const result = await service.processRemoteOps([repair, futureOp]); + + expect(result.blockedByIncompatibleOp).toBeTrue(); + expect(result.committedFullStateOpIds).toEqual([repair.id]); + expect(operationApplierServiceSpy.applyOperations).toHaveBeenCalled(); + }); + it('should show error snackbar and abort if version is below minimum supported', async () => { const remoteOps: Operation[] = [ { id: 'op1', schemaVersion: MIN_SUPPORTED_SCHEMA_VERSION - 1 } as Operation, diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index 69bd8f79f2..4d5a92a8d8 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -7,9 +7,9 @@ import { ConflictResult, EntityConflict, extractFullStateFromPayload, + FULL_STATE_OP_TYPES, isWrappedFullStatePayload, Operation, - OpType, VectorClock, } from '../core/operation.types'; import { OpLog } from '../../core/log'; @@ -105,6 +105,12 @@ export class RemoteOpsProcessingService { filteringImport?: Operation; isLocalUnsyncedImport: boolean; blockedByIncompatibleOp: boolean; + /** + * Full-state operations whose application completed (or was already + * deduplicated as applied) before this result was returned. Populated even + * when a later incompatible op blocks the remaining batch suffix. + */ + committedFullStateOpIds?: string[]; }> { // Validation failure surfaces via the SyncSessionValidationService latch // (#7330). `validateAfterSync` and the conflict-resolution validation path @@ -248,18 +254,16 @@ export class RemoteOpsProcessingService { } // ───────────────────────────────────────────────────────────────────────── - // STEP 3: Check for full-state operations (SYNC_IMPORT / BACKUP_IMPORT) + // STEP 3: Check for full-state operations // These replace the entire state, so conflict detection doesn't apply. // ───────────────────────────────────────────────────────────────────────── - const hasFullStateOp = validOps.some( - (op) => op.opType === OpType.SyncImport || op.opType === OpType.BackupImport, - ); + const hasFullStateOp = validOps.some((op) => FULL_STATE_OP_TYPES.has(op.opType)); if (hasFullStateOp) { OpLog.normal( 'RemoteOpsProcessingService: Full-state operation detected, skipping conflict detection.', ); - await this.applyNonConflictingOps(validOps); + const committedFullStateOpIds = await this.applyNonConflictingOps(validOps); // Clean Slate Semantics: SYNC_IMPORT/BACKUP_IMPORT replaces entire state. // Local synced ops are NOT replayed - the import is an explicit user action @@ -272,6 +276,7 @@ export class RemoteOpsProcessingService { filteredOpCount: 0, isLocalUnsyncedImport: false, blockedByIncompatibleOp, + committedFullStateOpIds, }; } @@ -434,7 +439,7 @@ export class RemoteOpsProcessingService { async applyNonConflictingOps( ops: Operation[], callerHoldsLock: boolean = false, - ): Promise { + ): Promise { const locallyReplayableOps = await this._withLocalOnlySyncSettingsForFullStateOps(ops); @@ -447,6 +452,7 @@ export class RemoteOpsProcessingService { // to leak into the next sync window with stale clocks. (#7700) let didApplyRemoteOps = false; let primaryIncompleteError: IncompleteRemoteOperationsError | undefined; + let committedFullStateOpIds: string[] = []; try { // Core owns the generic crash-safety ordering. Angular diagnostics, // validation, and user notifications stay in this service. @@ -464,6 +470,9 @@ export class RemoteOpsProcessingService { }); didApplyRemoteOps = result.appendedOps.length > 0; + committedFullStateOpIds = result.appendedOps + .filter((op) => FULL_STATE_OP_TYPES.has(op.opType)) + .map((op) => op.id); if (result.skippedCount > 0) { OpLog.verbose( @@ -526,6 +535,7 @@ export class RemoteOpsProcessingService { } } } + return committedFullStateOpIds; } private async _withLocalOnlySyncSettingsForFullStateOps( @@ -569,11 +579,7 @@ export class RemoteOpsProcessingService { } private _isFullStateOperation(op: Operation): boolean { - return ( - op.opType === OpType.SyncImport || - op.opType === OpType.BackupImport || - op.opType === OpType.Repair - ); + return FULL_STATE_OP_TYPES.has(op.opType); } /** diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts index 30d1a2fdcc..a332d53d6a 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.spec.ts @@ -118,13 +118,38 @@ describe('SyncImportConflictGateService', () => { expect(result.dialogData).toBeDefined(); }); - it('should protect a pending sync config change on a never-synced client', async () => { + it('should ignore the sync setup write on a never-synced client', async () => { opLogStoreSpy.hasSyncedOps.and.resolveTo(false); opLogStoreSpy.getUnsynced.and.resolveTo([createPendingConfigEntry()]); const result = await service.checkIncomingFullStateConflict([createOperation()]); + expect(result.hasMeaningfulPending).toBeFalse(); + expect(result.discardablePendingOpIds).toEqual(['local-config-update']); + expect(result.dialogData).toBeUndefined(); + }); + + it('should protect non-update operations targeting the sync config coordinates', async () => { + opLogStoreSpy.hasSyncedOps.and.resolveTo(false); + opLogStoreSpy.getUnsynced.and.resolveTo([ + createEntry( + createOperation({ + id: 'local-config-delete', + actionType: ActionType.GLOBAL_CONFIG_UPDATE_SECTION, + opType: OpType.Delete, + entityType: 'GLOBAL_CONFIG', + entityId: 'sync', + payload: { sectionKey: 'sync' }, + clientId: 'client-A', + vectorClock: { clientA: 1 }, + }), + ), + ]); + + const result = await service.checkIncomingFullStateConflict([createOperation()]); + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.discardablePendingOpIds).toEqual([]); expect(result.dialogData).toBeDefined(); }); diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index 1b31481a8a..5b2abd6763 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -1,6 +1,7 @@ import { inject, Injectable } from '@angular/core'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { + ActionType, FULL_STATE_OP_TYPES, Operation, OperationLogEntry, @@ -10,6 +11,21 @@ import { OperationWriteFlushService } from './operation-write-flush.service'; import { SyncImportConflictData } from './dialog-sync-import-conflict/dialog-sync-import-conflict.component'; import { isExampleTaskCreateOp } from '../validation/is-example-task-op.util'; +/** + * Enabling/configuring sync on a never-synced client necessarily writes the + * sync config section before the first download. That setup-only operation is + * not local user data divergence. Other GLOBAL_CONFIG sections remain protected. + */ +const isInitialSyncSetupOp = (entry: OperationLogEntry): boolean => + entry.op.actionType === ActionType.GLOBAL_CONFIG_UPDATE_SECTION && + entry.op.opType === OpType.Update && + entry.op.entityType === 'GLOBAL_CONFIG' && + entry.op.entityId === 'sync'; + +interface PendingOpClassificationOptions { + hasCompletedInitialSync: boolean; +} + export interface IncomingFullStateConflictGateResult { fullStateOp?: Operation; pendingOps: OperationLogEntry[]; @@ -34,14 +50,20 @@ export class SyncImportConflictGateService { private writeFlushService = inject(OperationWriteFlushService); /** - * Every pending op is user work unless it is an onboarding example-task create. + * Every pending op is user work unless it is an onboarding example-task create, + * or the sync-section setup write on a client that has never completed sync. * Full-state ops are always meaningful because applying a newer full-state op * can invalidate their local import/repair semantics. * * The lifecycle default is deliberately conservative: a caller that does not * know whether initial sync completed must protect startup-entity changes. */ - hasMeaningfulPendingOps(ops: OperationLogEntry[]): boolean { + hasMeaningfulPendingOps( + ops: OperationLogEntry[], + options: PendingOpClassificationOptions = { + hasCompletedInitialSync: true, + }, + ): boolean { return ops.some((entry) => { if (FULL_STATE_OP_TYPES.has(entry.op.opType as OpType)) { return true; @@ -49,10 +71,26 @@ export class SyncImportConflictGateService { if (isExampleTaskCreateOp(entry)) { return false; } + if (isInitialSyncSetupOp(entry)) { + return options.hasCompletedInitialSync; + } return true; }); } + getDiscardablePendingOpIds( + ops: OperationLogEntry[], + options: PendingOpClassificationOptions, + ): string[] { + return ops + .filter( + (entry) => + isExampleTaskCreateOp(entry) || + (!options.hasCompletedInitialSync && isInitialSyncSetupOp(entry)), + ) + .map((entry) => entry.op.id); + } + /** * @param options.preCapturedPendingOps - Exact pending ops selected by the upload * round. The piggyback path unions this snapshot with a live read so it @@ -89,14 +127,6 @@ export class SyncImportConflictGateService { ? this._mergePendingOps(options.preCapturedPendingOps, livePendingOps) : livePendingOps; - // Example-task ops that the caller may reject when it accepts the import silently. - // These must come from a LIVE read: with a pre-captured snapshot, example ops - // accepted earlier in the same upload round are already marked synced and must - // not be re-marked rejected by the caller. - const discardablePendingOpIds = livePendingOps - .filter(isExampleTaskCreateOp) - .map((entry) => entry.op.id); - // Preserve the cheap example-task-only path. If nothing could be meaningful // even for a synced client, there is no reason to read sync history. const canContainMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); @@ -105,13 +135,28 @@ export class SyncImportConflictGateService { fullStateOp, pendingOps, hasMeaningfulPending: false, - discardablePendingOpIds, + discardablePendingOpIds: this.getDiscardablePendingOpIds(livePendingOps, { + hasCompletedInitialSync: true, + }), }; } const isNeverSynced = options.isNeverSynced ?? !(await this.opLogStore.hasSyncedOps()); - const hasMeaningfulPending = this.hasMeaningfulPendingOps(pendingOps); + const classificationOptions = { + hasCompletedInitialSync: !isNeverSynced, + }; + const hasMeaningfulPending = this.hasMeaningfulPendingOps( + pendingOps, + classificationOptions, + ); + // Only live pending ops may be rejected. A pre-captured upload snapshot can + // include ops already acknowledged by the server, which must not be rewritten + // as rejected locally. + const discardablePendingOpIds = this.getDiscardablePendingOpIds( + livePendingOps, + classificationOptions, + ); const result = { fullStateOp, diff --git a/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts b/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts index cadfde53b1..597239bc6b 100644 --- a/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts +++ b/src/app/op-log/testing/integration/example-task-import-gate.integration.spec.ts @@ -36,13 +36,13 @@ describe('Example-task SYNC_IMPORT gate (integration)', () => { }, }); - const configOp = (): Operation => + const configOp = (sectionKey = 'sync'): Operation => local.createOperation({ actionType: ActionType.GLOBAL_CONFIG_UPDATE_SECTION, opType: OpType.Update, entityType: 'GLOBAL_CONFIG', - entityId: 'sync', - payload: { sectionKey: 'sync' }, + entityId: sectionKey, + payload: { sectionKey }, }); const incomingSyncImport = (): Operation => @@ -89,24 +89,20 @@ describe('Example-task SYNC_IMPORT gate (integration)', () => { ); }); - it('treats a pending config change as meaningful user work (widened gate) and shows the dialog', async () => { - // The gate deliberately has NO entity-wide exemptions beyond example-task - // creates: GLOBAL_CONFIG carries synced preferences, so silently discarding - // a pending config op on an incoming SYNC_IMPORT would lose user work. + it('treats the never-synced provider setup write as discardable startup work', async () => { const example = exampleTaskOp('example-task-1'); - await storeService.append(configOp(), 'local'); + const config = configOp(); + await storeService.append(config, 'local'); await storeService.append(example, 'local'); const result = await gate.checkIncomingFullStateConflict([incomingSyncImport()]); - expect(result.hasMeaningfulPending).toBeTrue(); - expect(result.dialogData).toBeDefined(); - // Example ops stay listed so the caller can discard them if the user - // accepts the import. - expect(result.discardablePendingOpIds).toEqual([example.id]); + expect(result.hasMeaningfulPending).toBeFalse(); + expect(result.dialogData).toBeUndefined(); + expect(result.discardablePendingOpIds.sort()).toEqual([config.id, example.id].sort()); }); - it('actually excludes example-task ops from getUnsynced after markRejected (so they are not uploaded)', async () => { + it('actually excludes discardable startup ops from getUnsynced after markRejected', async () => { const config = configOp(); const example = exampleTaskOp('example-task-1'); await storeService.append(config, 'local'); @@ -116,10 +112,21 @@ describe('Example-task SYNC_IMPORT gate (integration)', () => { await storeService.markRejected(result.discardablePendingOpIds); const remaining = (await storeService.getUnsynced()).map((e) => e.op.id); - expect(remaining).toContain(config.id); + expect(remaining).not.toContain(config.id); expect(remaining).not.toContain(example.id); }); + it('keeps another config section meaningful on a never-synced client', async () => { + const config = configOp('productivityHacks'); + await storeService.append(config, 'local'); + + const result = await gate.checkIncomingFullStateConflict([incomingSyncImport()]); + + expect(result.hasMeaningfulPending).toBeTrue(); + expect(result.dialogData).toBeDefined(); + expect(result.discardablePendingOpIds).toEqual([]); + }); + it('shows the dialog (and still lists the example id) when real user work is also pending', async () => { const example = exampleTaskOp('example-task-1'); const realTaskUpdate = local.createOperation({ From 01d12cacb77b109792fc04c12f5988776cb32c96 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 13:18:03 +0200 Subject: [PATCH 32/47] test(sync): target transient rejection assertion --- ...jected-ops-transient-download-8331.spec.ts | 62 +++++++++++++------ 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/e2e/tests/sync/supersync-rejected-ops-transient-download-8331.spec.ts b/e2e/tests/sync/supersync-rejected-ops-transient-download-8331.spec.ts index 9c2a73bbc4..233507e36d 100644 --- a/e2e/tests/sync/supersync-rejected-ops-transient-download-8331.spec.ts +++ b/e2e/tests/sync/supersync-rejected-ops-transient-download-8331.spec.ts @@ -10,29 +10,38 @@ import { type SimulatedE2EClient, } from '../../utils/supersync-helpers'; -// Reads the op-log store directly and counts terminally-rejected ops (rejectedAt -// set). This is the precise, timing-independent discriminator for #8331: the -// transient blip must NOT have rejected the pending edit. -const countRejectedOps = (page: Page): Promise => - page.evaluate(async () => { +// Reads the exact server-rejected op from the op-log. This is the precise, +// timing-independent discriminator for #8331: the transient blip must leave +// that edit pending, regardless of unrelated startup-only op cleanup. +const isOpPending = (page: Page, opId: string): Promise => + page.evaluate(async (id) => { const db = await new Promise((resolve, reject) => { const req = indexedDB.open('SUP_OPS'); req.onsuccess = () => resolve(req.result); req.onerror = () => reject(req.error); }); try { - if (!db.objectStoreNames.contains('ops')) return 0; - const entries = await new Promise<{ rejectedAt?: number }[]>((resolve, reject) => { + if (!db.objectStoreNames.contains('ops')) return false; + const entry = await new Promise< + { syncedAt?: number; rejectedAt?: number } | undefined + >((resolve, reject) => { const tx = db.transaction('ops', 'readonly'); - const r = tx.objectStore('ops').getAll(); - r.onsuccess = () => resolve(r.result as { rejectedAt?: number }[]); - r.onerror = () => reject(r.error); + const request = tx.objectStore('ops').index('byId').get(id); + request.onsuccess = () => + resolve( + request.result as { syncedAt?: number; rejectedAt?: number } | undefined, + ); + request.onerror = () => reject(request.error); }); - return entries.filter((e) => e.rejectedAt).length; + return ( + entry !== undefined && + entry.syncedAt === undefined && + entry.rejectedAt === undefined + ); } finally { db.close(); } - }); + }, opId); /** * Regression: transient download failure during rejected-ops resolution must @@ -45,11 +54,10 @@ const countRejectedOps = (page: Page): Promise => * locally but never reached other devices. The fix leaves the op pending so it * re-resolves on the next sync. * - * Discriminator: after the blip, B's pending edit must NOT be terminally - * rejected (op-log has zero rejectedAt ops) — with the bug the catch called - * markRejected() so that count is >= 1. End-to-end: once the fault clears, B's - * edit resolves and both clients converge to the same title; with the bug B's - * edit is dropped and the clients diverge. + * Discriminator: after the blip, the exact CONFLICT_CONCURRENT op must still be + * pending. With the bug, the catch called markRejected() on that op. End-to-end: + * once the fault clears, B's edit resolves and both clients converge to the + * same title; with the bug B's edit is dropped and the clients diverge. * * Prerequisites: * - super-sync-server running on localhost:1901 with TEST_MODE=true @@ -153,6 +161,7 @@ test.describe('@supersync Rejected-ops transient download (#8331)', () => { let abortedResolutionGets = 0; let strippedPreUploadDownloads = 0; let strippedConflictPiggybacks = 0; + let concurrentRejectedOpId: string | null = null; await clientB.page.route('**/api/sync/ops*', async (route) => { const method = route.request().method(); if (method === 'GET') { @@ -193,6 +202,15 @@ test.describe('@supersync Rejected-ops transient download (#8331)', () => { sawUploadAttempt = true; const response = await route.fetch(); const json = await response.json(); + const concurrentResult = Array.isArray(json.results) + ? json.results.find( + (result: { errorCode?: unknown }) => + result.errorCode === 'CONFLICT_CONCURRENT', + ) + : undefined; + if (typeof concurrentResult?.opId === 'string') { + concurrentRejectedOpId = concurrentResult.opId; + } if (Array.isArray(json.newOps) && json.newOps.length > 0) { json.newOps = []; json.hasMorePiggyback = false; @@ -226,11 +244,15 @@ test.describe('@supersync Rejected-ops transient download (#8331)', () => { expect(strippedPreUploadDownloads).toBeGreaterThanOrEqual(1); expect(strippedConflictPiggybacks).toBeGreaterThanOrEqual(1); expect(abortedResolutionGets).toBeGreaterThanOrEqual(1); + expect(concurrentRejectedOpId).not.toBeNull(); + if (!concurrentRejectedOpId) { + throw new Error('Expected a CONFLICT_CONCURRENT upload result'); + } // Precise discriminator (timing-independent): the blip must not have - // terminally rejected B's pending edit. With the bug, the catch called - // markRejected() so this is >= 1; with the fix the edit stays pending -> 0. - expect(await countRejectedOps(clientB.page)).toBe(0); + // terminally rejected B's exact pending edit. With the bug, the catch + // called markRejected(); with the fix the entry remains pending. + expect(await isOpPending(clientB.page, concurrentRejectedOpId)).toBe(true); // 7. Remove the fault and let B sync cleanly — the still-pending edit // resolves and uploads (the merged op may need a second flush). From f9610530c90de3ca303399a302c803b4e3633db9 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 14:49:08 +0200 Subject: [PATCH 33/47] fix(sync): converge rebuild capture races, unwedge archive recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remediate the findings of the 5-agent necessity review: - USE_REMOTE: a local capture racing the locked rebuild now throws a typed CaptureRacedRebuildError, and forceDownloadRemoteState retries phase 2 in-call (bounded, 3 attempts) through the existing crash-resume branch — raced ops fold into preservedLocalOps and the already-downloaded history is reused (WS downloads and immediate uploads stay gated by the marker, so no re-download is needed). Previously every attempt aborted while e.g. time tracking dispatched continuously, re-downloading the full history per sync trigger and churning the Undo snack (now shown on final failure only). - Hydrator archive retry: pass skipDeferredLocalActions and drain explicitly with a caught error. A drain throw from the coordinator's finally used to mask the archive result and escalate out of hydrateStore() into attemptRecovery(), which can import stale legacy data over a correctly hydrated store. - Incomplete-remote gate: run one in-session archive-only retry when the only blockers are quarantined failed/archive_pending ops, so a transient archive failure self-heals on the next sync instead of wedging until app restart. Never attempted while the rebuild marker is set or reducer-uncommitted pending rows exist. - Boot recovery: StartupService surfaces the pre-replace backup's persistent restore snack when a stranded raw_rebuild_incomplete marker is found, covering users who boot offline or disable sync after finding the app "emptied" by a mid-rebuild crash. - Snack correctness: latch the USE_REMOTE newer-schema warning once per session; guard _notifyBlockedOp and the LWW apply-failure snack with hasPendingPersistentAction() so they cannot destroy a visible Undo; drop the useless "Update app" action for below-minimum data. - Strings: MIGRATION_FAILED / VERSION_UNSUPPORTED now describe the blocking semantics instead of the removed skip behavior. - Store: getPendingRemoteOps excludes rejected rows (parity with getFailedRemoteOps) so a rejected-but-pending row cannot trip the sync gate for a session. - Server: compute the upload request fingerprint eagerly after the rate-limit gate — identical cost to the lazy closure in every path, minus the memoization machinery; laziness remains in the snapshot handler where it skips hashing multi-MB states. op-log suite 3004/3004, super-sync-server 831/831, checkFile clean on all touched files. Nine regression tests pin the new behavior. --- .../src/sync/sync.routes.ops-handler.ts | 40 +- src/app/core/startup/startup.service.ts | 26 ++ src/app/op-log/core/errors/sync-errors.ts | 17 + ...ion-log-hydrator.retry.integration.spec.ts | 1 + .../operation-log-hydrator.service.spec.ts | 36 +- .../operation-log-hydrator.service.ts | 98 +++-- .../operation-log-store.service.spec.ts | 13 + .../operation-log-store.service.ts | 6 +- .../sync/conflict-resolution.service.spec.ts | 6 +- .../sync/conflict-resolution.service.ts | 22 +- .../sync/operation-log-sync.service.spec.ts | 97 +++++ .../op-log/sync/operation-log-sync.service.ts | 395 +++++++++++------- .../remote-ops-processing.service.spec.ts | 20 +- .../sync/remote-ops-processing.service.ts | 31 +- .../sync/sync-import-conflict-gate.service.ts | 10 +- .../migration-handling.integration.spec.ts | 6 +- src/assets/i18n/en.json | 4 +- 17 files changed, 584 insertions(+), 244 deletions(-) diff --git a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts index 29f4e15c3a..568ad1c689 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts @@ -87,15 +87,6 @@ export const uploadOpsHandler = async ( } const { ops, clientId, lastKnownServerSeq, requestId } = parseResult.data; - // Lazy + memoized: hashing the full ops payload is expensive and must not - // run before the rate-limit gate below, nor on first-time requests — the - // dedup check only invokes it when an entry for this requestId exists. - let memoizedFingerprint: string | undefined; - const getRequestFingerprint = (): string => - (memoizedFingerprint ??= createOpsRequestFingerprint( - clientId, - ops as unknown as Operation[], - )); const syncService = getSyncService(); Logger.info( @@ -118,14 +109,25 @@ export const uploadOpsHandler = async ( }); } + // Compute the request fingerprint AFTER the rate-limit gate (a rate-limited + // client must not burn CPU on it) and BEFORE any processing: uploadOps and + // the quota prevalidation mutate the parsed ops (e.g. vector-clock + // sanitizing/pruning), so hashing later would never match a retry's + // pre-processing hash. Eager is as cheap as lazy here — every non-limited + // requestId-bearing request needs the hash exactly once (either the dedup + // check below or the post-upload cache write). + const requestFingerprint = requestId + ? createOpsRequestFingerprint(clientId, ops as unknown as Operation[]) + : undefined; + // Check for duplicate request (client retry) BEFORE quota check // This ensures retries after successful uploads don't fail with 413 // if the original upload pushed the user over quota - if (requestId) { + if (requestId && requestFingerprint) { const cachedResults = syncService.checkOpsRequestDedup( userId, requestId, - getRequestFingerprint, + () => requestFingerprint, ); if (cachedResults) { Logger.info(`[user:${userId}] Returning cached results for request ${requestId}`); @@ -176,14 +178,6 @@ export const uploadOpsHandler = async ( } } - // Pin the fingerprint BEFORE processing: uploadOps mutates the parsed ops - // (e.g. vector-clock pruning), so hashing after it would never match the - // retry's pre-processing hash. Still after the rate-limit gate above, so a - // rate-limited client cannot burn CPU on it. - if (requestId) { - getRequestFingerprint(); - } - const results = await syncService.runWithStorageUsageLock( userId, async () => { @@ -242,14 +236,10 @@ export const uploadOpsHandler = async ( // batch to it, so a single match means the transaction failed. #8332 if ( requestId && + requestFingerprint && !results.some((r) => r.errorCode === SYNC_ERROR_CODES.INTERNAL_ERROR) ) { - syncService.cacheOpsRequestResults( - userId, - requestId, - results, - getRequestFingerprint(), - ); + syncService.cacheOpsRequestResults(userId, requestId, results, requestFingerprint); } const accepted = results.filter((r) => r.accepted).length; diff --git a/src/app/core/startup/startup.service.ts b/src/app/core/startup/startup.service.ts index 2fe1bf006d..8623841fda 100644 --- a/src/app/core/startup/startup.service.ts +++ b/src/app/core/startup/startup.service.ts @@ -14,6 +14,7 @@ import { IS_ELECTRON } from '../../app.constants'; import { Log } from '../log'; import { T } from '../../t.const'; import { OperationLogStoreService } from '../../op-log/persistence/operation-log-store.service'; +import { OperationLogSyncService } from '../../op-log/sync/operation-log-sync.service'; import { LegacyPfDbService } from '../persistence/legacy-pf-db.service'; import { BannerId } from '../banner/banner.model'; import { isOnline$ } from '../../util/is-online'; @@ -169,6 +170,9 @@ export class StartupService { this._ratePromptService.init(); await this._initPlugins(); + // Last in the deferred body: the snack it may open is persistent and the + // single snack slot must not be reclaimed by the productivity tip above. + await this._offerInterruptedRebuildRecoveryIfNeeded(); }, DEFERRED_INIT_DELAY_MS); if (IS_ELECTRON) { @@ -258,6 +262,28 @@ export class StartupService { } } + /** + * An interrupted USE_REMOTE rebuild leaves the user booting into the rebuild + * baseline instead of their data. Sync (when it runs) resumes the rebuild by + * itself — but when it cannot (offline, or the user disabled sync after + * finding the app "emptied" by the crash), the pre-replace backup would have + * no visible entry point. Surfaces the persistent restore snack in that case. + */ + private async _offerInterruptedRebuildRecoveryIfNeeded(): Promise { + try { + if (await this._opLogStore.isRawRebuildIncomplete()) { + await this._injector + .get(OperationLogSyncService) + .offerInterruptedRebuildRecovery(); + } + } catch (err) { + Log.err({ + stage: 'interrupted-rebuild-recovery-check', + error: (err as Error)?.message, + }); + } + } + private async _checkIsSingleInstance(): Promise { const channel = new BroadcastChannel('superProductivityTab'); let isAnotherInstanceActive = false; diff --git a/src/app/op-log/core/errors/sync-errors.ts b/src/app/op-log/core/errors/sync-errors.ts index e145490b17..2df0ad44c3 100644 --- a/src/app/op-log/core/errors/sync-errors.ts +++ b/src/app/op-log/core/errors/sync-errors.ts @@ -122,6 +122,23 @@ export class PermanentDeferredWriteError extends Error { override name = 'PermanentDeferredWriteError'; } +/** + * A local action was captured while a USE_REMOTE rebuild held the op-log lock, + * after the destructive replacement committed. The attempt must abort — the + * raced action's reducer ran against live state the replay rewrites, so + * completing could let a later snapshot cover an op whose live effect is + * missing. The raced ops are preserved and re-applied by the retry/resume. + */ +export class CaptureRacedRebuildError extends Error { + override name = 'CaptureRacedRebuildError'; + + constructor() { + super( + 'USE_REMOTE incomplete: a local change arrived during the rebuild and will be restored on retry.', + ); + } +} + /** Previously downloaded operations have not completed reducer/archive recovery. */ export class IncompleteRemoteOperationsError extends Error { override name = 'IncompleteRemoteOperationsError'; diff --git a/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts index 1b42d0afd5..32ff621cdd 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.retry.integration.spec.ts @@ -135,6 +135,7 @@ describe('OperationLogHydratorService retryFailedRemoteOps (integration, real st // double-apply additive reducers. expect(applier.applyOperations.calls.argsFor(0)[1]).toEqual({ skipReducerDispatch: true, + skipDeferredLocalActions: true, }); }); diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts index a207c9d916..abd3dfb501 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.spec.ts @@ -31,6 +31,7 @@ import { MAX_VECTOR_CLOCK_SIZE } from '../core/operation-log.const'; import { IndexedDBOpenError } from '../core/errors/indexed-db-open.error'; import { IDB_OPEN_ERROR_RELOAD_KEY } from './operation-log-hydrator.service'; import { SyncProviderId } from '../sync-providers/provider.const'; +import { OperationLogEffects } from '../capture/operation-log.effects'; describe('OperationLogHydratorService', () => { let service: OperationLogHydratorService; @@ -44,6 +45,7 @@ describe('OperationLogHydratorService', () => { let mockRepairOperationService: jasmine.SpyObj; let mockVectorClockService: jasmine.SpyObj; let mockOperationApplierService: jasmine.SpyObj; + let mockOperationLogEffects: jasmine.SpyObj; let mockHydrationStateService: jasmine.SpyObj; let mockSnapshotService: jasmine.SpyObj; let mockRecoveryService: jasmine.SpyObj; @@ -139,6 +141,10 @@ describe('OperationLogHydratorService', () => { mockOperationApplierService = jasmine.createSpyObj('OperationApplierService', [ 'applyOperations', ]); + mockOperationLogEffects = jasmine.createSpyObj('OperationLogEffects', [ + 'processDeferredActions', + ]); + mockOperationLogEffects.processDeferredActions.and.resolveTo(); mockHydrationStateService = jasmine.createSpyObj('HydrationStateService', [ 'startApplyingRemoteOps', 'endApplyingRemoteOps', @@ -216,6 +222,7 @@ describe('OperationLogHydratorService', () => { { provide: RepairOperationService, useValue: mockRepairOperationService }, { provide: VectorClockService, useValue: mockVectorClockService }, { provide: OperationApplierService, useValue: mockOperationApplierService }, + { provide: OperationLogEffects, useValue: mockOperationLogEffects }, { provide: HydrationStateService, useValue: mockHydrationStateService }, { provide: OperationLogSnapshotService, useValue: mockSnapshotService }, { provide: OperationLogRecoveryService, useValue: mockRecoveryService }, @@ -478,7 +485,10 @@ describe('OperationLogHydratorService', () => { const [retriedOps, retryOptions] = mockOperationApplierService.applyOperations.calls.argsFor(0); expect(retriedOps.map((o: Operation) => o.id)).toEqual(['op-failed']); - expect(retryOptions).toEqual({ skipReducerDispatch: true }); + expect(retryOptions).toEqual({ + skipReducerDispatch: true, + skipDeferredLocalActions: true, + }); expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([6]); }); @@ -1457,7 +1467,10 @@ describe('OperationLogHydratorService', () => { await service.retryFailedRemoteOps(); const options = mockOperationApplierService.applyOperations.calls.argsFor(0)[1]; - expect(options).toEqual({ skipReducerDispatch: true }); + expect(options).toEqual({ + skipReducerDispatch: true, + skipDeferredLocalActions: true, + }); }); it('should apply failed ops in ascending seq order regardless of store order', async () => { @@ -1515,6 +1528,25 @@ describe('OperationLogHydratorService', () => { expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); }); + it('should not escalate a deferred-drain failure into hydration recovery', async () => { + // A drain throw here used to propagate out of hydrateStore() into + // attemptRecovery(), which can import stale legacy data over a correctly + // hydrated store. The failure is logged; buffered actions stay queued + // for the next drain point (e.g. the pre-sync flush). + mockOperationLogEffects.processDeferredActions.and.rejectWith( + new Error('drain failed'), + ); + mockOpLogStore.getFailedRemoteOps.and.resolveTo([failedEntry(40, 'op-a')]); + mockOperationApplierService.applyOperations.and.callFake((ops: Operation[]) => + Promise.resolve({ appliedOps: ops }), + ); + + await expectAsync(service.retryFailedRemoteOps()).toBeResolved(); + + // Bookkeeping completed despite the failed drain. + expect(mockOpLogStore.markApplied).toHaveBeenCalledWith([40]); + }); + it('should do nothing when there are no failed ops', async () => { mockOpLogStore.getFailedRemoteOps.and.returnValue(Promise.resolve([])); diff --git a/src/app/op-log/persistence/operation-log-hydrator.service.ts b/src/app/op-log/persistence/operation-log-hydrator.service.ts index 108d0f05a6..27386e7fd0 100644 --- a/src/app/op-log/persistence/operation-log-hydrator.service.ts +++ b/src/app/op-log/persistence/operation-log-hydrator.service.ts @@ -1,6 +1,7 @@ -import { inject, Injectable } from '@angular/core'; +import { inject, Injectable, Injector } from '@angular/core'; import { Store } from '@ngrx/store'; import { OperationLogStoreService } from './operation-log-store.service'; +import { processDeferredActions } from '../sync/process-deferred-actions-flush.util'; import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { OperationLogMigrationService } from './operation-log-migration.service'; import { @@ -54,6 +55,7 @@ export class OperationLogHydratorService { private vectorClockService = inject(VectorClockService); private operationApplierService = inject(OperationApplierService); private hydrationStateService = inject(HydrationStateService); + private injector = inject(Injector); private clientIdProvider: ClientIdProvider = inject(CLIENT_ID_PROVIDER); // Extracted services @@ -628,45 +630,67 @@ export class OperationLogHydratorService { // outstanding archive side effects: re-dispatching would double-apply // additive reducers (syncTimeSpent, increaseSimpleCounterCounterToday) // on every retry attempt. - const result = await this.operationApplierService.applyOperations(opsToApply, { - skipReducerDispatch: true, - }); + try { + const result = await this.operationApplierService.applyOperations(opsToApply, { + skipReducerDispatch: true, + // The drain runs in the finally below with its own error boundary. Left + // to the applier's finally, a drain throw would mask the archive result + // (markFailed below never runs) and escalate out of hydrateStore() into + // attemptRecovery(), which can import stale legacy data over a + // correctly hydrated store. + skipDeferredLocalActions: true, + }); - // Mark successfully applied ops. - const appliedSeqs = result.appliedOps - .map((op) => opIdToSeq.get(op.id)) - .filter((seq): seq is number => seq !== undefined); - if (appliedSeqs.length > 0) { - // The primary remote-apply path (applyRemoteOperations) merges clocks at - // reducer commit for the WHOLE batch, including ops whose archive - // handling later fails — so these clocks were usually merged already. - // Re-merging here is a harmless component-wise max and also covers ops - // that reached `failed`/`archive_pending` via crash recovery, where the - // reducer-commit callback (and its clock merge) may never have run. - await this.opLogStore.mergeRemoteOpClocks(result.appliedOps); - await this.opLogStore.markApplied(appliedSeqs); - OpLog.normal( - `OperationLogHydratorService: Successfully retried ${appliedSeqs.length} failed ops`, - ); - } + // Mark successfully applied ops. + const appliedSeqs = result.appliedOps + .map((op) => opIdToSeq.get(op.id)) + .filter((seq): seq is number => seq !== undefined); + if (appliedSeqs.length > 0) { + // The primary remote-apply path (applyRemoteOperations) merges clocks at + // reducer commit for the WHOLE batch, including ops whose archive + // handling later fails — so these clocks were usually merged already. + // Re-merging here is a harmless component-wise max and also covers ops + // that reached `failed`/`archive_pending` via crash recovery, where the + // reducer-commit callback (and its clock merge) may never have run. + await this.opLogStore.mergeRemoteOpClocks(result.appliedOps); + await this.opLogStore.markApplied(appliedSeqs); + OpLog.normal( + `OperationLogHydratorService: Successfully retried ${appliedSeqs.length} failed ops`, + ); + } - // On a partial failure the batch applier stops at the first archive error. - // Charge only that attempted operation: successors remain archive-pending - // without consuming retry budget and will run after the blocker succeeds. - // A persistent blocker stays failed so ordinary sync remains safely paused. - if (result.failedOp) { - const failedOpIds = [result.failedOp.op.id]; + // On a partial failure the batch applier stops at the first archive error. + // Charge only that attempted operation: successors remain archive-pending + // without consuming retry budget and will run after the blocker succeeds. + // A persistent blocker stays failed so ordinary sync remains safely paused. + if (result.failedOp) { + const failedOpIds = [result.failedOp.op.id]; - OpLog.warn( - `OperationLogHydratorService: Failed to retry op ${result.failedOp.op.id}`, - result.failedOp.error, - ); - // Keep archive failure visible to the sync safety gate. A retry cap that - // rejects it would hide incomplete downloaded work and allow false IN_SYNC. - await this.opLogStore.markFailed(failedOpIds); - OpLog.warn( - 'OperationLogHydratorService: Archive operation still failing after retry', - ); + OpLog.warn( + `OperationLogHydratorService: Failed to retry op ${result.failedOp.op.id}`, + result.failedOp.error, + ); + // Keep archive failure visible to the sync safety gate. A retry cap that + // rejects it would hide incomplete downloaded work and allow false IN_SYNC. + await this.opLogStore.markFailed(failedOpIds); + OpLog.warn( + 'OperationLogHydratorService: Archive operation still failing after retry', + ); + } + } finally { + // Local actions captured while the retry held the remote-apply window + // open. Runs after mergeRemoteOpClocks so their clocks dominate the + // retried remote ops (#7700). A failed drain keeps the actions buffered + // for the next drain point (e.g. the pre-sync flush) — never escalate + // it into hydration recovery. + try { + await processDeferredActions(this.injector, false); + } catch (drainError) { + OpLog.err( + 'OperationLogHydratorService: Deferred-action drain failed after archive retry; actions stay buffered.', + { name: (drainError as Error | undefined)?.name }, + ); + } } } diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 70280af355..1e47f8d613 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -1558,6 +1558,19 @@ describe('OperationLogStoreService', () => { expect(pending.length).toBe(0); }); + + it('should exclude rejected ops (parity with getFailedRemoteOps)', async () => { + // A rejected-but-still-pending row must not trip the incomplete-remote + // sync gate: nothing will ever apply it, so counting it would wedge sync + // for the whole session. + const op = createTestOperation({ entityId: 'rejected-pending' }); + await service.append(op, 'remote', { pendingApply: true }); + await service.markRejected([op.id]); + + const pending = await service.getPendingRemoteOps(); + + expect(pending.length).toBe(0); + }); }); describe('markFailed', () => { diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index a94ed39154..750de52454 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -765,8 +765,10 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort entry.source === 'remote' && entry.applicationStatus === 'pending', ); } - // Decode compact operations for backwards compatibility - return storedEntries.map(decodeStoredEntry); + // Exclude rejected ops (mirrors getFailedRemoteOps): a rejected-but-still- + // pending row must not trip the incomplete-remote sync gate — nothing will + // ever apply it, so it would wedge sync for the whole session. + return storedEntries.filter((e) => !e.rejectedAt).map(decodeStoredEntry); } async hasOp(id: string): Promise { diff --git a/src/app/op-log/sync/conflict-resolution.service.spec.ts b/src/app/op-log/sync/conflict-resolution.service.spec.ts index d618cfdeed..4ec9381732 100644 --- a/src/app/op-log/sync/conflict-resolution.service.spec.ts +++ b/src/app/op-log/sync/conflict-resolution.service.spec.ts @@ -64,7 +64,11 @@ describe('ConflictResolutionService', () => { 'mergeRemoteOpClocks', ]); mockOpLogStore.mergeRemoteOpClocks.and.resolveTo(undefined); - mockSnackService = jasmine.createSpyObj('SnackService', ['open']); + mockSnackService = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + mockSnackService.hasPendingPersistentAction.and.returnValue(false); mockValidateStateService = jasmine.createSpyObj('ValidateStateService', [ 'validateAndRepairCurrentState', ]); diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index f8538fa034..5c6e8ee11a 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -466,14 +466,20 @@ export class ConflictResolutionService { ); await this.opLogStore.markFailed(failedOpIds); - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.CONFLICT_RESOLUTION_FAILED, - actionStr: T.PS.RELOAD, - actionFn: (): void => { - window.location.reload(); - }, - }); + // Never replace a visible persistent recovery action (e.g. the + // USE_REMOTE Undo — the only entry point to the pre-replace backup). + // The IncompleteRemoteOperationsError thrown below still flips the + // sync status to ERROR via the wrapper's (equally guarded) handler. + if (!this.snackService.hasPendingPersistentAction()) { + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.CONFLICT_RESOLUTION_FAILED, + actionStr: T.PS.RELOAD, + actionFn: (): void => { + window.location.reload(); + }, + }); + } // FIX #6571: Throw on apply failure (parity with applyNonConflictingOps). // Previously, apply failures during LWW resolution were logged but not diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 0ac19981b6..35b00d15b9 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -2,6 +2,7 @@ import { TestBed } from '@angular/core/testing'; import { OperationLogSyncService } from './operation-log-sync.service'; import { FILE_BASED_SYNC_CONSTANTS } from '../sync-providers/file-based/file-based-sync.types'; import { SchemaMigrationService } from '../persistence/schema-migration.service'; +import { OperationLogHydratorService } from '../persistence/operation-log-hydrator.service'; import { SnackService } from '../../core/snack/snack.service'; import { OperationLogStoreService } from '../persistence/operation-log-store.service'; import { VectorClockService } from './vector-clock.service'; @@ -67,6 +68,7 @@ describe('OperationLogSyncService', () => { let lockServiceSpy: jasmine.SpyObj; let operationApplierSpy: jasmine.SpyObj; let operationLogEffectsSpy: jasmine.SpyObj; + let hydratorServiceSpy: jasmine.SpyObj; const createProviderSetupEntry = (): OperationLogEntry => ({ seq: 1, @@ -225,6 +227,10 @@ describe('OperationLogSyncService', () => { 'processDeferredActions', ]); operationLogEffectsSpy.processDeferredActions.and.resolveTo(); + hydratorServiceSpy = jasmine.createSpyObj('OperationLogHydratorService', [ + 'retryFailedRemoteOps', + ]); + hydratorServiceSpy.retryFailedRemoteOps.and.resolveTo(); TestBed.configureTestingModule({ providers: [ @@ -247,6 +253,7 @@ describe('OperationLogSyncService', () => { useValue: operationApplierSpy, }, { provide: OperationLogEffects, useValue: operationLogEffectsSpy }, + { provide: OperationLogHydratorService, useValue: hydratorServiceSpy }, { provide: ConflictResolutionService, useValue: jasmine.createSpyObj('ConflictResolutionService', [ @@ -398,6 +405,16 @@ describe('OperationLogSyncService', () => { expect(uploadServiceSpy.uploadPendingOps).not.toHaveBeenCalled(); }); + it('should not attempt the in-session archive retry while a raw rebuild is incomplete', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + + await expectAsync( + service.uploadPendingOps({} as OperationSyncCapable), + ).toBeRejectedWithError(IncompleteRemoteOperationsError); + + expect(hydratorServiceSpy.retryFailedRemoteOps).not.toHaveBeenCalled(); + }); + it('should return localWinOpsCreated: 0 when no piggybacked ops', async () => { opLogStoreSpy.getUnsynced.and.returnValue(Promise.resolve([])); uploadServiceSpy.uploadPendingOps.and.returnValue( @@ -1000,9 +1017,31 @@ describe('OperationLogSyncService', () => { service.downloadRemoteOps({} as OperationSyncCapable), ).toBeRejected(); + // The one in-session repair attempt ran but couldn't clear the gate. + expect(hydratorServiceSpy.retryFailedRemoteOps).toHaveBeenCalledTimes(1); expect(downloadServiceSpy.downloadRemoteOps).not.toHaveBeenCalled(); }); + it('should proceed when the in-session archive retry clears the incomplete-remote gate', async () => { + // Transient archive failure: quarantined at gate read, gone on re-check. + opLogStoreSpy.getFailedRemoteOps.and.returnValues( + Promise.resolve([{ applicationStatus: 'failed' } as OperationLogEntry]), + Promise.resolve([]), + ); + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + }); + + await service.downloadRemoteOps({} as OperationSyncCapable); + + expect(hydratorServiceSpy.retryFailedRemoteOps).toHaveBeenCalledTimes(1); + expect(downloadServiceSpy.downloadRemoteOps).toHaveBeenCalled(); + }); + it('should redo the raw rebuild when a prior USE_REMOTE replay was interrupted', async () => { // The normal download path excludes this client's own ops server-side, // so resuming an interrupted rebuild through it would silently lose them. @@ -3051,6 +3090,64 @@ describe('OperationLogSyncService', () => { expect(opLogStoreSpy.clearRawRebuildIncomplete).not.toHaveBeenCalled(); }); + it('should retry the rebuild in-call on a capture race and converge without re-downloading', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [makeRemoteOp()], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 50, + }); + // Attempt 1 trips the completion assert (e.g. a tracking tick landed in + // an unprotected gap); the in-call retry runs clean. + writeFlushServiceSpy.hasPendingWrites.and.returnValues(true, false, false); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as any; + + await service.forceDownloadRemoteState(mockProvider); + + // One network download, two local rebuild attempts. + expect(downloadServiceSpy.downloadRemoteOps).toHaveBeenCalledTimes(1); + expect(opLogStoreSpy.runRemoteStateReplacement).toHaveBeenCalledTimes(2); + // The retry re-enters through the crash-resume branch: the FIRST + // attempt's pre-replace backup is kept, never re-captured over. + expect(backupServiceSpy.captureImportBackup).toHaveBeenCalledTimes(1); + expect(opLogStoreSpy.loadImportBackup).toHaveBeenCalled(); + expect(opLogStoreSpy.clearRawRebuildIncomplete).toHaveBeenCalled(); + }); + + it('should warn only once per session when the remote history requires a newer app', async () => { + downloadServiceSpy.downloadRemoteOps.and.resolveTo({ + newOps: [{ ...makeRemoteOp(), schemaVersion: 9999 }], + needsFullStateUpload: false, + success: true, + providerMode: 'superSyncOps', + failedFileCount: 0, + latestServerSeq: 1, + }); + const mockProvider = { + supportsOperationSync: true, + setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), + } as any; + + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/newer schema version/); + await expectAsync( + service.forceDownloadRemoteState(mockProvider), + ).toBeRejectedWithError(/newer schema version/); + + const versionSnackCount = snackServiceSpy.open.calls + .allArgs() + .filter( + ([cfg]) => typeof cfg !== 'string' && cfg.msg === T.F.SYNC.S.VERSION_TOO_OLD, + ).length; + expect(versionSnackCount).toBe(1); + }); + it('should offer to restore the previous data after replacing (#8107)', async () => { downloadServiceSpy.downloadRemoteOps.and.resolveTo({ newOps: [makeRemoteOp()], diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 8e1416735d..28738c9ceb 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -22,6 +22,7 @@ import { OperationLogDownloadService } from './operation-log-download.service'; import { SnackService } from '../../core/snack/snack.service'; import { T } from '../../t.const'; import { + CaptureRacedRebuildError, IncompleteRemoteOperationsError, LocalDataConflictError, } from '../core/errors/sync-errors'; @@ -47,6 +48,7 @@ import { SchemaMigrationService, getOperationSchemaVersion, } from '../persistence/schema-migration.service'; +import { OperationLogHydratorService } from '../persistence/operation-log-hydrator.service'; import { SyncProviderManager } from '../sync-providers/provider-manager.service'; import { getDefaultMainModelData, MODEL_CONFIGS } from '../model/model-config'; import { loadAllData } from '../../root-store/meta/load-all-data.action'; @@ -162,6 +164,13 @@ export class OperationLogSyncService { private operationApplier = inject(OperationApplierService); private injector = inject(Injector); + /** + * Once-per-session latch for the USE_REMOTE newer-schema snack: the block + * persists until an app update, and every auto/WS-triggered resume attempt + * re-hits the preflight — without the latch the snack re-fires each time. + */ + private _hasWarnedRebuildVersionBlockThisSession = false; + /** * Checks if this client is "wholly fresh" - meaning it has never synced before * and has no local operation history. A fresh client accepting remote data @@ -1427,96 +1436,160 @@ export class OperationLogSyncService { let replacementCommitted = false; let backupSavedAt: number | undefined; let preservedLocalOps: Operation[] = []; - try { - // flushThenRunExclusive drains the capture pipeline BEFORE acquiring the - // op-log lock (flushPendingWrites re-acquires the same non-reentrant lock, - // so flushing while holding it deadlocks) and re-checks inside the lock — - // actions dispatched while the network request and preflight were in - // flight are durably written and included in the reversible safety backup. - backupSavedAt = await this.writeFlushService.flushThenRunExclusive(async () => { - let savedAt: number | undefined; - if (options?.isCrashResume) { - // Keep the first attempt's pre-replace backup (see option JSDoc). - savedAt = (await this.opLogStore.loadImportBackup())?.savedAt; - capturedBackupSavedAt = savedAt; - const marker = await this.opLogStore.loadRawRebuildIncomplete(); - const liveLocalOps = (await this.opLogStore.getUnsynced()).map( - (entry) => entry.op, - ); - preservedLocalOps = this._mergeOperationsById( - marker?.preservedLocalOps ?? [], - liveLocalOps, - ); - } else { - try { - savedAt = await this.backupService.captureImportBackup(); + // A capture racing the rebuild aborts the attempt (CaptureRacedRebuildError + // from the asserts below). Retry in-call from the already-downloaded + // history: the raced ops become durable in the next flush barrier and fold + // into preservedLocalOps via the resume branch, so each retry converges — + // important while e.g. active time tracking dispatches continuously, where + // waiting for the next sync trigger would re-download everything per + // attempt. In-memory reuse is safe: WS downloads and immediate uploads are + // gated while the rebuild marker is set. On exhaustion, the persisted + // marker hands over to the next-sync resume path as before. + const MAX_CAPTURE_RACE_ATTEMPTS = 3; + let isCrashResume = options?.isCrashResume ?? false; + for (let attempt = 1; ; attempt++) { + try { + // flushThenRunExclusive drains the capture pipeline BEFORE acquiring the + // op-log lock (flushPendingWrites re-acquires the same non-reentrant lock, + // so flushing while holding it deadlocks) and re-checks inside the lock — + // actions dispatched while the network request and preflight were in + // flight are durably written and included in the reversible safety backup. + backupSavedAt = await this.writeFlushService.flushThenRunExclusive(async () => { + let savedAt: number | undefined; + if (isCrashResume) { + // Keep the first attempt's pre-replace backup (see option JSDoc). + savedAt = (await this.opLogStore.loadImportBackup())?.savedAt; capturedBackupSavedAt = savedAt; - } catch (e) { - OpLog.warn( - 'OperationLogSyncService: Pre-replace safety backup failed; aborting force download.', - { name: (e as Error | undefined)?.name }, + const marker = await this.opLogStore.loadRawRebuildIncomplete(); + const liveLocalOps = (await this.opLogStore.getUnsynced()).map( + (entry) => entry.op, ); - throw new Error( - 'Pre-replace safety backup failed; aborting to preserve local state.', + preservedLocalOps = this._mergeOperationsById( + marker?.preservedLocalOps ?? [], + liveLocalOps, ); + } else { + try { + savedAt = await this.backupService.captureImportBackup(); + capturedBackupSavedAt = savedAt; + } catch (e) { + OpLog.warn( + 'OperationLogSyncService: Pre-replace safety backup failed; aborting force download.', + { name: (e as Error | undefined)?.name }, + ); + throw new Error( + 'Pre-replace safety backup failed; aborting to preserve local state.', + ); + } } - } - // The provider cursor lives outside SUP_OPS, so it cannot join the IDB - // transaction. Reset it first: a crash/failure before the transaction - // merely causes a safe re-download onto intact local state, while a - // commit can never become visible with a stale cursor that skips the - // remote history required to rebuild the baseline. - await syncProvider.setLastServerSeq(0); - await this.opLogStore.runRemoteStateReplacement({ - baselineState, - vectorClock: rebuiltClock, - schemaVersion: CURRENT_SCHEMA_VERSION, - snapshotEntityKeys: extractEntityKeysFromState( - baselineState as Parameters[0], - ), - archiveYoung, - archiveOld, - preservedLocalOps, - }); - replacementCommitted = true; + // The provider cursor lives outside SUP_OPS, so it cannot join the IDB + // transaction. Reset it first: a crash/failure before the transaction + // merely causes a safe re-download onto intact local state, while a + // commit can never become visible with a stale cursor that skips the + // remote history required to rebuild the baseline. + await syncProvider.setLastServerSeq(0); + await this.opLogStore.runRemoteStateReplacement({ + baselineState, + vectorClock: rebuiltClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + snapshotEntityKeys: extractEntityKeysFromState( + baselineState as Parameters[0], + ), + archiveYoung, + archiveOld, + preservedLocalOps, + }); + replacementCommitted = true; - OpLog.normal( - 'OperationLogSyncService: Replaced local persistence with remote baseline.', - ); - - // FILE-BASED SYNC: Handle snapshot state from force download. - // When downloading from seq 0 on file-based providers, we may receive a - // snapshotState instead of incremental ops. This happens when the remote - // has a SYNC_IMPORT (full state snapshot) with empty recentOps. - // hydrateFromRemoteSync persists its own state cache + vector clock. - if (result.providerMode === 'fileSnapshotOps' && snapshotState) { OpLog.normal( - 'OperationLogSyncService: Force download received snapshotState. Hydrating...', + 'OperationLogSyncService: Replaced local persistence with remote baseline.', ); - // Hydrate from snapshot - DON'T create SYNC_IMPORT since we're - // accepting remote state, not uploading local state. - await this.syncHydrationService.hydrateFromRemoteSync( - snapshotState, - result.snapshotVectorClock, - false, // Don't create SYNC_IMPORT - ); - - // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. - // Same rationale as downloadRemoteOps: file-based providers return ALL - // recentOps on every download and rely on getAppliedOpIds() to filter them. - if (migratedRemoteOps.length > 0) { - const appendResult = await this.opLogStore.appendBatchSkipDuplicates( - migratedRemoteOps, - 'remote', - ); + // FILE-BASED SYNC: Handle snapshot state from force download. + // When downloading from seq 0 on file-based providers, we may receive a + // snapshotState instead of incremental ops. This happens when the remote + // has a SYNC_IMPORT (full state snapshot) with empty recentOps. + // hydrateFromRemoteSync persists its own state cache + vector clock. + if (result.providerMode === 'fileSnapshotOps' && snapshotState) { OpLog.normal( - `OperationLogSyncService: Wrote ${appendResult.writtenOps.length} snapshot ops to IndexedDB ` + - 'after force-download hydration.' + - (appendResult.skippedCount > 0 - ? ` Skipped ${appendResult.skippedCount} duplicate(s).` - : ''), + 'OperationLogSyncService: Force download received snapshotState. Hydrating...', + ); + + // Hydrate from snapshot - DON'T create SYNC_IMPORT since we're + // accepting remote state, not uploading local state. + await this.syncHydrationService.hydrateFromRemoteSync( + snapshotState, + result.snapshotVectorClock, + false, // Don't create SYNC_IMPORT + ); + + // CRITICAL FIX: Write recentOps to IndexedDB after snapshot hydration. + // Same rationale as downloadRemoteOps: file-based providers return ALL + // recentOps on every download and rely on getAppliedOpIds() to filter them. + if (migratedRemoteOps.length > 0) { + const appendResult = await this.opLogStore.appendBatchSkipDuplicates( + migratedRemoteOps, + 'remote', + ); + OpLog.normal( + `OperationLogSyncService: Wrote ${appendResult.writtenOps.length} snapshot ops to IndexedDB ` + + 'after force-download hydration.' + + (appendResult.skippedCount > 0 + ? ` Skipped ${appendResult.skippedCount} duplicate(s).` + : ''), + ); + } + + await this._restorePreservedLocalOps(preservedLocalOps); + + await this._assertNoCaptureRacedWithRebuild(); + + // Update lastServerSeq after hydration + if (result.latestServerSeq !== undefined) { + await syncProvider.setLastServerSeq(result.latestServerSeq); + } + + await this._completeRawRebuild(); + + OpLog.normal( + 'OperationLogSyncService: Force download snapshot hydration complete.', + ); + return savedAt; + } + + // Reset live state to defaults, then replay the COMPLETE server history on + // top. A full-state op in the history replaces state again by its own + // semantics; a purely incremental history rebuilds from this baseline. + this.store.dispatch( + loadAllData({ + appDataComplete: defaultData as Parameters< + typeof loadAllData + >[0]['appDataComplete'], + }), + ); + // Brief yield to let NgRx process the state reset + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). + // Skip conflict detection because the NgRx store was just reset to empty state, + // which causes all entities to appear missing and CONCURRENT ops to be discarded. + // Validation failure is surfaced via the session-validation latch. (#7330) + const processResult = await this.remoteOpsProcessingService.processRemoteOps( + migratedRemoteOps, + { + skipConflictDetection: true, + callerHoldsOperationLogLock: true, + }, + ); + + if (processResult.blockedByIncompatibleOp) { + // Version blocks were pre-checked above; only a migration exception lands + // here. The rebuild is partial: keep the cursor at 0 so the next sync + // retries the remainder, and surface the failure — the Undo snack still + // offers the pre-replace backup. + throw new Error( + 'USE_REMOTE incomplete: an op failed schema migration during replay.', ); } @@ -1524,7 +1597,7 @@ export class OperationLogSyncService { await this._assertNoCaptureRacedWithRebuild(); - // Update lastServerSeq after hydration + // Update lastServerSeq if (result.latestServerSeq !== undefined) { await syncProvider.setLastServerSeq(result.latestServerSeq); } @@ -1532,67 +1605,33 @@ export class OperationLogSyncService { await this._completeRawRebuild(); OpLog.normal( - 'OperationLogSyncService: Force download snapshot hydration complete.', + `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, ); return savedAt; - } - - // Reset live state to defaults, then replay the COMPLETE server history on - // top. A full-state op in the history replaces state again by its own - // semantics; a purely incremental history rebuilds from this baseline. - this.store.dispatch( - loadAllData({ - appDataComplete: defaultData as Parameters< - typeof loadAllData - >[0]['appDataComplete'], - }), - ); - // Brief yield to let NgRx process the state reset - await new Promise((resolve) => setTimeout(resolve, 0)); - - // Process all remote ops (no confirmation needed - user already chose USE_REMOTE). - // Skip conflict detection because the NgRx store was just reset to empty state, - // which causes all entities to appear missing and CONCURRENT ops to be discarded. - // Validation failure is surfaced via the session-validation latch. (#7330) - const processResult = await this.remoteOpsProcessingService.processRemoteOps( - migratedRemoteOps, - { - skipConflictDetection: true, - callerHoldsOperationLogLock: true, - }, - ); - - if (processResult.blockedByIncompatibleOp) { - // Version blocks were pre-checked above; only a migration exception lands - // here. The rebuild is partial: keep the cursor at 0 so the next sync - // retries the remainder, and surface the failure — the Undo snack still - // offers the pre-replace backup. - throw new Error( - 'USE_REMOTE incomplete: an op failed schema migration during replay.', + }); + break; + } catch (e) { + if ( + e instanceof CaptureRacedRebuildError && + attempt < MAX_CAPTURE_RACE_ATTEMPTS + ) { + OpLog.warn( + `OperationLogSyncService: Local capture raced the rebuild; retrying phase 2 in-call ` + + `(attempt ${attempt}/${MAX_CAPTURE_RACE_ATTEMPTS}).`, ); + // The replacement committed before the assert threw, so the marker is + // set: re-enter through the crash-resume branch, which keeps the first + // attempt's backup and merges the newly-durable raced ops. + isCrashResume = true; + continue; } - - await this._restorePreservedLocalOps(preservedLocalOps); - - await this._assertNoCaptureRacedWithRebuild(); - - // Update lastServerSeq - if (result.latestServerSeq !== undefined) { - await syncProvider.setLastServerSeq(result.latestServerSeq); + // Final failure only — showing this per aborted attempt would churn the + // single snack slot. + if (replacementCommitted && capturedBackupSavedAt !== undefined) { + this._showRestorePreviousDataSnack(capturedBackupSavedAt); } - - await this._completeRawRebuild(); - - OpLog.normal( - `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, - ); - return savedAt; - }); - } catch (e) { - if (replacementCommitted && capturedBackupSavedAt !== undefined) { - this._showRestorePreviousDataSnack(capturedBackupSavedAt); + throw e; } - throw e; } // On a crash resume without a surviving backup there is nothing to offer. @@ -1624,6 +1663,38 @@ export class OperationLogSyncService { } private async _assertNoIncompleteRemoteOperations(): Promise { + const state = await this._readIncompleteRemoteOperationsState(); + if (!state.isBlocked) { + return; + } + + // One in-session repair attempt when the ONLY blockers are quarantined + // archive failures: their reducers already committed, and the archive-only + // retry is idempotent (ARCHIVE_AFFECTING_ACTION_TYPES invariant). Without + // this, a transient archive failure wedges sync until the next app start — + // the error snack's "restart" advice — even though a retry would succeed + // immediately. Never attempted while a raw rebuild is incomplete or + // reducer-uncommitted `pending` rows exist (retrying those would be wrong). + if (!state.isRawRebuildIncomplete && state.pendingCount === 0) { + await this.injector.get(OperationLogHydratorService).retryFailedRemoteOps(); + const recheck = await this._readIncompleteRemoteOperationsState(); + if (!recheck.isBlocked) { + OpLog.normal( + 'OperationLogSyncService: In-session archive retry cleared the incomplete-remote gate.', + ); + return; + } + } + + throw new IncompleteRemoteOperationsError(); + } + + private async _readIncompleteRemoteOperationsState(): Promise<{ + isBlocked: boolean; + isRawRebuildIncomplete: boolean; + pendingCount: number; + failedCount: number; + }> { const [isRawRebuildIncomplete, pendingRemoteOps, failedRemoteOps] = await Promise.all( [ this.opLogStore.isRawRebuildIncomplete(), @@ -1631,13 +1702,15 @@ export class OperationLogSyncService { this.opLogStore.getFailedRemoteOps(), ], ); - if ( - isRawRebuildIncomplete || - pendingRemoteOps.length > 0 || - failedRemoteOps.length > 0 - ) { - throw new IncompleteRemoteOperationsError(); - } + return { + isBlocked: + isRawRebuildIncomplete || + pendingRemoteOps.length > 0 || + failedRemoteOps.length > 0, + isRawRebuildIncomplete, + pendingCount: pendingRemoteOps.length, + failedCount: failedRemoteOps.length, + }; } private async _resumeInterruptedRawRebuild( @@ -1669,9 +1742,7 @@ export class OperationLogSyncService { private _assertNoCaptureRacedWithRebuild(): void { if (this.writeFlushService.hasPendingWrites()) { - throw new Error( - 'USE_REMOTE incomplete: a local change arrived during the rebuild and will be restored on retry.', - ); + throw new CaptureRacedRebuildError(); } } @@ -1744,13 +1815,19 @@ export class OperationLogSyncService { ); } if (version > CURRENT_SCHEMA_VERSION) { - this.snackService.open({ - type: 'ERROR', - msg: T.F.SYNC.S.VERSION_TOO_OLD, - actionStr: T.PS.UPDATE_APP, - actionFn: () => - window.open('https://super-productivity.com/download', '_blank'), - }); + if ( + !this._hasWarnedRebuildVersionBlockThisSession && + !this.snackService.hasPendingPersistentAction() + ) { + this._hasWarnedRebuildVersionBlockThisSession = true; + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.VERSION_TOO_OLD, + actionStr: T.PS.UPDATE_APP, + actionFn: () => + window.open('https://super-productivity.com/download', '_blank'), + }); + } throw new Error( 'USE_REMOTE aborted: remote history contains ops from a newer schema version — update the app first.', ); @@ -1801,6 +1878,18 @@ export class OperationLogSyncService { }); } + /** + * Boot-time entry point for the interrupted-rebuild affordance (see + * StartupService): the user woke up on the rebuild baseline, not their data. + * A running sync resumes the rebuild by itself, but when none will run + * (offline, or sync disabled facing the "emptied" app) the pre-replace + * backup would have no visible entry point. Non-destructive — only surfaces + * the persistent restore snack, deduped against a visible recovery action. + */ + async offerInterruptedRebuildRecovery(): Promise { + await this._offerStrandedRebuildBackup(); + } + /** * Offer the pre-replace Undo after an interrupted USE_REMOTE rebuild whose * resume could not finish. The first attempt already committed the destructive diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index d427953960..b72cbb6d00 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -75,7 +75,11 @@ describe('RemoteOpsProcessingService', () => { 'getCurrentVersion', 'migrateOperation', ]); - snackServiceSpy = jasmine.createSpyObj('SnackService', ['open']); + snackServiceSpy = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); opLogStoreSpy = jasmine.createSpyObj('OperationLogStoreService', [ 'getUnsynced', 'hasOp', @@ -670,6 +674,20 @@ describe('RemoteOpsProcessingService', () => { }); }); + it('should not replace a visible persistent recovery action with the version-block snack', async () => { + snackServiceSpy.hasPendingPersistentAction.and.returnValue(true); + + await service.processRemoteOps([{ id: 'op1', schemaVersion: 2 } as Operation]); + + expect(snackServiceSpy.open).not.toHaveBeenCalled(); + + // The latch stayed unset, so a later retry (after the recovery action + // resolves) still warns the user. + snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); + await service.processRemoteOps([{ id: 'op2', schemaVersion: 2 } as Operation]); + expect(snackServiceSpy.open).toHaveBeenCalledTimes(1); + }); + it('should show the version-block error only once per session', async () => { // Current version is 1 (set in beforeEach) const remoteOps1: Operation[] = [{ id: 'op1', schemaVersion: 2 } as Operation]; diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index 4d5a92a8d8..95c03e1337 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -396,6 +396,12 @@ export class RemoteOpsProcessingService { | 'INVALID_SCHEMA_VERSION' | 'MIGRATION_FAILED', ): void { + if (this.snackService.hasPendingPersistentAction()) { + // Never replace a visible persistent recovery action (e.g. the USE_REMOTE + // Undo — the only entry point to the pre-replace backup). The block + // persists, so the latch stays unset and a later retry re-warns. + return; + } if (reason === 'MIGRATION_FAILED' || reason === 'INVALID_SCHEMA_VERSION') { if (!this._hasWarnedMigrationFailureThisSession) { this._hasWarnedMigrationFailureThisSession = true; @@ -408,15 +414,22 @@ export class RemoteOpsProcessingService { } if (!this._hasWarnedVersionBlockThisSession) { this._hasWarnedVersionBlockThisSession = true; - this.snackService.open({ - type: 'ERROR', - msg: - reason === 'VERSION_UNSUPPORTED' - ? T.F.SYNC.S.VERSION_UNSUPPORTED - : T.F.SYNC.S.VERSION_TOO_OLD, - actionStr: T.PS.UPDATE_APP, - actionFn: () => window.open('https://super-productivity.com/download', '_blank'), - }); + if (reason === 'VERSION_UNSUPPORTED') { + // Below-minimum data: updating THIS device cannot help, so no + // download action — the fix lives on the device that produced it. + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.VERSION_UNSUPPORTED, + }); + } else { + this.snackService.open({ + type: 'ERROR', + msg: T.F.SYNC.S.VERSION_TOO_OLD, + actionStr: T.PS.UPDATE_APP, + actionFn: () => + window.open('https://super-productivity.com/download', '_blank'), + }); + } } } diff --git a/src/app/op-log/sync/sync-import-conflict-gate.service.ts b/src/app/op-log/sync/sync-import-conflict-gate.service.ts index 5b2abd6763..60198ffb8f 100644 --- a/src/app/op-log/sync/sync-import-conflict-gate.service.ts +++ b/src/app/op-log/sync/sync-import-conflict-gate.service.ts @@ -93,9 +93,13 @@ export class SyncImportConflictGateService { /** * @param options.preCapturedPendingOps - Exact pending ops selected by the upload - * round. The piggyback path unions this snapshot with a live read so it - * protects both accepted work from that upload and work created while the - * network request was in flight. + * round, unioned with a live read. NOTE: accepted ops of the current round + * are still in the live set when this gate runs (acknowledgement is + * deferred until piggyback processing commits), so the union is NOT what + * protects them. Its own coverage is the narrower set of ops removed from + * the live set mid-upload: a local SYNC_IMPORT deleted on + * SYNC_IMPORT_EXISTS, full-state ops perma-rejected by the server, and a + * concurrent tab marking ops synced between lock release and this read. */ async checkIncomingFullStateConflict( incomingOps: Operation[], diff --git a/src/app/op-log/testing/integration/migration-handling.integration.spec.ts b/src/app/op-log/testing/integration/migration-handling.integration.spec.ts index 09067ad346..aae2c542ab 100644 --- a/src/app/op-log/testing/integration/migration-handling.integration.spec.ts +++ b/src/app/op-log/testing/integration/migration-handling.integration.spec.ts @@ -30,7 +30,11 @@ describe('Migration Handling Integration', () => { let operationApplierSpy: jasmine.SpyObj; beforeEach(async () => { - snackServiceSpy = jasmine.createSpyObj('SnackService', ['open']); + snackServiceSpy = jasmine.createSpyObj('SnackService', [ + 'open', + 'hasPendingPersistentAction', + ]); + snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); operationApplierSpy = jasmine.createSpyObj('OperationApplierService', [ 'applyOperations', ]); diff --git a/src/assets/i18n/en.json b/src/assets/i18n/en.json index 6e39ab4cee..9f8a73b31b 100644 --- a/src/assets/i18n/en.json +++ b/src/assets/i18n/en.json @@ -1586,7 +1586,7 @@ "LOCK_TIMEOUT_ERROR": "Sync timed out waiting for a previous operation to finish. Please try again.", "LWW_CONFLICTS_AUTO_RESOLVED": "Sync conflicts auto-resolved: {{localWins}} local win(s), {{remoteWins}} remote win(s)", "LOCAL_FILE_RESELECT_REQUIRED": "Local file sync needs the sync folder to be selected again after a security update. Open Sync settings and choose the folder once more.", - "MIGRATION_FAILED": "Some sync data could not be migrated and was skipped. This may indicate data corruption or a bug.", + "MIGRATION_FAILED": "Some synced changes could not be migrated. Sync is paused before them to protect your data; updating the app may fix this.", "NETWORK_ERROR": "Sync request failed because of a temporary network problem. Your changes remain local and sync will retry.", "NEWER_VERSION_AVAILABLE": "Some changes are from a newer app version. Consider updating for full compatibility.", "OAUTH_STATE_INVALID": "OAuth state mismatch — the callback URL did not match this device's request. Please try authenticating again.", @@ -1615,7 +1615,7 @@ "UPLOAD_OPS_REJECTED": "{{count}} operation(s) were permanently rejected by the server and won't be synced.", "VECTOR_CLOCK_LIMIT_REACHED": "Sync tidied up old device entries ({{originalSize}} → {{maxSize}}). This is normal — no action needed.", "VERSION_TOO_OLD": "Your app version is too old for the synced data. Please update!", - "VERSION_UNSUPPORTED": "Cannot sync: data requires a newer app version. Please update.", + "VERSION_UNSUPPORTED": "Cannot sync: some synced data is from an app version that is no longer supported. Sync is paused before it; updating the app on your other devices may help.", "WEB_CRYPTO_NOT_AVAILABLE": "Encryption is not available on this device. On Android, encryption requires a secure context (HTTPS) which is not available. Please disable encryption in sync settings or sync from a desktop web browser." }, "SAFETY_BACKUP": { From bdb0fef9a97b52ec0cf31f21420b15fb33826154 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 17:59:42 +0200 Subject: [PATCH 34/47] fix(sync): make remote reducer checkpoint atomic with its clock merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A committed reducer must never be durable without its vector clock, and a merged clock must never be durable without its reducer checkpoint — either mismatch lets the next local operation be causally older than state already visible in NgRx after a crash. - markArchivePending + separate mergeRemoteOpClocks are replaced by one markReducersCommittedAndMergeClocks transaction (ops + vector_clock); the clock math is extracted into a pure calculateRemoteClockMerge so the standalone merge path keeps identical full-state-reset semantics. - The sync-core RemoteOperationApplyStorePort gains an onRemoteClocksDurable hook so deferred local actions drain exactly when clocks are durable, not merely when ops were applied; a checkpoint rejection can no longer mask the primary apply error. - Conflict resolution's local-wins path writes remote losers and rebased local compensations in one appendMixedSourceBatchSkipDuplicates transaction, so synthetic ops cannot reuse or regress the client counter. - DB_VERSION 7 -> 8 as a deliberate downgrade barrier: released v7 readers only understand 'failed' and would silently overlook outstanding 'archive_pending' work. op-log suite and sync-core suite cover the checkpoint rollback, atomic clock merge, mixed-source ordering and drain failure matrix. --- packages/sync-core/src/operation.types.ts | 7 +- packages/sync-core/src/remote-apply.ts | 60 +- packages/sync-core/tests/remote-apply.spec.ts | 167 +++++- .../tests/replay-coordinator.spec.ts | 39 ++ src/app/op-log/persistence/db-keys.const.ts | 18 +- src/app/op-log/persistence/db-upgrade.ts | 4 + .../indexed-db-op-log-adapter.spec.ts | 2 +- .../op-log/persistence/op-log-db-schema.ts | 2 +- .../operation-log-recovery.service.spec.ts | 80 +-- .../operation-log-recovery.service.ts | 50 +- .../operation-log-store.service.spec.ts | 396 ++++++++++++- .../operation-log-store.service.ts | 526 +++++++++++++++--- .../persistence/sqlite-op-log-adapter.spec.ts | 2 +- .../sync/conflict-resolution.service.spec.ts | 458 ++++++++++++--- .../sync/conflict-resolution.service.ts | 76 +-- .../remote-ops-processing.service.spec.ts | 239 ++++++-- .../sync/remote-ops-processing.service.ts | 54 +- ...emote-apply-store-port.integration.spec.ts | 65 +++ 18 files changed, 1868 insertions(+), 377 deletions(-) diff --git a/packages/sync-core/src/operation.types.ts b/packages/sync-core/src/operation.types.ts index 67e35f4ee1..c356f98064 100644 --- a/packages/sync-core/src/operation.types.ts +++ b/packages/sync-core/src/operation.types.ts @@ -148,8 +148,11 @@ export interface OperationLogEntry = Operat rejectedAt?: number; /** - * For remote ops only: tracks whether the op was successfully applied to local state. - * Used for crash recovery: on startup, any 'pending' or 'failed' remote ops are re-dispatched. + * For remote ops only: tracks whether the op was successfully applied to + * local state. `archive_pending` means reducers and clocks committed but + * archive side effects are outstanding; `failed` means an attempted archive + * side effect failed and retryCount was bumped. Startup recovery replays + * reducers status-blind and retries outstanding archive work. */ applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed'; diff --git a/packages/sync-core/src/remote-apply.ts b/packages/sync-core/src/remote-apply.ts index 1c5dc1cf26..50a18ad0b8 100644 --- a/packages/sync-core/src/remote-apply.ts +++ b/packages/sync-core/src/remote-apply.ts @@ -24,10 +24,10 @@ export interface RemoteOperationApplyStorePort< source: 'remote', options: { pendingApply: true }, ): Promise>; - markArchivePending(seqs: number[]): Promise; + mergeRemoteOpClocks(ops: TOperation[]): Promise; + markReducersCommittedAndMergeClocks(seqs: number[], ops: TOperation[]): Promise; markApplied(seqs: number[]): Promise; markFailed(opIds: string[]): Promise; - mergeRemoteOpClocks(ops: TOperation[]): Promise; clearFullStateOpsExcept(excludeIds: string[]): Promise; } @@ -38,6 +38,8 @@ export interface ApplyRemoteOperationsOptions< store: RemoteOperationApplyStorePort; applier: ReducerCommitAwareOperationApplyPort; isFullStateOperation?: (op: TOperation) => boolean; + /** Runs after incoming clocks are durable, before reducer application starts. */ + onRemoteClocksDurable?: (ops: TOperation[]) => void; } export interface RemoteApplyOperationsResult< @@ -71,14 +73,17 @@ const emptyRemoteApplyResult = < * the generic ordering: * * 1. append incoming remote ops as pending, skipping duplicates atomically; - * 2. bulk-apply only the newly appended ops through the host applier; - * 3. at reducer commit, mark the whole batch archive_pending and merge all - * reducer-committed vector clocks before archive side effects begin; - * 4. mark archive-complete seqs applied; - * 5. after applying a full-state op, clear older full-state ops while - * retaining every full-state op of this batch (incl. archive_pending); - * 6. mark only the attempted archive failure as failed; unattempted successors - * remain archive_pending for ordered startup recovery. + * 2. durably merge all newly appended remote clocks before reducers or a + * deferred-local-action window can start; + * 3. bulk-apply only the newly appended ops through the host applier; + * 4. at reducer commit, atomically checkpoint the whole batch as + * `archive_pending` and merge all reducer-committed vector clocks before + * archive side effects begin; + * 5. mark archive-complete seqs applied; + * 6. after applying a full-state op, clear older full-state ops while + * retaining every full-state op of this batch (incl. quarantined ones); + * 7. mark only the attempted archive failure as `failed`; unattempted + * successors stay `archive_pending` for ordered startup recovery. */ export const applyRemoteOperations = async < TOperation extends Operation = Operation, @@ -87,6 +92,7 @@ export const applyRemoteOperations = async < store, applier, isFullStateOperation = () => false, + onRemoteClocksDurable, }: ApplyRemoteOperationsOptions): Promise< RemoteApplyOperationsResult > => { @@ -105,6 +111,13 @@ export const applyRemoteOperations = async < }; } + // Pending rows make this pre-apply clock merge crash-safe: if anything fails + // after it commits, startup recovery still replays the same rows. Advancing + // the clock before entering the reducer window also means buffered local + // actions can always be drained against a durable remote frontier. + await store.mergeRemoteOpClocks(appendResult.writtenOps); + onRemoteClocksDurable?.(appendResult.writtenOps); + const opIdToSeq = new Map(); appendResult.writtenOps.forEach((op, index) => { const seq = appendResult.seqs[index]; @@ -117,6 +130,19 @@ export const applyRemoteOperations = async < let reducerCommitCallbackError: Error | undefined; let reducerCommitPromise: Promise | undefined; let applyResult: ApplyOperationsResult; + const observeReducerCommitPromisePreservingPrimaryError = async (): Promise => { + const promise = reducerCommitPromise; + if (!promise) { + return; + } + try { + await promise; + } catch { + // The already-selected apply/contract error is the primary failure. The + // checkpoint rejection is still observed here so it cannot become an + // unhandled rejection; pending rows make it recoverable at startup. + } + }; try { applyResult = await applier.applyOperations(appendResult.writtenOps, { onReducersCommitted: (reducerCommittedOps) => { @@ -149,8 +175,10 @@ export const applyRemoteOperations = async < 'applyRemoteOperations: reducer commit contained an operation outside the appended batch.', ); } - await store.markArchivePending(reducerCommittedSeqs); - await store.mergeRemoteOpClocks(appendResult.writtenOps); + await store.markReducersCommittedAndMergeClocks( + reducerCommittedSeqs, + appendResult.writtenOps, + ); })(); return reducerCommitPromise; }, @@ -159,15 +187,11 @@ export const applyRemoteOperations = async < // A valid first callback may already have started durable bookkeeping before // a duplicate callback (or another applier error) throws. Observe that // promise before propagating so it cannot become an unhandled rejection. - if (reducerCommitPromise) { - await reducerCommitPromise; - } + await observeReducerCommitPromisePreservingPrimaryError(); throw applyError; } if (reducerCommitCallbackError) { - if (reducerCommitPromise) { - await reducerCommitPromise; - } + await observeReducerCommitPromisePreservingPrimaryError(); throw reducerCommitCallbackError; } if (reducerCommitCallbackCount !== 1 || reducerCommitPromise === undefined) { diff --git a/packages/sync-core/tests/remote-apply.spec.ts b/packages/sync-core/tests/remote-apply.spec.ts index 478d9bedb3..d5e149365c 100644 --- a/packages/sync-core/tests/remote-apply.spec.ts +++ b/packages/sync-core/tests/remote-apply.spec.ts @@ -26,10 +26,10 @@ const createStore = (appendResult: { skippedCount: number; }): RemoteOperationApplyStorePort> => ({ appendBatchSkipDuplicates: vi.fn().mockResolvedValue(appendResult), - markArchivePending: vi.fn().mockResolvedValue(undefined), + mergeRemoteOpClocks: vi.fn().mockResolvedValue(undefined), + markReducersCommittedAndMergeClocks: vi.fn().mockResolvedValue(undefined), markApplied: vi.fn().mockResolvedValue(undefined), markFailed: vi.fn().mockResolvedValue(undefined), - mergeRemoteOpClocks: vi.fn().mockResolvedValue(undefined), clearFullStateOpsExcept: vi.fn().mockResolvedValue(0), }); @@ -62,9 +62,9 @@ describe('applyRemoteOperations', () => { [op2], jasmineLikeObjectContainingFunction(), ); - expect(store.markArchivePending).toHaveBeenCalledWith([10]); - expect(store.markApplied).toHaveBeenCalledWith([10]); expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([op2]); + expect(store.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith([10], [op2]); + expect(store.markApplied).toHaveBeenCalledWith([10]); expect(result).toEqual({ appendedOps: [op2], skippedCount: 1, @@ -85,6 +85,7 @@ describe('applyRemoteOperations', () => { expect(applier.applyOperations).not.toHaveBeenCalled(); expect(store.markApplied).not.toHaveBeenCalled(); expect(store.mergeRemoteOpClocks).not.toHaveBeenCalled(); + expect(store.markReducersCommittedAndMergeClocks).not.toHaveBeenCalled(); expect(result).toEqual({ appendedOps: [], skippedCount: 1, @@ -116,10 +117,10 @@ describe('applyRemoteOperations', () => { expect(result.clearedFullStateOpCount).toBe(3); }); - it('cleanup preserves a batch full-state op whose archive handling failed (archive_pending)', async () => { + it('cleanup preserves a batch full-state op whose archive handling failed (quarantined)', async () => { // Both imports reducer-committed; the LATER one failed only its archive // side effects. Cleanup keyed on the applied one must not delete the - // archive_pending entry, or markFailed misses it and the change is lost + // quarantined entry, or markFailed misses it and the change is lost // from the next startup replay. const appliedImport = createOperation('sync-import-1', 'SYNC_IMPORT'); const failedImport = createOperation('sync-import-2', 'SYNC_IMPORT'); @@ -170,8 +171,10 @@ describe('applyRemoteOperations', () => { }); expect(store.markApplied).toHaveBeenCalledWith([1]); - expect(store.markArchivePending).toHaveBeenCalledWith([1, 2, 3]); - expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([op1, op2, op3]); + expect(store.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1, 2, 3], + [op1, op2, op3], + ); expect(store.markFailed).toHaveBeenCalledWith(['op-2']); expect(result.failedOp).toEqual({ op: op2, error }); expect(result.failedOpIds).toEqual(['op-2']); @@ -274,7 +277,10 @@ describe('applyRemoteOperations', () => { isFullStateOperation: (op) => op.opType === 'SYNC_IMPORT', }); - expect(store.mergeRemoteOpClocks).toHaveBeenCalledWith([writtenImport]); + expect(store.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [11], + [writtenImport], + ); expect(store.clearFullStateOpsExcept).toHaveBeenCalledWith(['sync-import-1']); expect(result.appendedOps[0]).toBe(writtenImport); expect(result.appliedOps[0]).toBe(writtenImport); @@ -381,6 +387,149 @@ describe('applyRemoteOperations', () => { ); expect(duplicateCallbackThrewSynchronously).toBe(true); }); + + it('preserves the applier error when an unawaited reducer checkpoint also fails', async () => { + const op = createOperation('op-1'); + const applyError = new Error('dispatcher failed'); + const checkpointError = new Error('checkpoint failed'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + vi.mocked(store.markReducersCommittedAndMergeClocks).mockRejectedValue( + checkpointError, + ); + const applier: ReducerCommitAwareOperationApplyPort> = { + applyOperations: vi.fn(async (ops, options) => { + void options.onReducersCommitted(ops); + throw applyError; + }), + }; + + await expect(applyRemoteOperations({ ops: [op], store, applier })).rejects.toBe( + applyError, + ); + }); + + it('preserves a callback-contract error when the first reducer checkpoint also fails', async () => { + const op = createOperation('op-1'); + const checkpointError = new Error('checkpoint failed'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + vi.mocked(store.markReducersCommittedAndMergeClocks).mockRejectedValue( + checkpointError, + ); + let callbackContractError: unknown; + const applier: ReducerCommitAwareOperationApplyPort> = { + applyOperations: vi.fn(async (ops, options) => { + void options.onReducersCommitted(ops); + try { + void options.onReducersCommitted(ops); + } catch (error) { + callbackContractError = error; + throw error; + } + return { appliedOps: ops }; + }), + }; + + let thrown: unknown; + try { + await applyRemoteOperations({ ops: [op], store, applier }); + } catch (error) { + thrown = error; + } + + expect(callbackContractError).toBeInstanceOf(Error); + expect(thrown).toBe(callbackContractError); + }); + + it('does not start reducer application when the pre-apply clock merge fails', async () => { + const op = createOperation('op-1'); + const clockError = new Error('clock merge failed'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + vi.mocked(store.mergeRemoteOpClocks).mockRejectedValue(clockError); + const applier = createApplier({ appliedOps: [op] }); + const onRemoteClocksDurable = vi.fn(); + + await expect( + applyRemoteOperations({ + ops: [op], + store, + applier, + onRemoteClocksDurable, + }), + ).rejects.toBe(clockError); + + expect(onRemoteClocksDurable).not.toHaveBeenCalled(); + expect(applier.applyOperations).not.toHaveBeenCalled(); + expect(store.markReducersCommittedAndMergeClocks).not.toHaveBeenCalled(); + expect(store.markApplied).not.toHaveBeenCalled(); + }); + + it('signals durable remote clocks before a later atomic checkpoint failure', async () => { + const op = createOperation('op-1'); + const checkpointError = new Error('checkpoint failed'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + vi.mocked(store.markReducersCommittedAndMergeClocks).mockRejectedValue( + checkpointError, + ); + const onRemoteClocksDurable = vi.fn(); + + await expect( + applyRemoteOperations({ + ops: [op], + store, + applier: createApplier({ appliedOps: [op] }), + onRemoteClocksDurable, + }), + ).rejects.toBe(checkpointError); + + expect(onRemoteClocksDurable).toHaveBeenCalledWith([op]); + expect(store.markApplied).not.toHaveBeenCalled(); + }); + + it('signals durable remote clocks before later bookkeeping can fail', async () => { + const op = createOperation('op-1'); + const markAppliedError = new Error('mark applied failed'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + vi.mocked(store.markApplied).mockRejectedValue(markAppliedError); + const onRemoteClocksDurable = vi.fn(); + + await expect( + applyRemoteOperations({ + ops: [op], + store, + applier: createApplier({ appliedOps: [op] }), + onRemoteClocksDurable, + }), + ).rejects.toBe(markAppliedError); + + expect(onRemoteClocksDurable).toHaveBeenCalledWith([op]); + }); + + it('merges clocks before applying reducers and keeps the atomic checkpoint afterward', async () => { + const op = createOperation('op-1'); + const store = createStore({ seqs: [1], writtenOps: [op], skippedCount: 0 }); + const callOrder: string[] = []; + vi.mocked(store.mergeRemoteOpClocks).mockImplementation(async () => { + callOrder.push('mergeRemoteOpClocks'); + }); + vi.mocked(store.markReducersCommittedAndMergeClocks).mockImplementation(async () => { + callOrder.push('markReducersCommittedAndMergeClocks'); + }); + const applier: ReducerCommitAwareOperationApplyPort> = { + applyOperations: vi.fn(async (ops, options) => { + callOrder.push('applyOperations'); + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }), + }; + + await applyRemoteOperations({ ops: [op], store, applier }); + + expect(callOrder).toEqual([ + 'mergeRemoteOpClocks', + 'applyOperations', + 'markReducersCommittedAndMergeClocks', + ]); + }); }); const jasmineLikeObjectContainingFunction = (): object => diff --git a/packages/sync-core/tests/replay-coordinator.spec.ts b/packages/sync-core/tests/replay-coordinator.spec.ts index 0b866de933..9d738f8796 100644 --- a/packages/sync-core/tests/replay-coordinator.spec.ts +++ b/packages/sync-core/tests/replay-coordinator.spec.ts @@ -415,6 +415,45 @@ describe('replayOperationBatch', () => { ]); }); + it('closes the sync window and flushes deferred actions when reducer bookkeeping throws', async () => { + const callOrder: string[] = []; + const checkpointError = new Error('reducer checkpoint failed'); + + await expect( + replayOperationBatch({ + ops: [createOperation('op-1')], + dispatcher: { + dispatch: vi.fn(() => { + callOrder.push('dispatchBulk'); + }), + }, + createBulkApplyAction: (operations) => ({ + type: '[Test] Bulk Apply', + operations, + }), + remoteApplyWindow: createRemoteApplyWindow(callOrder), + deferredLocalActions: createDeferredLocalActions(callOrder), + onReducersCommitted: vi.fn(async () => { + callOrder.push('reducersCommitted'); + throw checkpointError; + }), + yieldToEventLoop: vi.fn(async () => { + callOrder.push('yield'); + }), + }), + ).rejects.toBe(checkpointError); + + expect(callOrder).toEqual([ + 'startApplyingRemoteOps', + 'dispatchBulk', + 'yield', + 'reducersCommitted', + 'startPostSyncCooldown', + 'endApplyingRemoteOps', + 'processDeferredActions', + ]); + }); + it('closes the sync window even when post-sync cooldown throws', async () => { const callOrder: string[] = []; const cooldownError = new Error('cooldown failed'); diff --git a/src/app/op-log/persistence/db-keys.const.ts b/src/app/op-log/persistence/db-keys.const.ts index dcce2bd4f1..c582dc6cdc 100644 --- a/src/app/op-log/persistence/db-keys.const.ts +++ b/src/app/op-log/persistence/db-keys.const.ts @@ -11,8 +11,15 @@ import { CompleteBackup } from '../sync-exports'; /** Database name */ export const DB_NAME = 'SUP_OPS'; -/** Current database schema version */ -export const DB_VERSION = 7; +/** + * Current database schema version. + * + * Version 8 is a deliberate downgrade barrier for the distinct + * `archive_pending` reducer checkpoint. Released v7 readers only understand + * `failed`; IndexedDB must reject their attempt to open a newer database rather + * than let outstanding archive work become invisible. + */ +export const DB_VERSION = 8; /** Object store names */ export const STORE_NAMES = { @@ -54,6 +61,13 @@ export const FULL_STATE_OPS_META_KEY = 'full_state_ops' as const; */ export const RAW_REBUILD_INCOMPLETE_META_KEY = 'raw_rebuild_incomplete' as const; +/** + * Meta key retaining the Undo provenance after a USE_REMOTE rebuild completed. + * This is separate from the incomplete/resume marker: completion atomically + * replaces the latter with this entry so a reload cannot strand the backup. + */ +export const RAW_REBUILD_RECOVERY_META_KEY = 'raw_rebuild_recovery' as const; + /** Versioned marker for the one-time legacy terminal remote failure repair. */ export const LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY = 'legacy_terminal_remote_failures_migration' as const; diff --git a/src/app/op-log/persistence/db-upgrade.ts b/src/app/op-log/persistence/db-upgrade.ts index cc532b280b..965a3c3e9a 100644 --- a/src/app/op-log/persistence/db-upgrade.ts +++ b/src/app/op-log/persistence/db-upgrade.ts @@ -146,4 +146,8 @@ export const runDbUpgrade = ( db.createObjectStore(STORE_NAMES.META); populateFullStateOpsMetaDuringUpgrade(transaction); } + + // Version 8: no shape change. The version itself is a downgrade barrier for + // `archive_pending`; v7 readers must fail closed instead of silently skipping + // reducer-committed operations whose archive work is still outstanding. }; diff --git a/src/app/op-log/persistence/indexed-db-op-log-adapter.spec.ts b/src/app/op-log/persistence/indexed-db-op-log-adapter.spec.ts index a259941124..d6677eb7e7 100644 --- a/src/app/op-log/persistence/indexed-db-op-log-adapter.spec.ts +++ b/src/app/op-log/persistence/indexed-db-op-log-adapter.spec.ts @@ -24,7 +24,7 @@ describe('IndexedDbOpLogAdapter', () => { const makeOpEntry = ( id: string, source: 'local' | 'remote', - applicationStatus?: 'pending' | 'applied' | 'failed', + applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed', syncedAt?: number, ): Record => ({ op: { id }, diff --git a/src/app/op-log/persistence/op-log-db-schema.ts b/src/app/op-log/persistence/op-log-db-schema.ts index 3162278010..38570c305f 100644 --- a/src/app/op-log/persistence/op-log-db-schema.ts +++ b/src/app/op-log/persistence/op-log-db-schema.ts @@ -40,7 +40,7 @@ export interface OpLogDbSchema { } /** - * Current `SUP_OPS` schema, mirroring db-upgrade.ts (currently v7). + * Current `SUP_OPS` schema, mirroring db-upgrade.ts (currently v8). * * `name`/`version` are reused from `db-keys.const.ts` (not re-literaled) so the * adapter opens at exactly the version `runDbUpgrade` migrates to — a future diff --git a/src/app/op-log/persistence/operation-log-recovery.service.spec.ts b/src/app/op-log/persistence/operation-log-recovery.service.spec.ts index 746c03e854..3d332d0f46 100644 --- a/src/app/op-log/persistence/operation-log-recovery.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-recovery.service.spec.ts @@ -5,7 +5,6 @@ import { OperationLogStoreService } from './operation-log-store.service'; import { LegacyPfDbService } from '../../core/persistence/legacy-pf-db.service'; import { ClientIdService } from '../../core/util/client-id.service'; import { ActionType, OpType } from '../core/operation.types'; -import { PENDING_OPERATION_EXPIRY_MS } from '../core/operation-log.const'; import { ValidateStateService } from '../validation/validate-state.service'; describe('OperationLogRecoveryService', () => { @@ -28,7 +27,7 @@ describe('OperationLogRecoveryService', () => { 'markRejected', 'markFailed', 'markApplied', - 'markArchivePending', + 'markReducersCommittedAndMergeClocks', 'getUnsynced', ]); mockOpLogStore.setVectorClock.and.resolveTo(undefined); @@ -187,89 +186,54 @@ describe('OperationLogRecoveryService', () => { await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markArchivePending).not.toHaveBeenCalled(); + expect(mockOpLogStore.markReducersCommittedAndMergeClocks).not.toHaveBeenCalled(); expect(mockOpLogStore.markRejected).not.toHaveBeenCalled(); expect(mockOpLogStore.markFailed).not.toHaveBeenCalled(); }); - it('should mark valid crash-interrupted ops as archive-pending', async () => { + it('should quarantine crash-interrupted ops for archive recovery', async () => { const now = Date.now(); const pendingOps = [ { seq: 1, op: { id: 'op1' }, appliedAt: now - 1000, source: 'remote' }, { seq: 2, op: { id: 'op2' }, appliedAt: now - 2000, source: 'remote' }, ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markArchivePending.and.resolveTo(undefined); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1, 2]); + expect(mockOpLogStore.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1, 2], + pendingOps.map((entry) => entry.op), + ); expect(mockOpLogStore.markApplied).not.toHaveBeenCalled(); }); - it('should quarantine ops that exceed PENDING_OPERATION_EXPIRY_MS', async () => { + it('should quarantine regardless of age without charging retry budget', async () => { + // The former PENDING_OPERATION_EXPIRY_MS split changed nothing: every + // crash-interrupted op lands in the same quarantine, and retryCount is + // only ever bumped for an actually attempted archive failure. const now = Date.now(); + const weekMs = 7 * 24 * 60 * 60 * 1000; const pendingOps = [ - { seq: 1, op: { id: 'valid' }, appliedAt: now - 1000, source: 'remote' }, // Valid + { seq: 1, op: { id: 'fresh' }, appliedAt: now - 1000, source: 'remote' }, { seq: 2, - op: { id: 'expired' }, - appliedAt: now - PENDING_OPERATION_EXPIRY_MS - 1, - source: 'remote', - }, // Expired - ] as any; - mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markArchivePending.and.resolveTo(undefined); - mockOpLogStore.markFailed.and.resolveTo(undefined); - - await service.recoverPendingRemoteOps(); - - expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1]); - expect(mockOpLogStore.markFailed).toHaveBeenCalledWith(['expired']); - }); - - it('should quarantine all expired ops', async () => { - const now = Date.now(); - const expiredTime = now - PENDING_OPERATION_EXPIRY_MS - 100000; - const pendingOps = [ - { seq: 1, op: { id: 'old1' }, appliedAt: expiredTime, source: 'remote' }, - { seq: 2, op: { id: 'old2' }, appliedAt: expiredTime - 1000, source: 'remote' }, - ] as any; - mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markFailed.and.resolveTo(undefined); - - await service.recoverPendingRemoteOps(); - - expect(mockOpLogStore.markFailed).toHaveBeenCalledWith(['old1', 'old2']); - expect(mockOpLogStore.markArchivePending).not.toHaveBeenCalled(); - }); - - it('should handle mixed valid and expired ops correctly', async () => { - const now = Date.now(); - const pendingOps = [ - { seq: 1, op: { id: 'valid1' }, appliedAt: now - 1000, source: 'remote' }, - { - seq: 2, - op: { id: 'expired1' }, - appliedAt: now - PENDING_OPERATION_EXPIRY_MS - 1, - source: 'remote', - }, - { seq: 3, op: { id: 'valid2' }, appliedAt: now - 5000, source: 'remote' }, - { - seq: 4, - op: { id: 'expired2' }, - appliedAt: now - PENDING_OPERATION_EXPIRY_MS - 2, + op: { id: 'week-old' }, + appliedAt: now - weekMs, source: 'remote', }, ] as any; mockOpLogStore.getPendingRemoteOps.and.resolveTo(pendingOps); - mockOpLogStore.markArchivePending.and.resolveTo(undefined); - mockOpLogStore.markFailed.and.resolveTo(undefined); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.resolveTo(undefined); await service.recoverPendingRemoteOps(); - expect(mockOpLogStore.markArchivePending).toHaveBeenCalledWith([1, 3]); - expect(mockOpLogStore.markFailed).toHaveBeenCalledWith(['expired1', 'expired2']); + expect(mockOpLogStore.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1, 2], + pendingOps.map((entry) => entry.op), + ); + expect(mockOpLogStore.markFailed).not.toHaveBeenCalled(); }); }); diff --git a/src/app/op-log/persistence/operation-log-recovery.service.ts b/src/app/op-log/persistence/operation-log-recovery.service.ts index b690f99373..f012b07983 100644 --- a/src/app/op-log/persistence/operation-log-recovery.service.ts +++ b/src/app/op-log/persistence/operation-log-recovery.service.ts @@ -8,7 +8,6 @@ import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { Operation, OpType, ActionType } from '../core/operation.types'; import { SINGLETON_ENTITY_ID } from '../core/entity-registry'; import { uuidv7 } from '../../util/uuid-v7'; -import { PENDING_OPERATION_EXPIRY_MS } from '../core/operation-log.const'; import { OpLog } from '../../core/log'; import { AppDataComplete } from '../model/model-config'; import { ValidateStateService } from '../validation/validate-state.service'; @@ -138,11 +137,8 @@ export class OperationLogRecoveryService { * Recovers from pending remote ops that were stored but not applied (crash recovery). * These ops are replayed through reducers during normal hydration, but a crash may * have happened before their archive side effects completed. Move them to the - * archive-pending checkpoint so hydration retries archive work without double-applying - * reducers. - * - * Operations pending for longer than PENDING_OPERATION_EXPIRY_MS are quarantined - * as failed so sync stays blocked until archive recovery succeeds. + * 'archive_pending' checkpoint so hydration retries archive work without + * double-applying reducers; sync stays blocked until that recovery succeeds. */ async recoverPendingRemoteOps(): Promise { const recoveredLegacyFailures = @@ -152,45 +148,23 @@ export class OperationLogRecoveryService { `OperationLogRecoveryService: Re-quarantined ${recoveredLegacyFailures} legacy terminal remote failure(s).`, ); } - const pendingOps = await this.opLogStore.getPendingRemoteOps(); if (pendingOps.length === 0) { return; } - const now = Date.now(); - const validOps = pendingOps.filter( - (e) => now - e.appliedAt < PENDING_OPERATION_EXPIRY_MS, + // Reducers are replayed status-blind during hydration; archive work is + // retried after. Age is irrelevant — every crash-interrupted op lands in + // the same quarantine, and retryCount stays untouched (no attempt was made). + const seqs = pendingOps.map((e) => e.seq); + await this.opLogStore.markReducersCommittedAndMergeClocks( + seqs, + pendingOps.map((entry) => entry.op), ); - const expiredOps = pendingOps.filter( - (e) => now - e.appliedAt >= PENDING_OPERATION_EXPIRY_MS, - ); - - // Keep expired ops visible to retryFailedRemoteOps and the sync safety gate. - if (expiredOps.length > 0) { - const expiredIds = expiredOps.map((e) => e.op.id); - await this.opLogStore.markFailed(expiredIds); - OpLog.warn( - `OperationLogRecoveryService: Quarantined ${expiredOps.length} expired pending remote ops ` + - `(pending > ${PENDING_OPERATION_EXPIRY_MS / (60 * 60 * 1000)}h). ` + - `Oldest was ${Math.round((now - Math.min(...expiredOps.map((e) => e.appliedAt))) / (60 * 60 * 1000))}h old.`, - ); - } - - // Reducers are replayed status-blind during hydration; archive work is retried after. - if (validOps.length > 0) { - const seqs = validOps.map((e) => e.seq); - await this.opLogStore.markArchivePending(seqs); - OpLog.warn( - `OperationLogRecoveryService: Found ${validOps.length} pending remote ops from previous crash. ` + - 'Marking archive work pending (reducers will replay during hydration).', - ); - } - - OpLog.normal( - `OperationLogRecoveryService: Recovered ${validOps.length} pending remote ops, ` + - `quarantined ${expiredOps.length} expired ops.`, + OpLog.warn( + `OperationLogRecoveryService: Found ${pendingOps.length} pending remote ops from previous crash. ` + + 'Quarantined their archive work (reducers will replay during hydration).', ); } diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 1e47f8d613..15e6bb460e 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -1411,6 +1411,143 @@ describe('OperationLogStoreService', () => { }); }); + describe('appendMixedSourceBatchSkipDuplicates', () => { + it('should atomically order remote losers before monotonically clocked local compensations', async () => { + await service.setVectorClock({ testClient: 5, existingClient: 2 }); + const remoteLoser = createTestOperation({ + id: 'remote-loser', + clientId: 'remoteClient', + vectorClock: { remoteClient: 7 }, + }); + const firstCompensation = createTestOperation({ + id: 'first-compensation', + vectorClock: { testClient: 3, remoteClient: 7 }, + }); + const secondCompensation = createTestOperation({ + id: 'second-compensation', + vectorClock: { testClient: 6, otherRemote: 4 }, + }); + + const result = await service.appendMixedSourceBatchSkipDuplicates([ + { ops: [remoteLoser], source: 'remote' }, + { ops: [firstCompensation, secondCompensation], source: 'local' }, + ]); + + expect(result.written.map(({ op }) => op.id)).toEqual([ + 'remote-loser', + 'first-compensation', + 'second-compensation', + ]); + expect(result.written.map(({ source }) => source)).toEqual([ + 'remote', + 'local', + 'local', + ]); + expect(result.written[1].op.vectorClock).toEqual({ + testClient: 6, + existingClient: 2, + remoteClient: 7, + }); + expect(result.written[2].op.vectorClock).toEqual({ + testClient: 7, + existingClient: 2, + remoteClient: 7, + otherRemote: 4, + }); + + const stored = await service.getOpsAfterSeq(0); + expect(stored.map(({ op }) => op.id)).toEqual([ + 'remote-loser', + 'first-compensation', + 'second-compensation', + ]); + expect(stored[1].op.vectorClock).toEqual(result.written[1].op.vectorClock); + expect(stored[2].op.vectorClock).toEqual(result.written[2].op.vectorClock); + expect(await service.getVectorClock()).toEqual(result.written[2].op.vectorClock); + }); + + it('should skip existing and intra-batch duplicate IDs without allocating clocks for them', async () => { + await service.setVectorClock({ testClient: 2 }); + const existingRemote = createTestOperation({ + id: 'existing-remote', + clientId: 'remoteClient', + }); + const newRemote = createTestOperation({ + id: 'new-remote', + clientId: 'remoteClient', + }); + const compensation = createTestOperation({ + id: 'compensation', + vectorClock: { testClient: 1 }, + }); + await service.append(existingRemote, 'remote'); + + const result = await service.appendMixedSourceBatchSkipDuplicates([ + { ops: [existingRemote, newRemote], source: 'remote' }, + { ops: [compensation, compensation], source: 'local' }, + ]); + + expect(result.skippedCount).toBe(2); + expect(result.written.map(({ op }) => op.id)).toEqual([ + 'new-remote', + 'compensation', + ]); + expect(result.written[1].op.vectorClock.testClient).toBe(3); + expect((await service.getVectorClock())?.testClient).toBe(3); + }); + + it('should roll back both source groups and the clock when the clock write fails', async () => { + await service.setVectorClock({ testClient: 4 }); + const adapter = ( + service as unknown as { + _adapter: OpLogDbAdapter; + } + )._adapter; + const originalTransaction = adapter.transaction.bind(adapter); + spyOn(adapter, 'transaction').and.callFake(async (stores, mode, callback) => + originalTransaction(stores, mode, async (tx) => { + const failingTx = new Proxy(tx, { + get: (target, property): unknown => { + if (property === 'put') { + return async (store: string, value: unknown, key?: string | number) => { + if (store === STORE_NAMES.VECTOR_CLOCK) { + throw new Error('injected mixed-batch clock failure'); + } + return target.put(store, value, key); + }; + } + const value = Reflect.get(target, property); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return callback(failingTx); + }), + ); + + await expectAsync( + service.appendMixedSourceBatchSkipDuplicates([ + { + ops: [ + createTestOperation({ + id: 'remote-loser', + clientId: 'remoteClient', + }), + ], + source: 'remote', + }, + { + ops: [createTestOperation({ id: 'compensation' })], + source: 'local', + }, + ]), + ).toBeRejectedWithError('injected mixed-batch clock failure'); + + expect(await service.getOpsAfterSeq(0)).toEqual([]); + service.clearVectorClockCache(); + expect(await service.getVectorClock()).toEqual({ testClient: 4 }); + }); + }); + describe('getOpById', () => { it('should return operation entry by ID', async () => { const op = createTestOperation(); @@ -1430,20 +1567,102 @@ describe('OperationLogStoreService', () => { }); describe('markApplied', () => { - it('should checkpoint reducer-committed operations as archive-pending', async () => { + it('should checkpoint reducer-committed operations as archive_pending', async () => { const op = createTestOperation(); const seq = await service.append(op, 'remote', { pendingApply: true }); - await service.markArchivePending([seq]); + await service.markReducersCommittedAndMergeClocks([seq], [op]); const [stored] = await service.getOpsAfterSeq(0); expect(stored.applicationStatus).toBe('archive_pending'); + // No attempt was made, so no retry budget is charged. + expect(stored.retryCount).toBeUndefined(); expect((await service.getPendingRemoteOps()).length).toBe(0); expect((await service.getFailedRemoteOps()).map((entry) => entry.op.id)).toEqual([ op.id, ]); }); + it('should atomically checkpoint reducer commit and merge its vector clock', async () => { + const op = createTestOperation(); + const seq = await service.append(op, 'remote', { pendingApply: true }); + await service.setVectorClock({ testClient: 2 }); + + await service.markReducersCommittedAndMergeClocks( + [seq], + [{ ...op, vectorClock: { remoteClient: 4 } }], + ); + + const [stored] = await service.getOpsAfterSeq(0); + expect(stored.applicationStatus).toBe('archive_pending'); + expect(await service.getVectorClock()).toEqual({ + testClient: 2, + remoteClient: 4, + }); + }); + + it('should roll back reducer checkpoint and clock when the atomic clock write fails', async () => { + const op = createTestOperation(); + const seq = await service.append(op, 'remote', { pendingApply: true }); + await service.setVectorClock({ testClient: 2 }); + + const adapter = ( + service as unknown as { + _adapter: OpLogDbAdapter; + } + )._adapter; + const originalTransaction = adapter.transaction.bind(adapter); + spyOn(adapter, 'transaction').and.callFake(async (stores, mode, callback) => + originalTransaction(stores, mode, async (tx) => { + const failingTx = new Proxy(tx, { + get: (target, property): unknown => { + if (property === 'put') { + return async (store: string, value: unknown, key?: string | number) => { + if (store === STORE_NAMES.VECTOR_CLOCK) { + throw new Error('injected vector-clock write failure'); + } + return target.put(store, value, key); + }; + } + const value = Reflect.get(target, property); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return callback(failingTx); + }), + ); + + await expectAsync( + service.markReducersCommittedAndMergeClocks( + [seq], + [{ ...op, vectorClock: { remoteClient: 4 } }], + ), + ).toBeRejectedWithError('injected vector-clock write failure'); + + const [stored] = await service.getOpsAfterSeq(0); + expect(stored.applicationStatus).toBe('pending'); + service.clearVectorClockCache(); + expect(await service.getVectorClock()).toEqual({ testClient: 2 }); + }); + + it('should abort the atomic checkpoint when a row is missing or no longer pending', async () => { + const op = createTestOperation(); + const seq = await service.append(op, 'remote'); + await service.setVectorClock({ testClient: 2 }); + + await expectAsync( + service.markReducersCommittedAndMergeClocks( + [seq], + [{ ...op, vectorClock: { remoteClient: 4 } }], + ), + ).toBeRejectedWithError(/requires pending remote operation/); + + const [stored] = await service.getOpsAfterSeq(0); + expect(stored.applicationStatus).toBe('applied'); + service.clearVectorClockCache(); + expect(await service.getVectorClock()).toEqual({ testClient: 2 }); + }); + it('should update applicationStatus from pending to applied', async () => { const op = createTestOperation(); const seq = await service.append(op, 'remote', { pendingApply: true }); @@ -1499,10 +1718,10 @@ describe('OperationLogStoreService', () => { expect(afterMarkApplied[0].applicationStatus).toBe('applied'); }); - it('should update applicationStatus from archive-pending to applied', async () => { + it('should update applicationStatus from reducer-commit checkpoint to applied', async () => { const op = createTestOperation(); const seq = await service.append(op, 'remote', { pendingApply: true }); - await service.markArchivePending([seq]); + await service.markReducersCommittedAndMergeClocks([seq], [op]); await service.markApplied([seq]); @@ -1998,16 +2217,29 @@ describe('OperationLogStoreService', () => { it('should save and load import backup', async () => { const state = { tasks: ['task1', 'task2'], projects: [] }; - const savedAt = await service.saveImportBackup(state); + const backupRef = await service.saveImportBackup(state); const backup = await service.loadImportBackup(); expect(backup).not.toBeNull(); expect(backup!.state).toEqual(state); expect(backup!.savedAt).toBeDefined(); expect(typeof backup!.savedAt).toBe('number'); - // The returned token is the provenance value persisted in the slot, so - // callers can later confirm the backup hasn't been replaced. (#8107) - expect(savedAt).toBe(backup!.savedAt); + expect(backup!.backupId).toBeDefined(); + expect(backupRef).toEqual({ + backupId: backup!.backupId, + savedAt: backup!.savedAt, + }); + }); + + it('should assign distinct opaque IDs to same-millisecond replacements', async () => { + spyOn(Date, 'now').and.returnValue(1234); + + const first = await service.saveImportBackup({ version: 1 }); + const second = await service.saveImportBackup({ version: 2 }); + + expect(first.savedAt).toBe(second.savedAt); + expect(first.backupId).not.toBe(second.backupId); + expect((await service.loadImportBackup())?.backupId).toBe(second.backupId); }); it('should return null when no backup exists', async () => { @@ -2036,6 +2268,17 @@ describe('OperationLogStoreService', () => { expect(backup).toBeNull(); }); + it('should not let a stale identity clear a replacement backup', async () => { + const first = await service.saveImportBackup({ version: 1 }); + const second = await service.saveImportBackup({ version: 2 }); + + await service.clearImportBackup(first.backupId); + + expect((await service.loadImportBackup())?.backupId).toBe(second.backupId); + await service.clearImportBackup(second.backupId); + expect(await service.loadImportBackup()).toBeNull(); + }); + it('should check if backup exists with hasImportBackup', async () => { expect(await service.hasImportBackup()).toBe(false); @@ -2513,6 +2756,140 @@ describe('OperationLogStoreService', () => { expect(await service.isRawRebuildIncomplete()).toBe(false); }); + it('should abort replacement if the captured backup slot was superseded', async () => { + const priorOp = createTestOperation({ id: 'prior-local-op' }); + await service.append(priorOp, 'local'); + const capturedBackup = await service.saveImportBackup({ version: 1 }); + const replacementBackup = await service.saveImportBackup({ version: 2 }); + + await expectAsync( + service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('young'), + archiveOld: createArchive('old'), + backupRef: capturedBackup, + }), + ).toBeRejectedWithError(/backup was superseded/); + + expect((await service.getOpsAfterSeq(0)).map(({ op }) => op.id)).toEqual([ + priorOp.id, + ]); + expect((await service.loadImportBackup())?.backupId).toBe( + replacementBackup.backupId, + ); + expect(await service.isRawRebuildIncomplete()).toBeFalse(); + }); + + it('atomically transitions a completed rebuild to a durable recovery token', async () => { + await service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('young'), + archiveOld: createArchive('old'), + }); + + const backupRef = { + backupId: 'backup-4242', + savedAt: 4242, + }; + await service.saveImportBackup({ original: true }); + const adapter = ( + service as unknown as { + _adapter: OpLogDbAdapter; + } + )._adapter; + await adapter.put(STORE_NAMES.IMPORT_BACKUP, { + id: SINGLETON_KEY, + state: { original: true }, + ...backupRef, + }); + + expect(await service.completeRawRebuild(backupRef)).toBeTrue(); + + expect(await service.isRawRebuildIncomplete()).toBe(false); + expect(await service.loadRawRebuildRecovery()).toEqual( + jasmine.objectContaining({ + backupId: 'backup-4242', + backupSavedAt: 4242, + }), + ); + + await service.clearRawRebuildRecovery('stale-backup'); + expect(await service.loadRawRebuildRecovery()).not.toBeNull(); + await service.clearRawRebuildRecovery('backup-4242'); + expect(await service.loadRawRebuildRecovery()).toBeNull(); + }); + + it('should identity-guard dismissal retirement of marker and backup', async () => { + const backupRef = await service.saveImportBackup({ original: true }); + await service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('young'), + archiveOld: createArchive('old'), + backupRef, + }); + expect(await service.completeRawRebuild(backupRef)).toBeTrue(); + + expect(await service.retireCompletedRawRebuildRecovery('stale-backup')).toBeFalse(); + expect(await service.loadRawRebuildRecovery()).not.toBeNull(); + expect(await service.loadImportBackup()).not.toBeNull(); + + expect( + await service.retireCompletedRawRebuildRecovery(backupRef.backupId), + ).toBeTrue(); + expect(await service.loadRawRebuildRecovery()).toBeNull(); + expect(await service.loadImportBackup()).toBeNull(); + }); + + it('rolls back the incomplete-to-recovery transition when the token write fails', async () => { + await service.runRemoteStateReplacement({ + baselineState: { task: { ids: [], entities: {} } }, + vectorClock: { remote: 1 }, + schemaVersion: 4, + snapshotEntityKeys: [], + archiveYoung: createArchive('young'), + archiveOld: createArchive('old'), + }); + const backupRef = await service.saveImportBackup({ original: true }); + const adapter = ( + service as unknown as { + _adapter: OpLogDbAdapter; + } + )._adapter; + const originalTransaction = adapter.transaction.bind(adapter); + spyOn(adapter, 'transaction').and.callFake(async (stores, mode, callback) => + originalTransaction(stores, mode, async (tx) => { + const failingTx = new Proxy(tx, { + get: (target, property): unknown => { + if (property === 'put') { + return async (): Promise => { + throw new Error('injected recovery-token write failure'); + }; + } + const value = Reflect.get(target, property); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return callback(failingTx); + }), + ); + + await expectAsync(service.completeRawRebuild(backupRef)).toBeRejectedWithError( + 'injected recovery-token write failure', + ); + + expect(await service.isRawRebuildIncomplete()).toBe(true); + expect(await service.loadRawRebuildRecovery()).toBeNull(); + }); + it('durably carries post-crash local ops in the rebuild marker', async () => { const preservedLocalOp = createTestOperation({ id: '01900000-0000-7000-8000-000000000091', @@ -2521,6 +2898,7 @@ describe('OperationLogStoreService', () => { vectorClock: { localClient: 2, remote: 1 }, }); + const backupRef = await service.saveImportBackup({ original: true }); await service.runRemoteStateReplacement({ baselineState: { task: { ids: [], entities: {} } }, vectorClock: { remote: 1 }, @@ -2529,6 +2907,7 @@ describe('OperationLogStoreService', () => { archiveYoung: createArchive('young'), archiveOld: createArchive('old'), preservedLocalOps: [preservedLocalOp], + backupRef, }); expect(await service.getOpsAfterSeq(0)).toEqual([]); @@ -2536,6 +2915,7 @@ describe('OperationLogStoreService', () => { jasmine.objectContaining({ incomplete: true, preservedLocalOps: [preservedLocalOp], + backupRef, }), ); }); diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index 750de52454..5e2b759ccb 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -21,6 +21,7 @@ import { LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION, RAW_REBUILD_INCOMPLETE_META_KEY, + RAW_REBUILD_RECOVERY_META_KEY, OPS_INDEXES, ArchiveStoreEntry, ProfileDataStoreEntry, @@ -53,6 +54,7 @@ import { decodeOperation, encodeOperation, } from './compact/operation-codec.service'; +import { uuidv7 } from '../../util/uuid-v7'; /** * Vector clock entry stored in the vector_clock object store. @@ -63,6 +65,27 @@ interface VectorClockEntry { lastUpdate: number; } +export interface MixedSourceOperationBatch { + ops: readonly Operation[]; + source: 'local' | 'remote'; + options?: { pendingApply?: boolean }; +} + +export interface MixedSourceWrittenOperation { + seq: number; + op: Operation; + source: 'local' | 'remote'; +} + +export interface ImportBackupRef { + backupId: string; + savedAt: number; +} + +export interface ImportBackupEntry extends ImportBackupRef { + state: unknown; +} + /** * Shape stored in the `state_cache` store (keyPath `id`). * @@ -86,6 +109,13 @@ export interface RawRebuildIncompleteEntry { incomplete: true; startedAt: number; preservedLocalOps: Operation[]; + backupRef?: ImportBackupRef; +} + +export interface RawRebuildRecoveryEntry { + backupId: string; + backupSavedAt: number; + completedAt: number; } interface LegacyTerminalRemoteFailuresMigrationEntry { @@ -95,6 +125,7 @@ interface LegacyTerminalRemoteFailuresMigrationEntry { type OpLogMetaEntry = | FullStateOpsMetaEntry | RawRebuildIncompleteEntry + | RawRebuildRecoveryEntry | LegacyTerminalRemoteFailuresMigrationEntry; /** @@ -141,6 +172,53 @@ const getOpId = (op: Operation | CompactOperation): string => { const getStoredOpType = (op: Operation | CompactOperation): string => isCompactOperation(op) ? op.o : op.opType; +/** + * Calculates the durable clock after a reducer-committed remote batch. + * + * Kept pure so both the standalone merge path and the atomic reducer checkpoint + * use exactly the same full-state reset and pruning semantics. + */ +const calculateRemoteClockMerge = ( + currentClock: VectorClock, + ops: readonly Operation[], + currentClientId: string | null, +): VectorClock => { + let mergedClock: VectorClock = { ...currentClock }; + + for (const op of ops) { + if (FULL_STATE_OP_TYPES.has(op.opType)) { + const clockBeforeReset = mergedClock; + if (!currentClientId) { + mergedClock = { ...op.vectorClock }; + continue; + } + + const resetClock: VectorClock = {}; + const importCounter = op.vectorClock[op.clientId]; + if (importCounter !== undefined) { + resetClock[op.clientId] = importCounter; + } + const ownCounter = Math.max( + clockBeforeReset[currentClientId] ?? 0, + op.vectorClock[currentClientId] ?? 0, + ); + if (ownCounter > 0) { + resetClock[currentClientId] = ownCounter; + } + mergedClock = resetClock; + continue; + } + + for (const [clientId, counter] of Object.entries(op.vectorClock)) { + mergedClock[clientId] = Math.max(mergedClock[clientId] ?? 0, counter); + } + } + + return currentClientId + ? limitVectorClockSize(mergedClock, currentClientId) + : mergedClock; +}; + // Note: DBSchema requires literal string keys matching STORE_NAMES values interface OpLogDB extends DBSchema { [STORE_NAMES.OPS]: { @@ -172,6 +250,7 @@ interface OpLogDB extends DBSchema { id: string; state: unknown; savedAt: number; + backupId?: string; }; }; /** @@ -698,21 +777,168 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + async appendMixedSourceBatchSkipDuplicates( + batches: readonly MixedSourceOperationBatch[], + ): Promise<{ written: MixedSourceWrittenOperation[]; skippedCount: number }> { + const nonEmptyBatches = batches.filter((batch) => batch.ops.length > 0); + if (nonEmptyBatches.length === 0) { + return { written: [], skippedCount: 0 }; + } + await this._ensureInit(); - await this._adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { - for (const seq of seqs) { - const entry = await tx.get(STORE_NAMES.OPS, seq); - if (entry?.applicationStatus === 'pending') { + const hasLocalOps = nonEmptyBatches.some((batch) => batch.source === 'local'); + const currentClientId = hasLocalOps + ? await this.clientIdProvider.loadClientId() + : null; + if (hasLocalOps && !currentClientId) { + throw new Error('Cannot append local operations without a current client ID.'); + } + + const storeNames: OpLogStoreName[] = [STORE_NAMES.OPS, STORE_NAMES.VECTOR_CLOCK]; + if ( + nonEmptyBatches.some((batch) => + batch.ops.some((op) => isFullStateOpType(op.opType)), + ) + ) { + storeNames.push(STORE_NAMES.META); + } + + const written: MixedSourceWrittenOperation[] = []; + let skippedCount = 0; + let committedClock: VectorClock | undefined; + + try { + await this._adapter.transaction(storeNames, 'readwrite', async (tx) => { + const currentClockEntry = hasLocalOps + ? await tx.get(STORE_NAMES.VECTOR_CLOCK, SINGLETON_KEY) + : undefined; + let runningClock: VectorClock = { ...(currentClockEntry?.clock ?? {}) }; + let didWriteLocal = false; + + for (const batch of nonEmptyBatches) { + for (const proposedOp of batch.ops) { + const existingKey = await tx.getKeyFromIndex( + STORE_NAMES.OPS, + OPS_INDEXES.BY_ID, + proposedOp.id, + ); + if (existingKey !== undefined) { + skippedCount++; + continue; + } + + let op = proposedOp; + if (batch.source === 'local') { + if (proposedOp.clientId !== currentClientId) { + throw new Error( + 'Cannot append a local operation for a non-current client ID.', + ); + } + const mergedClock: VectorClock = { ...runningClock }; + for (const [clientId, counter] of Object.entries(proposedOp.vectorClock)) { + mergedClock[clientId] = Math.max(mergedClock[clientId] ?? 0, counter); + } + mergedClock[currentClientId] = Math.max( + (runningClock[currentClientId] ?? 0) + 1, + proposedOp.vectorClock[currentClientId] ?? 0, + ); + runningClock = mergedClock; + op = { ...proposedOp, vectorClock: mergedClock }; + didWriteLocal = true; + } + + const entry = this._buildStoredEntry(op, batch.source, batch.options); + const seq = await tx.add(STORE_NAMES.OPS, entry); + await this._recordFullStateOpInTx(tx, entry.op, seq); + written.push({ seq, op, source: batch.source }); + } + } + + if (didWriteLocal) { + committedClock = runningClock; + await tx.put( + STORE_NAMES.VECTOR_CLOCK, + { clock: runningClock, lastUpdate: Date.now() } satisfies VectorClockEntry, + SINGLETON_KEY, + ); + } + }); + } catch (e) { + this._handleAppendError(e); + } + + if (committedClock) { + this._vectorClockCache = { ...committedClock }; + } + return { written, skippedCount }; + } + + /** + * Atomically records reducer completion and merges the corresponding clocks. + * A committed reducer must never be durable without its clock: that would let + * the next local operation be causally older than state already visible in + * NgRx. The in-memory cache is updated only after the transaction commits. + */ + async markReducersCommittedAndMergeClocks( + seqs: number[], + ops: Operation[], + ): Promise { + if (seqs.length !== ops.length) { + throw new Error( + 'markReducersCommittedAndMergeClocks requires one sequence per operation.', + ); + } + if (ops.length === 0) { + return; + } + + await this._ensureInit(); + const currentClientId = await this.clientIdProvider.loadClientId(); + let committedClock: VectorClock | undefined; + + await this._adapter.transaction( + [STORE_NAMES.OPS, STORE_NAMES.VECTOR_CLOCK], + 'readwrite', + async (tx) => { + const currentEntry = await tx.get( + STORE_NAMES.VECTOR_CLOCK, + SINGLETON_KEY, + ); + committedClock = calculateRemoteClockMerge( + currentEntry?.clock ?? {}, + ops, + currentClientId, + ); + + for (const seq of seqs) { + const entry = await tx.get(STORE_NAMES.OPS, seq); + if (entry?.applicationStatus !== 'pending') { + throw new Error( + `Reducer checkpoint requires pending remote operation at seq ${seq}.`, + ); + } entry.applicationStatus = 'archive_pending'; await tx.put(STORE_NAMES.OPS, entry); } - } - }); + + await tx.put( + STORE_NAMES.VECTOR_CLOCK, + { clock: committedClock, lastUpdate: Date.now() } satisfies VectorClockEntry, + SINGLETON_KEY, + ); + }, + ); + + this._vectorClockCache = committedClock ? { ...committedClock } : null; } /** @@ -725,7 +951,7 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { for (const seq of seqs) { const entry = await tx.get(STORE_NAMES.OPS, seq); - // Failed/archive-pending ops can be retried and cleared when successful. + // Reducer-committed/failed ops can be retried and cleared when successful. if ( entry && (entry.applicationStatus === 'pending' || @@ -1178,8 +1404,8 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { @@ -1531,37 +1757,74 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + async saveImportBackup(state: unknown): Promise { await this._ensureInit(); const savedAt = Date.now(); + const backupId = uuidv7(); await this._adapter.put(STORE_NAMES.IMPORT_BACKUP, { id: SINGLETON_KEY, state, savedAt, + backupId, }); - // Returned so callers can later confirm the (single-slot) backup is still - // the one they captured before restoring it — see BackupService. (#8107) - return savedAt; + return { backupId, savedAt }; } /** * Loads the import backup, if one exists. */ - async loadImportBackup(): Promise<{ state: unknown; savedAt: number } | null> { + async loadImportBackup(): Promise { await this._ensureInit(); - const backup = await this._adapter.get<{ state: unknown; savedAt: number }>( - STORE_NAMES.IMPORT_BACKUP, - SINGLETON_KEY, + return this._adapter.transaction( + [STORE_NAMES.IMPORT_BACKUP], + 'readwrite', + async (tx) => { + const backup = await tx.get<{ + state: unknown; + savedAt: number; + backupId?: string; + }>(STORE_NAMES.IMPORT_BACKUP, SINGLETON_KEY); + if (!backup) { + return null; + } + + // Lazily give pre-token backup rows an opaque identity. From this read + // onward even a same-millisecond slot replacement cannot masquerade as + // the backup offered by a durable Undo marker. + const backupId = backup.backupId ?? uuidv7(); + if (backup.backupId === undefined) { + await tx.put(STORE_NAMES.IMPORT_BACKUP, { + id: SINGLETON_KEY, + ...backup, + backupId, + }); + } + return { state: backup.state, savedAt: backup.savedAt, backupId }; + }, ); - return backup ? { state: backup.state, savedAt: backup.savedAt } : null; } /** * Clears the import backup. */ - async clearImportBackup(): Promise { + async clearImportBackup(expectedBackupId?: string): Promise { await this._ensureInit(); - await this._adapter.delete(STORE_NAMES.IMPORT_BACKUP, SINGLETON_KEY); + await this._adapter.transaction( + [STORE_NAMES.IMPORT_BACKUP], + 'readwrite', + async (tx) => { + if (expectedBackupId !== undefined) { + const current = await tx.get<{ backupId?: string }>( + STORE_NAMES.IMPORT_BACKUP, + SINGLETON_KEY, + ); + if (current?.backupId !== expectedBackupId) { + return; + } + } + await tx.delete(STORE_NAMES.IMPORT_BACKUP, SINGLETON_KEY); + }, + ); } /** @@ -1614,6 +1877,7 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { await this._ensureInit(); @@ -1627,9 +1891,21 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + if (opts.backupRef) { + const currentBackup = await tx.get<{ backupId?: string }>( + STORE_NAMES.IMPORT_BACKUP, + SINGLETON_KEY, + ); + if (currentBackup?.backupId !== opts.backupRef.backupId) { + throw new Error( + 'Pre-replace backup was superseded before remote replacement.', + ); + } + } await tx.clear(STORE_NAMES.OPS); await tx.put( STORE_NAMES.META, @@ -1646,9 +1922,13 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + await this._ensureInit(); + return this._adapter.transaction( + [STORE_NAMES.META, STORE_NAMES.IMPORT_BACKUP], + 'readwrite', + async (tx) => { + const currentBackup = backup + ? await tx.get<{ backupId?: string }>(STORE_NAMES.IMPORT_BACKUP, SINGLETON_KEY) + : undefined; + const hasMatchingBackup = + backup !== undefined && currentBackup?.backupId === backup.backupId; + + await tx.delete(STORE_NAMES.META, RAW_REBUILD_INCOMPLETE_META_KEY); + if (!hasMatchingBackup) { + await tx.delete(STORE_NAMES.META, RAW_REBUILD_RECOVERY_META_KEY); + return false; + } + + await tx.put( + STORE_NAMES.META, + { + backupId: backup.backupId, + backupSavedAt: backup.savedAt, + completedAt: Date.now(), + } satisfies RawRebuildRecoveryEntry, + RAW_REBUILD_RECOVERY_META_KEY, + ); + return true; + }, + ); + } + + async loadRawRebuildRecovery(): Promise { + await this._ensureInit(); + const entry = await this._adapter.get>( + STORE_NAMES.META, + RAW_REBUILD_RECOVERY_META_KEY, + ); + if ( + typeof entry?.backupSavedAt !== 'number' || + typeof entry.backupId !== 'string' || + typeof entry.completedAt !== 'number' + ) { + return null; + } + return { + backupId: entry.backupId, + backupSavedAt: entry.backupSavedAt, + completedAt: entry.completedAt, + }; + } + + async clearRawRebuildRecovery(expectedBackupId?: string): Promise { + await this._ensureInit(); + await this._adapter.transaction([STORE_NAMES.META], 'readwrite', async (tx) => { + if (expectedBackupId !== undefined) { + const current = await tx.get>( + STORE_NAMES.META, + RAW_REBUILD_RECOVERY_META_KEY, + ); + if (current?.backupId !== expectedBackupId) { + return; + } + } + await tx.delete(STORE_NAMES.META, RAW_REBUILD_RECOVERY_META_KEY); + }); + } + + /** + * Retires an explicitly dismissed completed-rebuild Undo. Both deletes are + * identity-guarded in one transaction so a stale snack can never clear a + * newer recovery marker or backup occupying the single slot. + */ + async retireCompletedRawRebuildRecovery(backupId: string): Promise { + await this._ensureInit(); + return this._adapter.transaction( + [STORE_NAMES.META, STORE_NAMES.IMPORT_BACKUP], + 'readwrite', + async (tx) => { + const recovery = await tx.get>( + STORE_NAMES.META, + RAW_REBUILD_RECOVERY_META_KEY, + ); + if (recovery?.backupId !== backupId) { + return false; + } + + await tx.delete(STORE_NAMES.META, RAW_REBUILD_RECOVERY_META_KEY); + const backup = await tx.get<{ backupId?: string }>( + STORE_NAMES.IMPORT_BACKUP, + SINGLETON_KEY, + ); + if (backup?.backupId === backupId) { + await tx.delete(STORE_NAMES.IMPORT_BACKUP, SINGLETON_KEY); + } + return true; + }, + ); + } + // ============================================================ // Vector Clock Management (Performance Optimization) // ============================================================ @@ -1796,10 +2189,8 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort FULL_STATE_OP_TYPES.has(op.opType)); - - const mergedClock = fullStateOp - ? { ...fullStateOp.vectorClock } - : { ...currentClock }; + let fullStateOp: Operation | undefined; + for (const op of ops) { + if (FULL_STATE_OP_TYPES.has(op.opType)) { + fullStateOp = op; + } + } if (fullStateOp) { Log.log( @@ -1839,12 +2226,6 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort; + const clockToStore = calculateRemoteClockMerge(currentClock, ops, currentClientId); if (fullStateOp && currentClientId) { - // CLOCK RESET: After a full-state op (SYNC_IMPORT / BACKUP_IMPORT / REPAIR), - // reset the working clock to minimal — only the import client's entry and our - // own entry. This prevents dead client IDs from accumulating in the clock. - // - // The full import clock is preserved in the stored operation for - // SyncImportFilterService to use when filtering pre-import ops. - // Post-import ops are recognized by having the import client's counter - // (see SyncImportFilterService's import-client-counter exception). - clockToStore = {}; - const importClientId = fullStateOp.clientId; - if (mergedClock[importClientId] !== undefined) { - clockToStore[importClientId] = mergedClock[importClientId]; - } - if (currentClientId !== importClientId) { - // Preserve our own counter using the maximum of: - // - mergedClock[currentClientId]: from any of the incoming remote ops - // - currentClock[currentClientId]: our own counter BEFORE the merge - // - // This matters when our own ops (e.g. GLOBAL_CONFIG) created a counter - // that is NOT reflected in the incoming full-state op's clock (because the - // full-state op was created by another client and doesn't know about our ops). - // Without this, the reset would drop our own counter, causing subsequent ops - // to reuse the same counter value and appear as EQUAL (duplicate) to remote - // clients that have already seen our earlier op with that counter. - const myCounter = Math.max( - mergedClock[currentClientId] ?? 0, - currentClock[currentClientId] ?? 0, - ); - if (myCounter > 0) { - clockToStore[currentClientId] = myCounter; - } - } Log.log( `[OpLogStore] mergeRemoteOpClocks: RESET clock to minimal after ${fullStateOp.opType}\n` + - ` Full merged clock (${Object.keys(mergedClock).length} entries): ${vectorClockToString(mergedClock)}\n` + ` Minimal clock (${Object.keys(clockToStore).length} entries): ${vectorClockToString(clockToStore)}`, ); - } else { - // Normal case: prune the merged clock to MAX_VECTOR_CLOCK_SIZE to break the - // inflate/prune cycle: without this, the union of all downloaded ops' - // clocks re-introduces pruned client IDs, exceeding the limit again. - // The server already prunes with the same algorithm on upload. - clockToStore = currentClientId - ? limitVectorClockSize(mergedClock, currentClientId) - : mergedClock; } // DIAGNOSTIC LOGGING: Log merged clock after merge @@ -2012,6 +2352,7 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { await this._ensureInit(); @@ -2034,6 +2375,9 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { + if (opts.requiredImportBackupId !== undefined) { + const currentBackup = await tx.get<{ backupId?: string }>( + STORE_NAMES.IMPORT_BACKUP, + SINGLETON_KEY, + ); + if (currentBackup?.backupId !== opts.requiredImportBackupId) { + throw new Error('Recovery backup was superseded before destructive restore.'); + } + } // Rotate the clientId first, inside this same atomic transaction. // Writing it before the OPS clear means an interrupt injected into a // later step still aborts this queued put — exercising the genuine @@ -2068,6 +2421,7 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort => ({ op: { id }, diff --git a/src/app/op-log/sync/conflict-resolution.service.spec.ts b/src/app/op-log/sync/conflict-resolution.service.spec.ts index 4ec9381732..9db6cf2b52 100644 --- a/src/app/op-log/sync/conflict-resolution.service.spec.ts +++ b/src/app/op-log/sync/conflict-resolution.service.spec.ts @@ -9,7 +9,11 @@ import { BannerId } from '../../core/banner/banner.model'; import { ValidateStateService } from '../validation/validate-state.service'; import { of } from 'rxjs'; import { ActionType, EntityConflict, OpType, Operation } from '../core/operation.types'; -import { VectorClock, VectorClockComparison } from '../../core/util/vector-clock'; +import { + compareVectorClocks, + VectorClock, + VectorClockComparison, +} from '../../core/util/vector-clock'; import { CLIENT_ID_PROVIDER } from '../util/client-id.provider'; import { MAX_VECTOR_CLOCK_SIZE } from '../core/operation-log.const'; import { buildEntityRegistry, ENTITY_REGISTRY } from '../core/entity-registry'; @@ -43,6 +47,30 @@ describe('ConflictResolutionService', () => { schemaVersion: 1, }); + const getMixedLocalOps = (): readonly Operation[] => + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.calls + .allArgs() + .flatMap(([batches]) => + batches.filter((batch) => batch.source === 'local').flatMap((batch) => batch.ops), + ); + + const getMixedRemoteOps = (): readonly Operation[] => + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.calls + .allArgs() + .flatMap(([batches]) => + batches + .filter((batch) => batch.source === 'remote') + .flatMap((batch) => batch.ops), + ); + + const getFirstMixedLocalOp = (): Operation => { + const op = getMixedLocalOps()[0]; + if (!op) { + throw new Error('Expected a local operation in the mixed-resolution batch'); + } + return op; + }; + beforeEach(() => { mockStore = jasmine.createSpyObj('Store', ['select']); // Default: select returns of(undefined) - can be overridden in specific tests @@ -55,15 +83,16 @@ describe('ConflictResolutionService', () => { 'hasOp', 'append', 'appendBatchSkipDuplicates', - 'appendWithVectorClockUpdate', - 'markArchivePending', + 'appendMixedSourceBatchSkipDuplicates', + 'mergeRemoteOpClocks', + 'markReducersCommittedAndMergeClocks', 'markApplied', 'markRejected', 'markFailed', 'getUnsyncedByEntity', - 'mergeRemoteOpClocks', ]); mockOpLogStore.mergeRemoteOpClocks.and.resolveTo(undefined); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.resolveTo(undefined); mockSnackService = jasmine.createSpyObj('SnackService', [ 'open', 'hasPendingPersistentAction', @@ -110,6 +139,16 @@ describe('ConflictResolutionService', () => { skippedCount: 0, }), ); + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.and.callFake(async (batches) => ({ + written: batches.flatMap((batch) => + batch.ops.map((op, index) => ({ + seq: index + 1, + op, + source: batch.source, + })), + ), + skippedCount: 0, + })); }); describe('getCurrentEntityState', () => { @@ -407,9 +446,6 @@ describe('ConflictResolutionService', () => { // Default mock behaviors for LWW tests mockOpLogStore.hasOp.and.resolveTo(false); mockOpLogStore.append.and.callFake((op: Operation) => Promise.resolve(1)); - mockOpLogStore.appendWithVectorClockUpdate.and.callFake((op: Operation) => - Promise.resolve(1), - ); mockOpLogStore.markApplied.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); mockOperationApplier.applyOperations.and.resolveTo({ appliedOps: [] }); @@ -457,11 +493,9 @@ describe('ConflictResolutionService', () => { // Both local and remote ops should be rejected expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['local-1']); - // Remote ops need to be appended first (via batch), then rejected - expect(mockOpLogStore.appendBatchSkipDuplicates).toHaveBeenCalledWith( - [jasmine.objectContaining({ id: 'remote-1' })], - 'remote', - undefined, + // Remote loser and its compensation are persisted atomically, then rejected. + expect(getMixedRemoteOps()).toEqual( + jasmine.arrayContaining([jasmine.objectContaining({ id: 'remote-1' })]), ); expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['remote-1']); // Snack should show local wins @@ -786,11 +820,9 @@ describe('ConflictResolutionService', () => { ); // Second conflict: local wins (newer timestamp) - // Remote op should be appended (via batch) then rejected - expect(mockOpLogStore.appendBatchSkipDuplicates).toHaveBeenCalledWith( + // Remote loser and its compensation are persisted atomically, then rejected. + expect(getMixedRemoteOps()).toEqual( jasmine.arrayContaining([jasmine.objectContaining({ id: 'remote-2' })]), - 'remote', - undefined, ); // Snack notification should reflect both outcomes @@ -1713,6 +1745,100 @@ describe('ConflictResolutionService', () => { expect(result).toEqual({ localWinOpsCreated: 1 }); }); + it('should persist local-win compensation before applying mixed remote winners', async () => { + const now = Date.now(); + const remoteWinner = createOpWithTimestamp('remote-winner', 'client-b', now); + const remoteLoser = createOpWithTimestamp( + 'remote-loser', + 'client-b', + now - 1000, + OpType.Update, + 'task-2', + ); + const localWinOp = createMockLocalWinOp('task-2'); + const conflicts: EntityConflict[] = [ + createConflict( + 'task-1', + [createOpWithTimestamp('local-loser', 'client-a', now - 1000)], + [remoteWinner], + ), + createConflict( + 'task-2', + [ + createOpWithTimestamp( + 'local-winner', + 'client-a', + now, + OpType.Update, + 'task-2', + ), + ], + [remoteLoser], + ), + ]; + const callOrder: string[] = []; + spyOn(service, '_createLocalWinUpdateOp').and.resolveTo(localWinOp); + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.and.callFake( + async (batches) => { + callOrder.push('persist-mixed-resolution'); + return { + written: batches.flatMap((batch) => + batch.ops.map((op, index) => ({ + seq: index + 1, + op, + source: batch.source, + })), + ), + skippedCount: 0, + }; + }, + ); + mockOpLogStore.markRejected.and.callFake(async () => { + callOrder.push('mark-rejected'); + }); + mockOperationApplier.applyOperations.and.callFake(async () => { + callOrder.push('apply-remote'); + throw new Error('remote archive apply failed'); + }); + + await expectAsync( + service.autoResolveConflictsLWW(conflicts), + ).toBeRejectedWithError('remote archive apply failed'); + + expect(callOrder).toEqual([ + 'persist-mixed-resolution', + 'mark-rejected', + 'mark-rejected', + 'apply-remote', + ]); + }); + + it('should not reject or apply remote rows when local-win compensation cannot persist', async () => { + const now = Date.now(); + const conflicts: EntityConflict[] = [ + createConflict( + 'task-1', + [createOpWithTimestamp('local-winner', 'client-a', now)], + [createOpWithTimestamp('remote-loser', 'client-b', now - 1000)], + ), + ]; + const persistenceError = new Error('compensation persistence failed'); + spyOn(service, '_createLocalWinUpdateOp').and.resolveTo( + createMockLocalWinOp('task-1'), + ); + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.and.rejectWith( + persistenceError, + ); + + await expectAsync(service.autoResolveConflictsLWW(conflicts)).toBeRejectedWith( + persistenceError, + ); + + expect(mockOpLogStore.markRejected).not.toHaveBeenCalled(); + expect(mockOperationApplier.applyOperations).not.toHaveBeenCalled(); + expect(mockOpLogStore.appendBatchSkipDuplicates).not.toHaveBeenCalled(); + }); + it('should return 0 when local wins but entity not found', async () => { const now = Date.now(); const conflicts: EntityConflict[] = [ @@ -1736,7 +1862,7 @@ describe('ConflictResolutionService', () => { }); describe('vector clock update', () => { - it('should use appendWithVectorClockUpdate for local-win ops to ensure vector clock is updated atomically', async () => { + it('should atomically append remote losers before local-win compensation', async () => { const now = Date.now(); const conflicts: EntityConflict[] = [ createConflict( @@ -1765,18 +1891,94 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - // Verify appendWithVectorClockUpdate is called (not plain append) - // This ensures the vector clock store is updated atomically with the operation - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledWith( + const batches = + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.calls.mostRecent().args[0]; + expect(batches.map((batch) => batch.source)).toEqual(['remote', 'local']); + expect(batches[1].ops).toEqual([ jasmine.objectContaining({ id: 'lww-update-task-1', actionType: '[TASK] LWW Update' as ActionType, }), - 'local', - ); + ]); }); - it('should call mergeRemoteOpClocks after applying remote ops (remote wins case)', async () => { + it('should create a dominating GLOBAL_CONFIG:tasks compensation for a migrated local row', async () => { + const migratedLocalOp: Operation = { + id: 'legacy-local-config_tasks', + actionType: ActionType.GLOBAL_CONFIG_UPDATE_SECTION, + opType: OpType.Update, + entityType: 'GLOBAL_CONFIG', + entityId: 'tasks', + entityIds: ['tasks'], + payload: { + actionPayload: { + sectionKey: 'tasks', + sectionCfg: { isConfirmBeforeDelete: true }, + }, + entityChanges: [], + }, + clientId: 'localClient', + vectorClock: { localClient: 4 }, + timestamp: 2_000, + schemaVersion: 2, + }; + const currentRemoteOp: Operation = { + id: 'remote-current-tasks', + actionType: ActionType.GLOBAL_CONFIG_UPDATE_SECTION, + opType: OpType.Update, + entityType: 'GLOBAL_CONFIG', + entityId: 'tasks', + entityIds: ['tasks'], + payload: { + actionPayload: { + sectionKey: 'tasks', + sectionCfg: { isConfirmBeforeDelete: false }, + }, + entityChanges: [], + }, + clientId: 'remoteClient', + vectorClock: { remoteClient: 3 }, + timestamp: 1_000, + schemaVersion: 2, + }; + mockClientIdProvider.loadClientId.and.resolveTo('localClient'); + mockStore.select.and.returnValue( + of({ + misc: { unrelatedMiscSetting: 'keep-me' }, + tasks: { isConfirmBeforeDelete: true }, + }), + ); + + const result = await service.autoResolveConflictsLWW([ + { + entityType: 'GLOBAL_CONFIG', + entityId: 'tasks', + localOps: [migratedLocalOp], + remoteOps: [currentRemoteOp], + suggestedResolution: 'manual', + }, + ]); + + const compensation = getFirstMixedLocalOp(); + expect(result.localWinOpsCreated).toBe(1); + expect(getMixedRemoteOps()).toEqual([currentRemoteOp]); + expect(compensation).toEqual( + jasmine.objectContaining({ + actionType: '[GLOBAL_CONFIG] LWW Update' as ActionType, + entityType: 'GLOBAL_CONFIG', + entityId: 'tasks', + vectorClock: { localClient: 5, remoteClient: 3 }, + }), + ); + expect( + compareVectorClocks(compensation.vectorClock, migratedLocalOp.vectorClock), + ).toBe(VectorClockComparison.GREATER_THAN); + expect( + compareVectorClocks(compensation.vectorClock, currentRemoteOp.vectorClock), + ).toBe(VectorClockComparison.GREATER_THAN); + }); + + it('should merge remote-winner clocks before apply and checkpoint them with reducer status', async () => { // REGRESSION TEST: Bug where remote ops applied via conflict resolution // didn't have their clocks merged into the local clock store. // Without clock merge, subsequent local ops would have clocks that are @@ -1802,8 +2004,11 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - // CRITICAL: mergeRemoteOpClocks must be called with applied remote ops expect(mockOpLogStore.mergeRemoteOpClocks).toHaveBeenCalledWith([remoteOp]); + expect(mockOpLogStore.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1], + [remoteOp], + ); }); it('should process deferred actions after merging remote clocks when caller holds lock', async () => { @@ -1819,20 +2024,20 @@ describe('ConflictResolutionService', () => { const callOrder: string[] = []; mockOpLogStore.hasOp.and.resolveTo(false); + mockOpLogStore.mergeRemoteOpClocks.and.callFake(async () => { + callOrder.push('mergeRemoteOpClocks'); + }); mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { callOrder.push('applyOperations'); await options?.onReducersCommitted?.(ops); return { appliedOps: [remoteOp] }; }); - mockOpLogStore.markArchivePending.and.callFake(async () => { - callOrder.push('markArchivePending'); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.callFake(async () => { + callOrder.push('checkpointReducersAndClocks'); }); mockOpLogStore.markApplied.and.callFake(async () => { callOrder.push('markApplied'); }); - mockOpLogStore.mergeRemoteOpClocks.and.callFake(async () => { - callOrder.push('mergeRemoteOpClocks'); - }); mockOpLogStore.markRejected.and.resolveTo(undefined); mockOperationLogEffects.processDeferredActions.and.callFake(async () => { callOrder.push('processDeferredActions'); @@ -1853,15 +2058,15 @@ describe('ConflictResolutionService', () => { callerHoldsOperationLogLock: true, }); expect(callOrder).toEqual([ - 'applyOperations', - 'markArchivePending', 'mergeRemoteOpClocks', + 'applyOperations', + 'checkpointReducersAndClocks', 'markApplied', 'processDeferredActions', ]); }); - it('should call mergeRemoteOpClocks for non-conflicting ops piggybacked through conflict resolution', async () => { + it('should checkpoint clocks for non-conflicting ops piggybacked through conflict resolution', async () => { const now = Date.now(); const conflictRemoteOp = createOpWithTimestamp('remote-1', 'client-b', now); const nonConflictingOp = createOpWithTimestamp( @@ -1895,6 +2100,10 @@ describe('ConflictResolutionService', () => { conflictRemoteOp, nonConflictingOp, ]); + expect(mockOpLogStore.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1, 1], + [conflictRemoteOp, nonConflictingOp], + ); }); }); @@ -2400,9 +2609,6 @@ describe('ConflictResolutionService', () => { beforeEach(() => { mockOpLogStore.hasOp.and.resolveTo(false); mockOpLogStore.append.and.callFake((op: Operation) => Promise.resolve(1)); - mockOpLogStore.appendWithVectorClockUpdate.and.callFake((op: Operation) => - Promise.resolve(1), - ); mockOpLogStore.markApplied.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); }); @@ -2452,11 +2658,10 @@ describe('ConflictResolutionService', () => { // Local archive op should be rejected (will be replaced by new archive op) expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['local-archive']); // New archive op should be appended - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledWith( + expect(getMixedLocalOps()).toContain( jasmine.objectContaining({ actionType: ActionType.TASK_SHARED_MOVE_TO_ARCHIVE, }), - 'local', ); }); @@ -2472,8 +2677,7 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - const appendedOp = mockOpLogStore.appendWithVectorClockUpdate.calls.first() - .args[0] as Operation; + const appendedOp = getFirstMixedLocalOp(); // Merged clock should include entries from both sides and be incremented expect(appendedOp.vectorClock['client-a']).toBeGreaterThanOrEqual(1); expect(appendedOp.vectorClock['client-b']).toBeGreaterThanOrEqual(1); @@ -2497,8 +2701,7 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - const appendedOp = mockOpLogStore.appendWithVectorClockUpdate.calls.first() - .args[0] as Operation; + const appendedOp = getFirstMixedLocalOp(); expect(appendedOp.payload).toEqual(archiveOp.payload); expect(appendedOp.entityIds).toEqual(['task-1', 'task-2', 'task-3']); expect(appendedOp.entityId).toBe('task-1'); @@ -2538,7 +2741,7 @@ describe('ConflictResolutionService', () => { const result = await service.autoResolveConflictsLWW(conflicts); // No local-win op should be created (clientId unavailable) - expect(mockOpLogStore.appendWithVectorClockUpdate).not.toHaveBeenCalled(); + expect(getMixedLocalOps()).toEqual([]); expect(result.localWinOpsCreated).toBe(0); }); @@ -2564,11 +2767,10 @@ describe('ConflictResolutionService', () => { 'local-archive', ]); expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['remote-upd']); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledWith( + expect(getMixedLocalOps()).toContain( jasmine.objectContaining({ actionType: ActionType.TASK_SHARED_MOVE_TO_ARCHIVE, }), - 'local', ); }); @@ -2594,11 +2796,10 @@ describe('ConflictResolutionService', () => { // Archive should still win — it has moveToArchive, so archive-wins logic kicks in // regardless of remote op type expect(mockOpLogStore.markRejected).toHaveBeenCalledWith(['remote-del']); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledWith( + expect(getMixedLocalOps()).toContain( jasmine.objectContaining({ actionType: ActionType.TASK_SHARED_MOVE_TO_ARCHIVE, }), - 'local', ); }); @@ -2650,11 +2851,10 @@ describe('ConflictResolutionService', () => { // Archive-win op should be created (from Conflict 1) expect(result.localWinOpsCreated).toBe(1); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalledWith( + expect(getMixedLocalOps()).toContain( jasmine.objectContaining({ actionType: ActionType.TASK_SHARED_MOVE_TO_ARCHIVE, }), - 'local', ); // No remote ops should be applied to the store — both conflicts resolve @@ -2704,9 +2904,6 @@ describe('ConflictResolutionService', () => { beforeEach(() => { mockOpLogStore.hasOp.and.resolveTo(false); mockOpLogStore.append.and.callFake((op: Operation) => Promise.resolve(1)); - mockOpLogStore.appendWithVectorClockUpdate.and.callFake((op: Operation) => - Promise.resolve(1), - ); mockOpLogStore.markApplied.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); }); @@ -2732,9 +2929,8 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalled(); - const appendedOp = mockOpLogStore.appendWithVectorClockUpdate.calls.first() - .args[0] as Operation; + expect(getMixedLocalOps().length).toBeGreaterThan(0); + const appendedOp = getFirstMixedLocalOp(); // All keys preserved — no client-side pruning (server handles it) expect(Object.keys(appendedOp.vectorClock).length).toBeGreaterThan( MAX_VECTOR_CLOCK_SIZE, @@ -2778,9 +2974,8 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalled(); - const appendedOp = mockOpLogStore.appendWithVectorClockUpdate.calls.first() - .args[0] as Operation; + expect(getMixedLocalOps().length).toBeGreaterThan(0); + const appendedOp = getFirstMixedLocalOp(); // All keys preserved — no client-side pruning (server handles it) expect(Object.keys(appendedOp.vectorClock).length).toBeGreaterThan( MAX_VECTOR_CLOCK_SIZE, @@ -2814,8 +3009,7 @@ describe('ConflictResolutionService', () => { await service.autoResolveConflictsLWW(conflicts); - const appendedOp = mockOpLogStore.appendWithVectorClockUpdate.calls.first() - .args[0] as Operation; + const appendedOp = getFirstMixedLocalOp(); // No client-side pruning — all keys preserved including protected client expect(appendedOp.vectorClock[protectedId]).toBeDefined(); expect(appendedOp.vectorClock[TEST_CLIENT_ID]).toBeDefined(); @@ -2866,7 +3060,7 @@ describe('ConflictResolutionService', () => { // Local archive wins — a new op should be created expect(result.localWinOpsCreated).toBe(1); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalled(); + expect(getMixedLocalOps().length).toBeGreaterThan(0); }); it('should resolve remote moveToArchive winning over local UPDATE with later timestamp', async () => { @@ -2963,7 +3157,7 @@ describe('ConflictResolutionService', () => { // Local archive wins over remote DELETE expect(result.localWinOpsCreated).toBe(1); - expect(mockOpLogStore.appendWithVectorClockUpdate).toHaveBeenCalled(); + expect(getMixedLocalOps().length).toBeGreaterThan(0); }); }); @@ -3899,7 +4093,6 @@ describe('ConflictResolutionService', () => { beforeEach(() => { mockOpLogStore.hasOp.and.resolveTo(false); mockOpLogStore.append.and.callFake(() => Promise.resolve(1)); - mockOpLogStore.appendWithVectorClockUpdate.and.callFake(() => Promise.resolve(1)); mockOpLogStore.markApplied.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); mockOpLogStore.markFailed.and.resolveTo(undefined); @@ -3919,9 +4112,12 @@ describe('ConflictResolutionService', () => { }, ]; - mockOperationApplier.applyOperations.and.resolveTo({ - appliedOps: [], - failedOp: { op: remoteOp, error: new Error('Apply failed for task-1') }, + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { + appliedOps: [], + failedOp: { op: remoteOp, error: new Error('Apply failed for task-1') }, + }; }); // FIXED: Should throw on apply failure (parity with applyNonConflictingOps) @@ -3955,8 +4151,9 @@ describe('ConflictResolutionService', () => { ]; const callOrder: string[] = []; - mockOperationApplier.applyOperations.and.callFake(async () => { + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { callOrder.push('applyOperations'); + await options?.onReducersCommitted?.(ops); return { appliedOps: [], failedOp: { op: remoteOp, error: new Error('Apply failed for task-1') }, @@ -3999,9 +4196,12 @@ describe('ConflictResolutionService', () => { suggestedResolution: 'manual', }, ]; - mockOperationApplier.applyOperations.and.resolveTo({ - appliedOps: [], - failedOp: { op: remoteOp, error: new Error('archive failed') }, + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { + appliedOps: [], + failedOp: { op: remoteOp, error: new Error('archive failed') }, + }; }); mockOperationLogEffects.processDeferredActions.and.rejectWith( new Error('deferred drain failed'), @@ -4017,6 +4217,126 @@ describe('ConflictResolutionService', () => { expect(thrown).toBeInstanceOf(IncompleteRemoteOperationsError); expect((thrown as Error).message).toBe('archive failed'); }); + + it('should not start apply or drain deferred actions when the pre-apply clock merge fails', async () => { + const localOp = createOpForBug('local-1', 'client-a', now - 1000); + const remoteOp = createOpForBug('remote-1', 'client-b', now); + const clockError = new Error('clock merge failed'); + mockOpLogStore.mergeRemoteOpClocks.and.rejectWith(clockError); + + await expectAsync( + service.autoResolveConflictsLWW([ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ]), + ).toBeRejectedWith(clockError); + + expect(mockOperationApplier.applyOperations).not.toHaveBeenCalled(); + expect(mockOpLogStore.markReducersCommittedAndMergeClocks).not.toHaveBeenCalled(); + expect(mockOperationLogEffects.processDeferredActions).not.toHaveBeenCalled(); + }); + + it('should drain deferred actions when the atomic reducer+clock checkpoint fails after pre-merge', async () => { + const localOp = createOpForBug('local-1', 'client-a', now - 1000); + const remoteOp = createOpForBug('remote-1', 'client-b', now); + const checkpointError = new Error('checkpoint failed'); + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.rejectWith(checkpointError); + + await expectAsync( + service.autoResolveConflictsLWW([ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ]), + ).toBeRejectedWith(checkpointError); + + expect(mockOperationLogEffects.processDeferredActions).toHaveBeenCalled(); + }); + + it('should drain deferred actions when reducer dispatch fails after pre-merge', async () => { + const localOp = createOpForBug('local-1', 'client-a', now - 1000); + const remoteOp = createOpForBug('remote-1', 'client-b', now); + const dispatchError = new Error('dispatcher failed'); + mockOperationApplier.applyOperations.and.rejectWith(dispatchError); + + await expectAsync( + service.autoResolveConflictsLWW([ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ]), + ).toBeRejectedWith(dispatchError); + + expect(mockOperationLogEffects.processDeferredActions).toHaveBeenCalled(); + }); + + it('should drain deferred actions when bookkeeping fails after the atomic checkpoint', async () => { + const localOp = createOpForBug('local-1', 'client-a', now - 1000); + const remoteOp = createOpForBug('remote-1', 'client-b', now); + const markAppliedError = new Error('mark applied failed'); + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + mockOpLogStore.markApplied.and.rejectWith(markAppliedError); + + await expectAsync( + service.autoResolveConflictsLWW([ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ]), + ).toBeRejectedWith(markAppliedError); + + expect(mockOperationLogEffects.processDeferredActions).toHaveBeenCalled(); + }); + + it('should preserve a bookkeeping error when the deferred drain also fails', async () => { + const localOp = createOpForBug('local-1', 'client-a', now - 1000); + const remoteOp = createOpForBug('remote-1', 'client-b', now); + const markAppliedError = new Error('mark applied failed'); + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + mockOpLogStore.markApplied.and.rejectWith(markAppliedError); + mockOperationLogEffects.processDeferredActions.and.rejectWith( + new Error('deferred drain failed'), + ); + + await expectAsync( + service.autoResolveConflictsLWW([ + { + entityType: 'TASK', + entityId: 'task-1', + localOps: [localOp], + remoteOps: [remoteOp], + suggestedResolution: 'manual', + }, + ]), + ).toBeRejectedWith(markAppliedError); + }); }); describe('LWW Update payload always has top-level id (#7330)', () => { diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index 5c6e8ee11a..279f2a8b31 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -318,6 +318,7 @@ export class ConflictResolutionService { } = lwwPartitions; const localOpsToReject = [...lwwPartitions.localOpsToReject]; const localOpsToRejectSet = new Set(localOpsToReject); + let writtenLocalWinOps: Operation[] = []; for (const resolution of resolutions) { // Note: localWinOp is undefined for archive-wins sibling conflicts @@ -354,11 +355,29 @@ export class ConflictResolutionService { } // ───────────────────────────────────────────────────────────────────────── - // Batch process local-wins remote ops: filter duplicates and append in batch - // Uses retry to handle race condition (issue #6213) + // Atomically persist remote losers followed by their local-win + // compensations. Hydration is status-blind, so exposing a durable loser + // without its later compensation would let the loser overwrite local state + // after a crash. // ───────────────────────────────────────────────────────────────────────── - if (localWinsRemoteOps.length > 0) { - await this._filterAndAppendOpsWithRetry(localWinsRemoteOps, 'remote'); + if (localWinsRemoteOps.length > 0 || newLocalWinOps.length > 0) { + const result = await this.opLogStore.appendMixedSourceBatchSkipDuplicates([ + { ops: localWinsRemoteOps, source: 'remote' }, + { ops: newLocalWinOps, source: 'local' }, + ]); + writtenLocalWinOps = result.written + .filter((entry) => entry.source === 'local') + .map((entry) => entry.op); + if (result.skippedCount > 0) { + OpLog.verbose( + `ConflictResolutionService: Skipped ${result.skippedCount} duplicate mixed-resolution op(s)`, + ); + } + for (const op of writtenLocalWinOps) { + OpLog.normal( + `ConflictResolutionService: Appended local-win update op ${op.id} for ${op.entityType}:${op.entityId}`, + ); + } } // ───────────────────────────────────────────────────────────────────────── @@ -413,19 +432,21 @@ export class ConflictResolutionService { } // ───────────────────────────────────────────────────────────────────────── - // STEP 5+6: Apply remote ops (single batch) and append local-win update ops. - // The deferred-actions flush in `finally` runs whether the apply succeeded - // or threw — matches the pre-fix replayOperationBatch.finally semantics - // and guarantees buffered local actions don't leak to the next sync. (#7700) + // STEP 5: Apply remote ops in a single batch. + // Merge their clocks before entering the reducer/deferred-action window. + // Pending rows make this durable frontier crash-safe, and any subsequent + // dispatch/checkpoint/bookkeeping failure can drain buffered local actions. + // (#7700) // ───────────────────────────────────────────────────────────────────────── - let didApplyRemoteOps = false; - let primaryIncompleteError: IncompleteRemoteOperationsError | undefined; + let canDrainDeferredActions = false; + let hasPrimaryError = false; try { if (allOpsToApply.length > 0) { OpLog.normal( `ConflictResolutionService: Applying ${allOpsToApply.length} ops in single batch`, ); - didApplyRemoteOps = true; + await this.opLogStore.mergeRemoteOpClocks(allOpsToApply); + canDrainDeferredActions = true; const opIdToSeq = new Map(allStoredOps.map((o) => [o.id, o.seq])); const applyResult = await this.operationApplier.applyOperations(allOpsToApply, { @@ -439,8 +460,10 @@ export class ConflictResolutionService { 'ConflictResolutionService: reducer commit contained an unknown operation.', ); } - await this.opLogStore.markArchivePending(reducerCommittedSeqs); - await this.opLogStore.mergeRemoteOpClocks(reducerCommittedOps); + await this.opLogStore.markReducersCommittedAndMergeClocks( + reducerCommittedSeqs, + reducerCommittedOps, + ); }, }); @@ -489,35 +512,22 @@ export class ConflictResolutionService { throw new IncompleteRemoteOperationsError(applyResult.failedOp.error); } } - - // ─────────────────────────────────────────────────────────────────────── - // Append new update ops for local wins (will sync on next cycle) - // Uses appendWithVectorClockUpdate to ensure vector clock store stays in sync - // ─────────────────────────────────────────────────────────────────────── - for (const op of newLocalWinOps) { - await this.opLogStore.appendWithVectorClockUpdate(op, 'local'); - OpLog.normal( - `ConflictResolutionService: Appended local-win update op ${op.id} for ${op.entityType}:${op.entityId}`, - ); - } } catch (error) { - if (error instanceof IncompleteRemoteOperationsError) { - primaryIncompleteError = error; - } + hasPrimaryError = true; throw error; } finally { - if (didApplyRemoteOps) { + if (canDrainDeferredActions) { try { await processDeferredActionsAfterRemoteApply( this.injector, options.callerHoldsOperationLogLock ?? false, ); } catch (deferredError) { - if (!primaryIncompleteError) { + if (!hasPrimaryError) { throw deferredError; } OpLog.err( - 'ConflictResolutionService: Deferred-action drain also failed after incomplete remote application', + 'ConflictResolutionService: Deferred-action drain also failed after the primary remote-apply error', { name: (deferredError as Error | undefined)?.name }, ); } @@ -525,7 +535,7 @@ export class ConflictResolutionService { } // ───────────────────────────────────────────────────────────────────────── - // STEP 7: Show non-blocking notification + // STEP 6: Show non-blocking notification // // Distinguish "routine" self-healing (reschedule/repeat/archive/done churn // that resolves correctly on its own) from resolutions that discarded a real @@ -538,14 +548,14 @@ export class ConflictResolutionService { } // ───────────────────────────────────────────────────────────────────────── - // STEP 8: Validate and repair state after resolution + // STEP 7: Validate and repair state after resolution // Validation failure flips the SyncSessionValidationService latch — the // wrapper reads it before deciding IN_SYNC vs ERROR. (#7330) // ───────────────────────────────────────────────────────────────────────── const isValid = await this._validateAndRepairAfterResolution(); if (!isValid) this.sessionValidation.setFailed(); - return { localWinOpsCreated: newLocalWinOps.length }; + return { localWinOpsCreated: writtenLocalWinOps.length }; } /** diff --git a/src/app/op-log/sync/remote-ops-processing.service.spec.ts b/src/app/op-log/sync/remote-ops-processing.service.spec.ts index b72cbb6d00..5848c2c44c 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.spec.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.spec.ts @@ -51,6 +51,7 @@ describe('RemoteOpsProcessingService', () => { let compactionServiceSpy: jasmine.SpyObj; let syncImportFilterServiceSpy: jasmine.SpyObj; let operationLogEffectsSpy: jasmine.SpyObj; + let writeFlushServiceSpy: jasmine.SpyObj; const applyAllWithReducerCommit = async ( ops: Operation[], @@ -85,11 +86,10 @@ describe('RemoteOpsProcessingService', () => { 'hasOp', 'append', 'appendBatchSkipDuplicates', - 'appendWithVectorClockUpdate', - 'markArchivePending', + 'mergeRemoteOpClocks', + 'markReducersCommittedAndMergeClocks', 'markApplied', 'markFailed', - 'mergeRemoteOpClocks', 'getUnsyncedByEntity', 'getOpsAfterSeq', 'getLatestFullStateOp', @@ -116,8 +116,9 @@ describe('RemoteOpsProcessingService', () => { ); // By default, no full-state ops in store opLogStoreSpy.getLatestFullStateOp.and.returnValue(Promise.resolve(undefined)); - // By default, mergeRemoteOpClocks succeeds + // By default, both durable clock transitions succeed opLogStoreSpy.mergeRemoteOpClocks.and.resolveTo(); + opLogStoreSpy.markReducersCommittedAndMergeClocks.and.resolveTo(); // By default, clearFullStateOps returns 0 (no ops cleared) opLogStoreSpy.clearFullStateOps.and.resolveTo(0); // By default, clearFullStateOpsExcept returns 0 (no ops cleared) @@ -248,6 +249,17 @@ describe('RemoteOpsProcessingService', () => { isLocalUnsyncedImport: false, }), ); + writeFlushServiceSpy = jasmine.createSpyObj('OperationWriteFlushService', [ + 'flushPendingWrites', + 'flushThenRunExclusive', + ]); + writeFlushServiceSpy.flushPendingWrites.and.resolveTo(); + writeFlushServiceSpy.flushThenRunExclusive.and.callFake( + async (fn: () => Promise) => { + await writeFlushServiceSpy.flushPendingWrites(); + return fn(); + }, + ); TestBed.configureTestingModule({ providers: [ @@ -264,22 +276,10 @@ describe('RemoteOpsProcessingService', () => { { provide: LockService, useValue: lockServiceSpy }, { provide: OperationLogCompactionService, useValue: compactionServiceSpy }, { provide: SyncImportFilterService, useValue: syncImportFilterServiceSpy }, - { - provide: OperationWriteFlushService, - useValue: jasmine.createSpyObj('OperationWriteFlushService', [ - 'flushPendingWrites', - ]), - }, + { provide: OperationWriteFlushService, useValue: writeFlushServiceSpy }, ], }); - // Default: flush resolves immediately - ( - TestBed.inject( - OperationWriteFlushService, - ) as unknown as jasmine.SpyObj - ).flushPendingWrites.and.resolveTo(); - service = TestBed.inject(RemoteOpsProcessingService); schemaMigrationServiceSpy.getCurrentVersion.and.returnValue(1); // Default migration: return op as is @@ -346,10 +346,7 @@ describe('RemoteOpsProcessingService', () => { // Track call order const callOrder: string[] = []; - const writeFlushService = TestBed.inject( - OperationWriteFlushService, - ) as unknown as jasmine.SpyObj; - writeFlushService.flushPendingWrites.and.callFake(async () => { + writeFlushServiceSpy.flushPendingWrites.and.callFake(async () => { callOrder.push('flushPendingWrites'); }); lockServiceSpy.request.and.callFake( @@ -774,6 +771,96 @@ describe('RemoteOpsProcessingService', () => { expect(service.detectConflicts).not.toHaveBeenCalled(); }); + it('should flush and hold the operation-log lock through full-state apply and validation', async () => { + const syncImportOp: Operation = { + id: 'sync-import-exclusive', + opType: OpType.SyncImport, + actionType: '[All] Load All Data' as ActionType, + entityType: 'ALL', + payload: {}, + clientId: 'client-1', + vectorClock: { client1: 1 }, + timestamp: Date.now(), + schemaVersion: 1, + }; + const callOrder: string[] = []; + writeFlushServiceSpy.flushPendingWrites.and.callFake(async () => { + callOrder.push('flush'); + }); + writeFlushServiceSpy.flushThenRunExclusive.and.callFake( + async (fn: () => Promise) => { + await writeFlushServiceSpy.flushPendingWrites(); + callOrder.push('exclusive:start'); + const value = await fn(); + callOrder.push('exclusive:end'); + return value; + }, + ); + opLogStoreSpy.mergeRemoteOpClocks.and.callFake(async () => { + callOrder.push('premerge'); + }); + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + callOrder.push('reducers'); + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + operationLogEffectsSpy.processDeferredActions.and.callFake(async () => { + callOrder.push('deferred'); + }); + const validationSpy = spyOn(service, 'validateAfterSync').and.callFake( + async (callerHoldsOperationLogLock) => { + expect(callerHoldsOperationLogLock).toBeTrue(); + callOrder.push('validation'); + return true; + }, + ); + + await service.processRemoteOps([syncImportOp]); + + expect(writeFlushServiceSpy.flushThenRunExclusive).toHaveBeenCalledTimes(1); + expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ + callerHoldsOperationLogLock: true, + }); + expect(validationSpy).toHaveBeenCalledWith(true); + expect(callOrder).toEqual([ + 'flush', + 'exclusive:start', + 'premerge', + 'reducers', + 'deferred', + 'validation', + 'exclusive:end', + ]); + }); + + it('should propagate an already-held operation-log lock through full-state apply and validation', async () => { + const syncImportOp: Operation = { + id: 'sync-import-under-lock', + opType: OpType.SyncImport, + actionType: '[All] Load All Data' as ActionType, + entityType: 'ALL', + payload: {}, + clientId: 'client-1', + vectorClock: { client1: 1 }, + timestamp: Date.now(), + schemaVersion: 1, + }; + const applySpy = spyOn(service, 'applyNonConflictingOps').and.callThrough(); + const validationSpy = spyOn(service, 'validateAfterSync').and.resolveTo(true); + + await service.processRemoteOps([syncImportOp], { + callerHoldsOperationLogLock: true, + }); + + expect(applySpy).toHaveBeenCalledWith([syncImportOp], true); + expect(validationSpy).toHaveBeenCalledWith(true); + expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ + callerHoldsOperationLogLock: true, + }); + expect(writeFlushServiceSpy.flushThenRunExclusive).not.toHaveBeenCalled(); + expect(writeFlushServiceSpy.flushPendingWrites).not.toHaveBeenCalled(); + }); + it('should log incoming full-state op shape and prior receiver state for diagnostics', async () => { const syncImportOp: Operation = { id: 'sync-import-diag', @@ -1443,7 +1530,7 @@ describe('RemoteOpsProcessingService', () => { ...partial, }); - it('should merge remote ops clocks after applying', async () => { + it('should durably merge clocks before apply and checkpoint them with reducer status', async () => { const remoteOps: Operation[] = [ createFullOp({ id: 'remote-1', vectorClock: { remoteClient: 1 } }), ]; @@ -1458,6 +1545,10 @@ describe('RemoteOpsProcessingService', () => { await service.applyNonConflictingOps(remoteOps); expect(opLogStoreSpy.mergeRemoteOpClocks).toHaveBeenCalledWith(remoteOps); + expect(opLogStoreSpy.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1], + remoteOps, + ); }); it('should flush deferred actions after remote clocks are merged while reusing caller lock', async () => { @@ -1471,15 +1562,15 @@ describe('RemoteOpsProcessingService', () => { await options?.onReducersCommitted?.(ops); return { appliedOps: remoteOps }; }); - opLogStoreSpy.markArchivePending.and.callFake(async () => { - callOrder.push('markArchivePending'); + opLogStoreSpy.mergeRemoteOpClocks.and.callFake(async () => { + callOrder.push('mergeRemoteOpClocks'); + }); + opLogStoreSpy.markReducersCommittedAndMergeClocks.and.callFake(async () => { + callOrder.push('checkpointReducersAndClocks'); }); opLogStoreSpy.markApplied.and.callFake(async () => { callOrder.push('markApplied'); }); - opLogStoreSpy.mergeRemoteOpClocks.and.callFake(async () => { - callOrder.push('mergeRemoteOpClocks'); - }); operationLogEffectsSpy.processDeferredActions.and.callFake(async () => { callOrder.push('processDeferredActions'); }); @@ -1497,15 +1588,15 @@ describe('RemoteOpsProcessingService', () => { callerHoldsOperationLogLock: true, }); expect(callOrder).toEqual([ - 'applyOperations', - 'markArchivePending', 'mergeRemoteOpClocks', + 'applyOperations', + 'checkpointReducersAndClocks', 'markApplied', 'processDeferredActions', ]); }); - it('should NOT call mergeRemoteOpClocks when no ops are applied', async () => { + it('should NOT checkpoint reducers or clocks when no ops are applied', async () => { const remoteOps: Operation[] = [ createFullOp({ id: 'remote-1', vectorClock: { remoteClient: 1 } }), ]; @@ -1518,6 +1609,7 @@ describe('RemoteOpsProcessingService', () => { await service.applyNonConflictingOps(remoteOps); expect(opLogStoreSpy.mergeRemoteOpClocks).not.toHaveBeenCalled(); + expect(opLogStoreSpy.markReducersCommittedAndMergeClocks).not.toHaveBeenCalled(); }); it('should charge only the attempted archive failure and run validation', async () => { @@ -1540,8 +1632,10 @@ describe('RemoteOpsProcessingService', () => { await expectAsync(service.applyNonConflictingOps(remoteOps)).toBeRejected(); - expect(opLogStoreSpy.markArchivePending).toHaveBeenCalledWith([1, 2, 3]); - expect(opLogStoreSpy.mergeRemoteOpClocks).toHaveBeenCalledWith(remoteOps); + expect(opLogStoreSpy.markReducersCommittedAndMergeClocks).toHaveBeenCalledWith( + [1, 2, 3], + remoteOps, + ); expect(opLogStoreSpy.markFailed).toHaveBeenCalledWith(['op-2']); // Should run validation after partial failure expect(validateStateServiceSpy.validateAndRepairCurrentState).toHaveBeenCalledWith( @@ -1577,6 +1671,87 @@ describe('RemoteOpsProcessingService', () => { expect((thrown as Error).message).toBe('archive failed'); }); + it('should not start apply or drain deferred actions when the pre-apply clock merge fails', async () => { + const remoteOps: Operation[] = [createFullOp({ id: 'remote-1' })]; + const clockError = new Error('clock merge failed'); + opLogStoreSpy.mergeRemoteOpClocks.and.rejectWith(clockError); + + await expectAsync(service.applyNonConflictingOps(remoteOps, true)).toBeRejectedWith( + clockError, + ); + + expect(operationApplierServiceSpy.applyOperations).not.toHaveBeenCalled(); + expect(opLogStoreSpy.markReducersCommittedAndMergeClocks).not.toHaveBeenCalled(); + expect(operationLogEffectsSpy.processDeferredActions).not.toHaveBeenCalled(); + }); + + it('should drain deferred actions when the reducer+clock checkpoint fails after the clock merge', async () => { + const remoteOps: Operation[] = [createFullOp({ id: 'remote-1' })]; + const checkpointError = new Error('checkpoint failed'); + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + opLogStoreSpy.markReducersCommittedAndMergeClocks.and.rejectWith(checkpointError); + + await expectAsync(service.applyNonConflictingOps(remoteOps, true)).toBeRejectedWith( + checkpointError, + ); + + expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ + callerHoldsOperationLogLock: true, + }); + }); + + it('should drain deferred actions when reducer dispatch fails after the clock merge', async () => { + const remoteOps: Operation[] = [createFullOp({ id: 'remote-1' })]; + const dispatchError = new Error('dispatcher failed'); + operationApplierServiceSpy.applyOperations.and.rejectWith(dispatchError); + + await expectAsync(service.applyNonConflictingOps(remoteOps, true)).toBeRejectedWith( + dispatchError, + ); + + expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ + callerHoldsOperationLogLock: true, + }); + }); + + it('should drain deferred actions when bookkeeping fails after the reducer+clock checkpoint', async () => { + const remoteOps: Operation[] = [createFullOp({ id: 'remote-1' })]; + const markAppliedError = new Error('mark applied failed'); + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + opLogStoreSpy.markApplied.and.rejectWith(markAppliedError); + + await expectAsync(service.applyNonConflictingOps(remoteOps, true)).toBeRejectedWith( + markAppliedError, + ); + + expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ + callerHoldsOperationLogLock: true, + }); + }); + + it('should preserve a bookkeeping error when the deferred drain also fails', async () => { + const remoteOps: Operation[] = [createFullOp({ id: 'remote-1' })]; + const markAppliedError = new Error('mark applied failed'); + operationApplierServiceSpy.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + opLogStoreSpy.markApplied.and.rejectWith(markAppliedError); + operationLogEffectsSpy.processDeferredActions.and.rejectWith( + new Error('deferred drain failed'), + ); + + await expectAsync(service.applyNonConflictingOps(remoteOps)).toBeRejectedWith( + markAppliedError, + ); + }); + it('should flip the session-validation latch when partial-failure validation fails', async () => { const remoteOps: Operation[] = [ createFullOp({ id: 'op-1' }), diff --git a/src/app/op-log/sync/remote-ops-processing.service.ts b/src/app/op-log/sync/remote-ops-processing.service.ts index 95c03e1337..beb02ef103 100644 --- a/src/app/op-log/sync/remote-ops-processing.service.ts +++ b/src/app/op-log/sync/remote-ops-processing.service.ts @@ -263,13 +263,27 @@ export class RemoteOpsProcessingService { OpLog.normal( 'RemoteOpsProcessingService: Full-state operation detected, skipping conflict detection.', ); - const committedFullStateOpIds = await this.applyNonConflictingOps(validOps); + const callerHoldsOperationLogLock = options?.callerHoldsOperationLogLock ?? false; + const applyAndValidateWithOperationLogLockHeld = async (): Promise => { + const committedFullStateOpIds = await this.applyNonConflictingOps(validOps, true); - // Clean Slate Semantics: SYNC_IMPORT/BACKUP_IMPORT replaces entire state. - // Local synced ops are NOT replayed - the import is an explicit user action - // to restore all clients to a specific point in time. + // Clean Slate Semantics: SYNC_IMPORT/BACKUP_IMPORT replaces entire state. + // Local synced ops are NOT replayed - the import is an explicit user action + // to restore all clients to a specific point in time. + await this.validateAfterSync(true); + return committedFullStateOpIds; + }; + + // Keep the pending-write cutoff, remote-clock premerge, full-state reducer, + // deferred capture drain, and validation in one operation-log critical + // section. Callers already inside that section must not re-enter its + // non-reentrant lock or flush while holding it. + const committedFullStateOpIds = callerHoldsOperationLogLock + ? await applyAndValidateWithOperationLogLockHeld() + : await this.writeFlushService.flushThenRunExclusive( + applyAndValidateWithOperationLogLockHeld, + ); - await this.validateAfterSync(); return { localWinOpsCreated: 0, allOpsFilteredBySyncImport: false, @@ -458,13 +472,12 @@ export class RemoteOpsProcessingService { await this._logFullStateApplyDiagnostics(locallyReplayableOps); - // Mirror autoResolveConflictsLWW: wrap apply in try/finally so deferred - // local actions are flushed whether the apply succeeded or threw. - // Without this, an apply-time throw (e.g. dispatcher error inside the - // wrapped operationApplier.applyOperations) would leave buffered actions - // to leak into the next sync window with stale clocks. (#7700) - let didApplyRemoteOps = false; - let primaryIncompleteError: IncompleteRemoteOperationsError | undefined; + // Mirror autoResolveConflictsLWW: once the incoming remote clocks are + // durable, flush deferred local actions even if reducer dispatch or later + // apply bookkeeping throws. A failed pre-apply clock merge never starts + // the reducer/deferred-action window. (#7700) + let canDrainDeferredActions = false; + let hasPrimaryError = false; let committedFullStateOpIds: string[] = []; try { // Core owns the generic crash-safety ordering. Angular diagnostics, @@ -480,9 +493,14 @@ export class RemoteOpsProcessingService { }), }, isFullStateOperation: this._isFullStateOperation, + // The core invokes this before reducer application starts, after the + // incoming clock frontier is durable. Any subsequent failure can safely + // drain actions buffered by the reducer window. + onRemoteClocksDurable: () => { + canDrainDeferredActions = true; + }, }); - didApplyRemoteOps = result.appendedOps.length > 0; committedFullStateOpIds = result.appendedOps .filter((op) => FULL_STATE_OP_TYPES.has(op.opType)) .map((op) => op.id); @@ -529,20 +547,18 @@ export class RemoteOpsProcessingService { throw new IncompleteRemoteOperationsError(result.failedOp.error); } } catch (error) { - if (error instanceof IncompleteRemoteOperationsError) { - primaryIncompleteError = error; - } + hasPrimaryError = true; throw error; } finally { - if (didApplyRemoteOps) { + if (canDrainDeferredActions) { try { await processDeferredActionsAfterRemoteApply(this.injector, callerHoldsLock); } catch (deferredError) { - if (!primaryIncompleteError) { + if (!hasPrimaryError) { throw deferredError; } OpLog.err( - 'RemoteOpsProcessingService: Deferred-action drain also failed after incomplete remote application', + 'RemoteOpsProcessingService: Deferred-action drain also failed after the primary remote-apply error', { name: (deferredError as Error | undefined)?.name }, ); } diff --git a/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts b/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts index 269f19ceed..054b4647e0 100644 --- a/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts +++ b/src/app/op-log/testing/integration/remote-apply-store-port.integration.spec.ts @@ -205,6 +205,71 @@ const defineStorePortContract = ( testClient: 7, }); }); + + it('should retain a third-client clock from an operation after a full-state import', async () => { + const importOp = createOperation('ordered-import', { + opType: OpType.SyncImport, + clientId: 'importClient', + vectorClock: { importClient: 9, obsoleteClient: 4 }, + }); + const postImportOp = createOperation('ordered-post-import', { + clientId: 'thirdClient', + vectorClock: { importClient: 9, thirdClient: 6 }, + }); + await store.setVectorClock({ staleClient: 5, testClient: 7 }); + const { applier } = createApplier({ appliedOps: [importOp, postImportOp] }); + + await applyRemoteOperations({ + ops: [importOp, postImportOp], + store, + applier, + isFullStateOperation: (op) => op.opType === OpType.SyncImport, + }); + + expect(await store.getVectorClock()).toEqual({ + importClient: 9, + testClient: 7, + thirdClient: 6, + }); + }); + + it('should reset at each full-state operation and merge only the final suffix', async () => { + const firstImport = createOperation('first-import', { + opType: OpType.SyncImport, + clientId: 'firstImportClient', + vectorClock: { firstImportClient: 1 }, + }); + const betweenImports = createOperation('between-imports', { + clientId: 'betweenClient', + vectorClock: { firstImportClient: 1, betweenClient: 3 }, + }); + const secondImport = createOperation('second-import', { + opType: OpType.BackupImport, + clientId: 'secondImportClient', + vectorClock: { secondImportClient: 4, obsoleteImportEntry: 8 }, + }); + const finalSuffix = createOperation('final-suffix', { + clientId: 'suffixClient', + vectorClock: { secondImportClient: 4, suffixClient: 6 }, + }); + await store.setVectorClock({ staleClient: 5, testClient: 7 }); + const ops = [firstImport, betweenImports, secondImport, finalSuffix]; + const { applier } = createApplier({ appliedOps: ops }); + + await applyRemoteOperations({ + ops, + store, + applier, + isFullStateOperation: (op) => + op.opType === OpType.SyncImport || op.opType === OpType.BackupImport, + }); + + expect(await store.getVectorClock()).toEqual({ + secondImportClient: 4, + testClient: 7, + suffixClient: 6, + }); + }); }); }; From ffca0f13970eb8d4bf3982f0cea95ae712f8484f Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:00:30 +0200 Subject: [PATCH 35/47] fix(sync): guard rebuild backup identity and keep undo across reload The pre-replace import backup and the USE_REMOTE rebuild recovery marker were matched by timestamp only, so a concurrent capture or a reload between rebuild completion and snack dismissal could clear or restore the wrong backup. - Import backups carry an opaque backupId (uuidv7); clearImportBackup, replaceAllForRawRebuild and applyRemoteReplaceWithSnapshot verify the expected identity inside their transactions before acting. - Completing a raw rebuild atomically replaces the incomplete marker with a durable raw_rebuild_recovery entry, so the "restore previous data" Undo survives a reload; StartupService re-offers it at boot and dismissal retires exactly the offered backup. - SnackService treats a sticky recovery/update action as a single persistent slot: unrelated transient snacks no longer destroy a visible Undo, and dismissal awaits the snack's dismissFn. --- .../snack-custom.component.spec.ts | 54 ++++++ .../snack-custom/snack-custom.component.ts | 3 +- src/app/core/snack/snack.model.ts | 1 + src/app/core/snack/snack.service.spec.ts | 76 ++++++++ src/app/core/snack/snack.service.ts | 22 ++- src/app/core/startup/startup.service.spec.ts | 53 ++++++ src/app/core/startup/startup.service.ts | 6 +- src/app/op-log/backup/backup.service.spec.ts | 97 ++++++++-- src/app/op-log/backup/backup.service.ts | 89 +++++---- .../sync/operation-log-sync.service.spec.ts | 180 ++++++++++++++++-- .../op-log/sync/operation-log-sync.service.ts | 131 ++++++++++--- 11 files changed, 614 insertions(+), 98 deletions(-) create mode 100644 src/app/core/snack/snack-custom/snack-custom.component.spec.ts create mode 100644 src/app/core/snack/snack.service.spec.ts diff --git a/src/app/core/snack/snack-custom/snack-custom.component.spec.ts b/src/app/core/snack/snack-custom/snack-custom.component.spec.ts new file mode 100644 index 0000000000..cac207a55c --- /dev/null +++ b/src/app/core/snack/snack-custom/snack-custom.component.spec.ts @@ -0,0 +1,54 @@ +import { TestBed } from '@angular/core/testing'; +import { MAT_SNACK_BAR_DATA, MatSnackBarRef } from '@angular/material/snack-bar'; +import { SnackParams } from '../snack.model'; +import { SnackCustomComponent } from './snack-custom.component'; + +describe('SnackCustomComponent', () => { + const createComponent = ( + data: SnackParams, + ): { + component: SnackCustomComponent; + ref: jasmine.SpyObj>; + } => { + const ref = jasmine.createSpyObj>( + 'MatSnackBarRef', + ['dismiss', 'dismissWithAction'], + ); + TestBed.configureTestingModule({ + providers: [ + { provide: MAT_SNACK_BAR_DATA, useValue: data }, + { provide: MatSnackBarRef, useValue: ref }, + ], + }); + const component = TestBed.runInInjectionContext(() => new SnackCustomComponent()); + return { component, ref }; + }; + + afterEach(() => TestBed.resetTestingModule()); + + it('should invoke the non-action dismissal callback from the close control', async () => { + const dismissFn = jasmine.createSpy('dismissFn'); + const { component, ref } = createComponent({ msg: 'Undo', dismissFn }); + + await component.close(); + + expect(dismissFn).toHaveBeenCalled(); + expect(ref.dismiss).toHaveBeenCalled(); + }); + + it('should not invoke the dismissal callback when the action is used', () => { + const actionFn = jasmine.createSpy('actionFn'); + const dismissFn = jasmine.createSpy('dismissFn'); + const { component, ref } = createComponent({ + msg: 'Undo', + actionFn, + dismissFn, + }); + + component.actionClick(); + + expect(actionFn).toHaveBeenCalled(); + expect(dismissFn).not.toHaveBeenCalled(); + expect(ref.dismissWithAction).toHaveBeenCalled(); + }); +}); diff --git a/src/app/core/snack/snack-custom/snack-custom.component.ts b/src/app/core/snack/snack-custom/snack-custom.component.ts index 2e07181c80..39260c9a19 100644 --- a/src/app/core/snack/snack-custom/snack-custom.component.ts +++ b/src/app/core/snack/snack-custom/snack-custom.component.ts @@ -58,8 +58,9 @@ export class SnackCustomComponent implements OnInit, OnDestroy { this.snackBarRef.dismissWithAction(); } - close(ev?: MouseEvent): void { + async close(ev?: MouseEvent): Promise { ev?.stopPropagation(); + await this.data.dismissFn?.(); this.snackBarRef.dismiss(); } } diff --git a/src/app/core/snack/snack.model.ts b/src/app/core/snack/snack.model.ts index 55f30713a9..fca47158b0 100644 --- a/src/app/core/snack/snack.model.ts +++ b/src/app/core/snack/snack.model.ts @@ -14,6 +14,7 @@ export interface SnackParams { actionId?: string; // eslint-disable-next-line actionFn?: Function; + dismissFn?: () => void | Promise; actionPayload?: unknown; config?: MatSnackBarConfig; isSpinner?: boolean; diff --git a/src/app/core/snack/snack.service.spec.ts b/src/app/core/snack/snack.service.spec.ts new file mode 100644 index 0000000000..875f78ebbb --- /dev/null +++ b/src/app/core/snack/snack.service.spec.ts @@ -0,0 +1,76 @@ +import { TestBed } from '@angular/core/testing'; +import { MatSnackBar } from '@angular/material/snack-bar'; +import { TranslateService } from '@ngx-translate/core'; +import { Store } from '@ngrx/store'; +import { EMPTY } from 'rxjs'; +import { LOCAL_ACTIONS } from '../../util/local-actions.token'; +import { SnackParams } from './snack.model'; +import { SnackService } from './snack.service'; + +describe('SnackService', () => { + let service: SnackService; + let openSnackSpy: jasmine.Spy<(params: SnackParams) => void>; + + beforeEach(() => { + TestBed.configureTestingModule({ + providers: [ + SnackService, + { provide: Store, useValue: { dispatch: jasmine.createSpy('dispatch') } }, + { + provide: TranslateService, + useValue: { instant: (value: string): string => value }, + }, + { provide: LOCAL_ACTIONS, useValue: EMPTY }, + { provide: MatSnackBar, useValue: {} }, + ], + }); + service = TestBed.inject(SnackService); + openSnackSpy = spyOn( + service as unknown as { _openSnack: (params: SnackParams) => void }, + '_openSnack', + ); + }); + + it('should not let an ordinary snack replace a persistent recovery action', () => { + service.open({ + msg: 'Recovery available', + actionStr: 'Undo', + config: { duration: 0 }, + }); + + service.open({ msg: 'Sync complete', type: 'SUCCESS' }); + service.open('Another ordinary message'); + + expect(openSnackSpy).toHaveBeenCalledTimes(1); + expect(service.hasPendingPersistentAction()).toBeTrue(); + }); + + it('should allow a newer persistent action to replace the current one', () => { + service.open({ + msg: 'Recovery available', + actionStr: 'Undo', + config: { duration: 0 }, + }); + service.open({ + msg: 'Update required', + actionStr: 'Update', + config: { duration: 0 }, + }); + + expect(openSnackSpy).toHaveBeenCalledTimes(2); + }); + + it('should accept ordinary snacks after the persistent action is closed', () => { + service.open({ + msg: 'Recovery available', + actionStr: 'Undo', + config: { duration: 0 }, + }); + service.close(); + + service.open({ msg: 'Restore complete', type: 'SUCCESS' }); + + expect(openSnackSpy).toHaveBeenCalledTimes(2); + expect(service.hasPendingPersistentAction()).toBeFalse(); + }); +}); diff --git a/src/app/core/snack/snack.service.ts b/src/app/core/snack/snack.service.ts index 08eeda69fa..a5745c96ea 100644 --- a/src/app/core/snack/snack.service.ts +++ b/src/app/core/snack/snack.service.ts @@ -38,15 +38,20 @@ export class SnackService { if (typeof params === 'string') { params = { msg: params }; } + const isPersistentAction = !!(params.actionStr && params.config?.duration === 0); + // A sticky recovery/update action must survive unrelated success, info and + // error feedback in the app's single snack slot. Another sticky actionable + // snack may still replace it intentionally. + if (this._hasPendingPersistentAction && !isPersistentAction) { + return; + } // Track a persistent recovery action (a sticky, actionable snack) // synchronously, before the debounced render, so an immediate follow-up // check (e.g. the header's post-sync success feedback) cannot unknowingly // replace it. `_openSnack` is debounced trailing-edge, so this last-writer // value matches the snack it will actually render — including the case // where a non-persistent snack supersedes a persistent one. - this._hasPendingPersistentAction = !!( - params.actionStr && params.config?.duration === 0 - ); + this._hasPendingPersistentAction = isPersistentAction; this._openSnack(params); } @@ -135,6 +140,17 @@ export class SnackService { } }); + if (actionStr && openedRef) { + openedRef + .onAction() + .pipe(take(1)) + .subscribe(() => { + if (this._ref === openedRef) { + this._hasPendingPersistentAction = false; + } + }); + } + if (actionStr && actionId && this._ref) { this._ref .onAction() diff --git a/src/app/core/startup/startup.service.spec.ts b/src/app/core/startup/startup.service.spec.ts index 2cf692626b..f942f4f5c3 100644 --- a/src/app/core/startup/startup.service.spec.ts +++ b/src/app/core/startup/startup.service.spec.ts @@ -222,6 +222,59 @@ describe('StartupService', () => { }); }); + describe('raw rebuild recovery', () => { + const callRecoveryCheck = async (): Promise => { + await ( + service as unknown as { + _offerInterruptedRebuildRecoveryIfNeeded: () => Promise; + } + )._offerInterruptedRebuildRecoveryIfNeeded(); + }; + + it('should re-offer Undo after reload when a completed recovery token exists', async () => { + const opLogStore = jasmine.createSpyObj('OperationLogStoreService', [ + 'isRawRebuildIncomplete', + 'loadRawRebuildRecovery', + ]); + opLogStore.isRawRebuildIncomplete.and.resolveTo(false); + opLogStore.loadRawRebuildRecovery.and.resolveTo({ + backupId: 'backup-4242', + backupSavedAt: 4242, + completedAt: 5000, + }); + const syncService = jasmine.createSpyObj('OperationLogSyncService', [ + 'offerInterruptedRebuildRecovery', + ]); + syncService.offerInterruptedRebuildRecovery.and.resolveTo(); + Object.assign(service as object, { + _opLogStore: opLogStore, + _injector: { get: () => syncService }, + }); + + await callRecoveryCheck(); + + expect(syncService.offerInterruptedRebuildRecovery).toHaveBeenCalled(); + }); + + it('should not instantiate sync recovery when no marker exists', async () => { + const opLogStore = jasmine.createSpyObj('OperationLogStoreService', [ + 'isRawRebuildIncomplete', + 'loadRawRebuildRecovery', + ]); + opLogStore.isRawRebuildIncomplete.and.resolveTo(false); + opLogStore.loadRawRebuildRecovery.and.resolveTo(null); + const get = jasmine.createSpy('get'); + Object.assign(service as object, { + _opLogStore: opLogStore, + _injector: { get }, + }); + + await callRecoveryCheck(); + + expect(get).not.toHaveBeenCalled(); + }); + }); + describe('_isTourLikelyToBeShown (private)', () => { it('should return false if IS_SKIP_TOUR is set', () => { (localStorage.getItem as jasmine.Spy).and.callFake((key: string) => { diff --git a/src/app/core/startup/startup.service.ts b/src/app/core/startup/startup.service.ts index 8623841fda..4ce370d88b 100644 --- a/src/app/core/startup/startup.service.ts +++ b/src/app/core/startup/startup.service.ts @@ -271,7 +271,11 @@ export class StartupService { */ private async _offerInterruptedRebuildRecoveryIfNeeded(): Promise { try { - if (await this._opLogStore.isRawRebuildIncomplete()) { + const [isIncomplete, completedRecovery] = await Promise.all([ + this._opLogStore.isRawRebuildIncomplete(), + this._opLogStore.loadRawRebuildRecovery(), + ]); + if (isIncomplete || completedRecovery) { await this._injector .get(OperationLogSyncService) .offerInterruptedRebuildRecovery(); diff --git a/src/app/op-log/backup/backup.service.spec.ts b/src/app/op-log/backup/backup.service.spec.ts index 8b5e85ea73..8a8c0fae31 100644 --- a/src/app/op-log/backup/backup.service.spec.ts +++ b/src/app/op-log/backup/backup.service.spec.ts @@ -19,6 +19,7 @@ describe('BackupService', () => { let mockOpLogStore: jasmine.SpyObj; let mockOperationWriteFlushService: jasmine.SpyObj; let mockLockService: jasmine.SpyObj; + const backupRef = { backupId: 'backup-123', savedAt: 123 }; // eslint-disable-next-line @typescript-eslint/explicit-function-return-type const createMinimalValidBackup = () => ({ @@ -116,7 +117,7 @@ describe('BackupService', () => { mockStateSnapshotService.getStateSnapshotAsync.and.resolveTo( createMinimalValidBackup() as any, ); - mockOpLogStore.saveImportBackup.and.resolveTo(123); + mockOpLogStore.saveImportBackup.and.resolveTo(backupRef); mockOpLogStore.loadImportBackup.and.resolveTo(null); mockOpLogStore.clearImportBackup.and.resolveTo(); mockOpLogStore.runDestructiveStateReplacement.and.resolveTo(); @@ -152,12 +153,13 @@ describe('BackupService', () => { expect(mockOpLogStore.saveImportBackup).toHaveBeenCalledWith(snapshot); }); - it('should return the savedAt provenance token from the store', async () => { - mockOpLogStore.saveImportBackup.and.resolveTo(456); + it('should return the opaque backup reference from the store', async () => { + const expectedRef = { backupId: 'backup-456', savedAt: 456 }; + mockOpLogStore.saveImportBackup.and.resolveTo(expectedRef); const token = await service.captureImportBackup(); - expect(token).toBe(456); + expect(token).toEqual(expectedRef); }); it('should propagate errors so the caller can abort the destructive op', async () => { @@ -170,13 +172,20 @@ describe('BackupService', () => { describe('restoreImportBackup (#8107)', () => { it('should import the saved snapshot and return true when one exists', async () => { const saved = createMinimalValidBackup(); - mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, savedAt: 123 }); + mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, ...backupRef }); const importSpy = spyOn(service, 'importCompleteBackup').and.resolveTo(); const result = await service.restoreImportBackup(); expect(result).toBe(true); - expect(importSpy).toHaveBeenCalledWith(saved as any, true, true, true); + expect(importSpy).toHaveBeenCalledWith( + saved as any, + true, + true, + true, + true, + backupRef.backupId, + ); }); it('should return false and not import when no backup exists', async () => { @@ -191,42 +200,83 @@ describe('BackupService', () => { it('should clear the single-slot backup after a successful restore', async () => { const saved = createMinimalValidBackup(); - mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, savedAt: 123 }); + mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, ...backupRef }); spyOn(service, 'importCompleteBackup').and.resolveTo(); await service.restoreImportBackup(); - expect(mockOpLogStore.clearImportBackup).toHaveBeenCalled(); + expect(mockOpLogStore.clearImportBackup).toHaveBeenCalledWith(backupRef.backupId); }); it('should restore when the provenance token matches the stored backup', async () => { const saved = createMinimalValidBackup(); - mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, savedAt: 777 }); + const expectedRef = { backupId: 'backup-777', savedAt: 777 }; + mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, ...expectedRef }); const importSpy = spyOn(service, 'importCompleteBackup').and.resolveTo(); - const result = await service.restoreImportBackup(777); + const result = await service.restoreImportBackup(expectedRef); expect(result).toBe(true); - expect(importSpy).toHaveBeenCalledWith(saved as any, true, true, true); + expect(importSpy).toHaveBeenCalledWith( + saved as any, + true, + true, + true, + true, + expectedRef.backupId, + ); }); it('should refuse to restore (and not clear) when the backup was superseded since capture', async () => { // The single slot is shared with the backup-import flow; an intervening - // write changes savedAt. Restoring it would silently roll back to the + // write changes backupId. Restoring it would silently roll back to the // wrong snapshot, so we must refuse. (#8107) const saved = createMinimalValidBackup(); - mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, savedAt: 999 }); + mockOpLogStore.loadImportBackup.and.resolveTo({ + state: saved, + backupId: 'replacement-backup', + savedAt: 777, + }); const importSpy = spyOn(service, 'importCompleteBackup').and.resolveTo(); - const result = await service.restoreImportBackup(777); + const result = await service.restoreImportBackup({ + backupId: 'expected-backup', + savedAt: 777, + }); expect(result).toBe(false); expect(importSpy).not.toHaveBeenCalled(); expect(mockOpLogStore.clearImportBackup).not.toHaveBeenCalled(); }); + + it('should keep the same backup usable when a restore attempt fails', async () => { + const saved = createMinimalValidBackup(); + mockOpLogStore.loadImportBackup.and.resolveTo({ state: saved, ...backupRef }); + const importSpy = spyOn(service, 'importCompleteBackup').and.rejectWith( + new Error('restore failed'), + ); + + await expectAsync(service.restoreImportBackup(backupRef)).toBeRejected(); + + expect(mockOpLogStore.clearImportBackup).not.toHaveBeenCalled(); + importSpy.and.resolveTo(); + await expectAsync(service.restoreImportBackup(backupRef)).toBeResolvedTo(true); + }); }); describe('importCompleteBackup', () => { + it('should reject inconsistent skip-backup provenance arguments', async () => { + const backup = createMinimalValidBackup() as any; + + await expectAsync( + service.importCompleteBackup(backup, true, true, true, true), + ).toBeRejectedWithError(/requires exactly one verified recovery backup ID/); + await expectAsync( + service.importCompleteBackup(backup, true, true, true, false, backupRef.backupId), + ).toBeRejectedWithError(/requires exactly one verified recovery backup ID/); + expect(mockOpLogStore.runDestructiveStateReplacement).not.toHaveBeenCalled(); + }); + it('should dispatch loadAllData with the imported data', async () => { const backupData = createMinimalValidBackup(); @@ -470,6 +520,25 @@ describe('BackupService', () => { expect(mockOpLogStore.saveImportBackup).toHaveBeenCalledWith(currentState); }); + it('should keep the recovery slot unchanged while restoring that backup', async () => { + await service.importCompleteBackup( + createMinimalValidBackup() as any, + true, + true, + true, + true, + backupRef.backupId, + ); + + expect(mockStateSnapshotService.getStateSnapshotAsync).not.toHaveBeenCalled(); + expect(mockOpLogStore.saveImportBackup).not.toHaveBeenCalled(); + expect(mockOpLogStore.runDestructiveStateReplacement).toHaveBeenCalled(); + expect( + mockOpLogStore.runDestructiveStateReplacement.calls.mostRecent().args[0] + .requiredImportBackupId, + ).toBe(backupRef.backupId); + }); + it('should pass snapshotEntityKeys derived from the imported data', async () => { const backupData = createMinimalValidBackup(); diff --git a/src/app/op-log/backup/backup.service.ts b/src/app/op-log/backup/backup.service.ts index a9e38a3e90..aa70102be7 100644 --- a/src/app/op-log/backup/backup.service.ts +++ b/src/app/op-log/backup/backup.service.ts @@ -2,7 +2,10 @@ import { inject, Injectable } from '@angular/core'; import { Store } from '@ngrx/store'; import { ImexViewService } from '../../imex/imex-meta/imex-view.service'; import { StateSnapshotService } from './state-snapshot.service'; -import { OperationLogStoreService } from '../persistence/operation-log-store.service'; +import { + ImportBackupRef, + OperationLogStoreService, +} from '../persistence/operation-log-store.service'; import { generateClientId } from '../../core/util/generate-client-id'; import { Operation, OpType, ActionType } from '../core/operation.types'; import { CURRENT_SCHEMA_VERSION } from '../persistence/schema-migration.service'; @@ -73,13 +76,24 @@ export class BackupService { * @param isSkipLegacyWarnings - If true, skip legacy data format warnings * @param isSkipReload - If true, don't reload the page after import * @param isForceConflict - If true, reload page after import + * @param isSkipPreImportBackup - Keep an existing recovery backup in its + * single slot while restoring that exact backup. + * @param requiredImportBackupId - Abort the destructive commit unless this + * backup still occupies the single recovery slot. */ async importCompleteBackup( data: AppDataComplete | CompleteBackup, isSkipLegacyWarnings: boolean = false, isSkipReload: boolean = false, isForceConflict: boolean = false, + isSkipPreImportBackup: boolean = false, + requiredImportBackupId?: string, ): Promise { + if (isSkipPreImportBackup !== (requiredImportBackupId !== undefined)) { + throw new Error( + 'BackupService: Skipping the pre-import backup requires exactly one verified recovery backup ID.', + ); + } try { this._imexViewService.setDataImportInProgress(true); @@ -147,7 +161,11 @@ export class BackupService { // 4. Persist to operation log await this._operationWriteFlushService.flushPendingWrites(); await this._lockService.request(LOCK_NAMES.OPERATION_LOG, async () => { - await this._persistImportToOperationLog(validatedData); + await this._persistImportToOperationLog( + validatedData, + isSkipPreImportBackup, + requiredImportBackupId, + ); }); // 4b. Reset all sync providers' lastServerSeq to 0. @@ -180,11 +198,11 @@ export class BackupService { * * Mirrors the pre-import backup taken in `_persistImportToOperationLog`. Errors * propagate so the caller can abort the destructive operation rather than wipe - * local data without a recovery point. Returns the backup's `savedAt` token so - * the caller can later verify the (single-slot) backup hasn't been replaced by - * an unrelated write before restoring it. (#8107) + * local data without a recovery point. Returns the backup's opaque ID plus its + * display timestamp so the caller can verify that the single slot has not been + * replaced by an unrelated write before restoring it. (#8107) */ - async captureImportBackup(): Promise { + async captureImportBackup(): Promise { const currentState = await this._stateSnapshotService.getStateSnapshotAsync(); return this._opLogStore.saveImportBackup(currentState); } @@ -194,22 +212,21 @@ export class BackupService { * before a backup import) — if one exists. Returns false when there is nothing * to restore. Used by the post-replace "Undo" affordance. * - * The backup state is read BEFORE `importCompleteBackup` runs (which itself - * re-snapshots the current state into the same slot), so the good state is - * already in hand and round-trips correctly. + * The backup state is read before `importCompleteBackup` runs. This recovery + * path skips the normal pre-import snapshot so the original remains durable + * until the destructive import has fully succeeded. * - * @param expectedSavedAt - When provided, only restore if the stored backup - * still carries this `savedAt` token. The slot is shared with the backup- + * @param expectedBackup - When provided, only restore if the stored backup + * still carries this opaque backup ID. The slot is shared with the backup- * import flow, so an intervening import (or a second "Use Server Data") - * would overwrite it; restoring that wrong snapshot is silent data loss, so - * we refuse instead. (#8107) + * would overwrite it; restoring that wrong snapshot is silent data loss. */ - async restoreImportBackup(expectedSavedAt?: number): Promise { + async restoreImportBackup(expectedBackup?: ImportBackupRef): Promise { const backup = await this._opLogStore.loadImportBackup(); if (!backup) { return false; } - if (expectedSavedAt !== undefined && backup.savedAt !== expectedSavedAt) { + if (expectedBackup && backup.backupId !== expectedBackup.backupId) { OpLog.warn( 'BackupService: Import backup was superseded since capture; skipping restore to avoid restoring the wrong snapshot.', ); @@ -220,15 +237,20 @@ export class BackupService { true, // isSkipLegacyWarnings true, // isSkipReload - loadAllData updates state live true, // isForceConflict + true, // keep this exact recovery backup until the full restore succeeds + backup.backupId, ); - // Restored data is now live; drop the (now stale) single-slot backup so a - // full copy of the replaced state doesn't linger in IndexedDB. (#8107) - await this._opLogStore.clearImportBackup(); + // Retire the restored slot only if it still has the same opaque identity. + // The import path or another tab may have created a newer safety backup + // while the async restore ran; that newer backup must survive. (#8107) + await this._opLogStore.clearImportBackup(backup.backupId); return true; } private async _persistImportToOperationLog( importedData: AppDataComplete, + isSkipPreImportBackup: boolean, + requiredImportBackupId?: string, ): Promise { OpLog.normal('BackupService: Persisting import to operation log...'); @@ -237,20 +259,22 @@ export class BackupService { // after this method returns, so silently skipping the destructive write // would leave the device in a hybrid state (imported NgRx/archives, old // op-log) that is worse than either outcome. - try { - const currentState = await this._stateSnapshotService.getStateSnapshotAsync(); - OpLog.normal('BackupService: Backing up current state before import...'); - await this._opLogStore.saveImportBackup(currentState); - } catch (e) { - // `message` is intentionally omitted: log history is user-exportable - // (CLAUDE.md sync rule 9), and a future validator/IDB error type could - // interpolate user content into its message. Log the error `name` only. - OpLog.warn('BackupService: Failed to backup state before import:', { - name: (e as Error | undefined)?.name, - }); - throw new Error( - 'BackupService: Pre-import backup failed; aborting import to preserve local state.', - ); + if (!isSkipPreImportBackup) { + try { + const currentState = await this._stateSnapshotService.getStateSnapshotAsync(); + OpLog.normal('BackupService: Backing up current state before import...'); + await this._opLogStore.saveImportBackup(currentState); + } catch (e) { + // `message` is intentionally omitted: log history is user-exportable + // (CLAUDE.md sync rule 9), and a future validator/IDB error type could + // interpolate user content into its message. Log the error `name` only. + OpLog.warn('BackupService: Failed to backup state before import:', { + name: (e as Error | undefined)?.name, + }); + throw new Error( + 'BackupService: Pre-import backup failed; aborting import to preserve local state.', + ); + } } // Mint a fresh clientId for the new sync baseline. It is pure here — @@ -287,6 +311,7 @@ export class BackupService { snapshotEntityKeys: extractEntityKeysFromState(importedData), archiveYoung: importedData.archiveYoung, archiveOld: importedData.archiveOld, + requiredImportBackupId, }); OpLog.normal('BackupService: Import persisted to operation log.'); diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index 35b00d15b9..d0373ff6ac 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -69,6 +69,9 @@ describe('OperationLogSyncService', () => { let operationApplierSpy: jasmine.SpyObj; let operationLogEffectsSpy: jasmine.SpyObj; let hydratorServiceSpy: jasmine.SpyObj; + const defaultBackupRef = { backupId: 'backup-1', savedAt: 1 }; + const backupRef4242 = { backupId: 'backup-4242', savedAt: 4242 }; + const backupRef12345 = { backupId: 'backup-12345', savedAt: 12345 }; const createProviderSetupEntry = (): OperationLogEntry => ({ seq: 1, @@ -91,6 +94,7 @@ describe('OperationLogSyncService', () => { beforeEach(() => { snackServiceSpy = jasmine.createSpyObj('SnackService', [ 'open', + 'close', 'hasPendingPersistentAction', ]); snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); @@ -112,6 +116,10 @@ describe('OperationLogSyncService', () => { 'isRawRebuildIncomplete', 'loadRawRebuildIncomplete', 'clearRawRebuildIncomplete', + 'completeRawRebuild', + 'loadRawRebuildRecovery', + 'clearRawRebuildRecovery', + 'retireCompletedRawRebuildRecovery', 'loadImportBackup', ]); opLogStoreSpy.hasSyncedOps.and.resolveTo(true); @@ -131,6 +139,10 @@ describe('OperationLogSyncService', () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo(null); opLogStoreSpy.clearRawRebuildIncomplete.and.resolveTo(); + opLogStoreSpy.completeRawRebuild.and.resolveTo(true); + opLogStoreSpy.loadRawRebuildRecovery.and.resolveTo(null); + opLogStoreSpy.clearRawRebuildRecovery.and.resolveTo(); + opLogStoreSpy.retireCompletedRawRebuildRecovery.and.resolveTo(true); opLogStoreSpy.loadImportBackup.and.resolveTo(null); schemaMigrationServiceSpy = jasmine.createSpyObj('SchemaMigrationService', [ @@ -173,7 +185,7 @@ describe('OperationLogSyncService', () => { 'captureImportBackup', 'restoreImportBackup', ]); - backupServiceSpy.captureImportBackup.and.resolveTo(1); + backupServiceSpy.captureImportBackup.and.resolveTo(defaultBackupRef); backupServiceSpy.restoreImportBackup.and.resolveTo(true); remoteOpsProcessingServiceSpy = jasmine.createSpyObj('RemoteOpsProcessingService', [ @@ -397,6 +409,12 @@ describe('OperationLogSyncService', () => { it('should block upload while a raw rebuild remains incomplete', async () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [], + backupRef: backupRef4242, + }); await expectAsync( service.uploadPendingOps({} as OperationSyncCapable), @@ -407,6 +425,12 @@ describe('OperationLogSyncService', () => { it('should not attempt the in-session archive retry while a raw rebuild is incomplete', async () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [], + backupRef: backupRef4242, + }); await expectAsync( service.uploadPendingOps({} as OperationSyncCapable), @@ -1010,7 +1034,7 @@ describe('OperationLogSyncService', () => { describe('downloadRemoteOps', () => { it('should block download while a prior remote operation is incompletely applied', async () => { opLogStoreSpy.getFailedRemoteOps.and.resolveTo([ - { applicationStatus: 'archive_pending' } as OperationLogEntry, + { applicationStatus: 'failed' } as OperationLogEntry, ]); await expectAsync( @@ -1088,7 +1112,16 @@ describe('OperationLogSyncService', () => { it('should flush deferred local work before entering crash-resume rebuild', async () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); - opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [], + backupRef: backupRef4242, + }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef4242, + }); operationLogEffectsSpy.processDeferredActions.and.rejectWith( new Error('deferred write failed'), ); @@ -1111,10 +1144,19 @@ describe('OperationLogSyncService', () => { // empty/newer-schema remote). Without an escape hatch the user is stuck // on the baseline with the pre-replace backup hidden — surface Undo. opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [], + backupRef: backupRef4242, + }); spyOn(service, 'forceDownloadRemoteState').and.rejectWith( new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'), ); - opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef4242, + }); snackServiceSpy.hasPendingPersistentAction.and.returnValue(false); const mockProvider = { isReady: () => Promise.resolve(true) } as any; @@ -1127,13 +1169,22 @@ describe('OperationLogSyncService', () => { it('should allow uploads after stranded-rebuild Undo clears the marker', async () => { let isRawRebuildIncomplete = true; + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [], + backupRef: backupRef4242, + }); opLogStoreSpy.isRawRebuildIncomplete.and.callFake( async () => isRawRebuildIncomplete, ); spyOn(service, 'forceDownloadRemoteState').and.rejectWith( new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'), ); - opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef4242, + }); backupServiceSpy.restoreImportBackup.and.callFake(async () => { isRawRebuildIncomplete = false; return true; @@ -1165,16 +1216,99 @@ describe('OperationLogSyncService', () => { await service.uploadPendingOps(mockProvider); - expect(backupServiceSpy.restoreImportBackup).toHaveBeenCalledWith(4242); + expect(backupServiceSpy.restoreImportBackup).toHaveBeenCalledWith(backupRef4242); + expect(opLogStoreSpy.clearRawRebuildRecovery).toHaveBeenCalledWith( + backupRef4242.backupId, + ); expect(uploadServiceSpy.uploadPendingOps).toHaveBeenCalled(); }); + it('should re-offer a completed rebuild Undo from its durable token', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); + opLogStoreSpy.loadRawRebuildRecovery.and.resolveTo({ + backupId: backupRef4242.backupId, + backupSavedAt: 4242, + completedAt: 5000, + }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef4242, + }); + + await service.offerInterruptedRebuildRecovery(); + + expect(snackServiceSpy.open).toHaveBeenCalledWith( + jasmine.objectContaining({ + msg: T.F.SYNC.S.LOCAL_DATA_REPLACE_UNDO, + actionStr: T.G.UNDO, + config: { duration: 0 }, + }), + ); + + const recoverySnack = snackServiceSpy.open.calls.mostRecent().args[0]; + if (typeof recoverySnack === 'string' || recoverySnack.dismissFn === undefined) { + throw new Error('Expected durable recovery dismissal callback'); + } + await recoverySnack.dismissFn(); + expect(opLogStoreSpy.retireCompletedRawRebuildRecovery).toHaveBeenCalledWith( + backupRef4242.backupId, + ); + + // A later startup sees no marker and does not resurrect dismissed Undo. + snackServiceSpy.open.calls.reset(); + opLogStoreSpy.loadRawRebuildRecovery.and.resolveTo(null); + await service.offerInterruptedRebuildRecovery(); + expect(snackServiceSpy.open).not.toHaveBeenCalled(); + }); + + it('should not offer an incomplete rebuild backup whose identity no longer matches', async () => { + opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ + incomplete: true, + startedAt: 1, + preservedLocalOps: [], + backupRef: backupRef4242, + }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + backupId: 'replacement-backup', + savedAt: 4242, + }); + + await service.offerInterruptedRebuildRecovery(); + + expect(snackServiceSpy.open).not.toHaveBeenCalled(); + }); + + it('should discard a completed recovery token when the backup slot was superseded', async () => { + opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); + opLogStoreSpy.loadRawRebuildRecovery.and.resolveTo({ + backupId: backupRef4242.backupId, + backupSavedAt: 4242, + completedAt: 5000, + }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + backupId: 'replacement-backup', + savedAt: 9999, + }); + + await service.offerInterruptedRebuildRecovery(); + + expect(opLogStoreSpy.clearRawRebuildRecovery).toHaveBeenCalledWith( + backupRef4242.backupId, + ); + expect(snackServiceSpy.open).not.toHaveBeenCalled(); + }); + it('should not respawn the recovery snack while one is already showing', async () => { opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(true); spyOn(service, 'forceDownloadRemoteState').and.rejectWith( new Error('USE_REMOTE aborted: remote returned no data to rebuild from.'), ); - opLogStoreSpy.loadImportBackup.and.resolveTo({ savedAt: 4242 } as any); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef4242, + }); // A persistent recovery snack from a previous resume attempt is still up. snackServiceSpy.hasPendingPersistentAction.and.returnValue(true); const mockProvider = { isReady: () => Promise.resolve(true) } as any; @@ -2859,7 +2993,7 @@ describe('OperationLogSyncService', () => { }); backupServiceSpy.captureImportBackup.and.callFake(async () => { callOrder.push('captureImportBackup'); - return 1; + return defaultBackupRef; }); writeFlushServiceSpy.flushPendingWrites.and.callFake(async () => { callOrder.push('flushPendingWrites'); @@ -2924,8 +3058,9 @@ describe('OperationLogSyncService', () => { blockedByIncompatibleOp: false, }; }); - opLogStoreSpy.clearRawRebuildIncomplete.and.callFake(async () => { - callOrder.push('clearRawRebuildIncomplete'); + opLogStoreSpy.completeRawRebuild.and.callFake(async () => { + callOrder.push('completeRawRebuild'); + return true; }); const mockProvider = { supportsOperationSync: true, @@ -2934,7 +3069,8 @@ describe('OperationLogSyncService', () => { await service.forceDownloadRemoteState(mockProvider); - expect(callOrder).toEqual(['processRemoteOps', 'clearRawRebuildIncomplete']); + expect(callOrder).toEqual(['processRemoteOps', 'completeRawRebuild']); + expect(opLogStoreSpy.completeRawRebuild).toHaveBeenCalledWith(defaultBackupRef); }); it('should NOT clear the raw-rebuild-incomplete marker when the replay is blocked', async () => { @@ -2960,7 +3096,7 @@ describe('OperationLogSyncService', () => { await expectAsync(service.forceDownloadRemoteState(mockProvider)).toBeRejected(); - expect(opLogStoreSpy.clearRawRebuildIncomplete).not.toHaveBeenCalled(); + expect(opLogStoreSpy.completeRawRebuild).not.toHaveBeenCalled(); }); it('should keep the first attempt backup on crash resume instead of re-capturing', async () => { @@ -2972,7 +3108,10 @@ describe('OperationLogSyncService', () => { failedFileCount: 0, latestServerSeq: 1, }); - opLogStoreSpy.loadImportBackup.and.resolveTo({ state: {}, savedAt: 12345 }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef12345, + }); const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), @@ -3013,7 +3152,10 @@ describe('OperationLogSyncService', () => { failedFileCount: 0, latestServerSeq: 4, }); - opLogStoreSpy.loadImportBackup.and.resolveTo({ state: {}, savedAt: 12345 }); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...backupRef12345, + }); opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo({ incomplete: true, startedAt: 1, @@ -3061,7 +3203,7 @@ describe('OperationLogSyncService', () => { local: 3, }); expect(restoreOrder).toEqual(['merge-clock', 'apply']); - expect(opLogStoreSpy.clearRawRebuildIncomplete).toHaveBeenCalled(); + expect(opLogStoreSpy.completeRawRebuild).toHaveBeenCalledWith(backupRef12345); expect(operationLogEffectsSpy.processDeferredActions).toHaveBeenCalledWith({ callerHoldsOperationLogLock: true, }); @@ -3087,7 +3229,7 @@ describe('OperationLogSyncService', () => { ).toBeRejectedWithError(/local change arrived/); expect(setLastServerSeq).not.toHaveBeenCalledWith(50); - expect(opLogStoreSpy.clearRawRebuildIncomplete).not.toHaveBeenCalled(); + expect(opLogStoreSpy.completeRawRebuild).not.toHaveBeenCalled(); }); it('should retry the rebuild in-call on a capture race and converge without re-downloading', async () => { @@ -3102,6 +3244,10 @@ describe('OperationLogSyncService', () => { // Attempt 1 trips the completion assert (e.g. a tracking tick landed in // an unprotected gap); the in-call retry runs clean. writeFlushServiceSpy.hasPendingWrites.and.returnValues(true, false, false); + opLogStoreSpy.loadImportBackup.and.resolveTo({ + state: {}, + ...defaultBackupRef, + }); const mockProvider = { supportsOperationSync: true, setLastServerSeq: jasmine.createSpy('setLastServerSeq').and.resolveTo(), @@ -3116,7 +3262,7 @@ describe('OperationLogSyncService', () => { // attempt's pre-replace backup is kept, never re-captured over. expect(backupServiceSpy.captureImportBackup).toHaveBeenCalledTimes(1); expect(opLogStoreSpy.loadImportBackup).toHaveBeenCalled(); - expect(opLogStoreSpy.clearRawRebuildIncomplete).toHaveBeenCalled(); + expect(opLogStoreSpy.completeRawRebuild).toHaveBeenCalledWith(defaultBackupRef); }); it('should warn only once per session when the remote history requires a newer app', async () => { diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 28738c9ceb..472901bab3 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -8,7 +8,10 @@ import { mergeVectorClocks, } from '../../core/util/vector-clock'; import { FILE_BASED_SYNC_CONSTANTS } from '../sync-providers/file-based/file-based-sync.types'; -import { OperationLogStoreService } from '../persistence/operation-log-store.service'; +import { + ImportBackupRef, + OperationLogStoreService, +} from '../persistence/operation-log-store.service'; import { BackupService } from '../backup/backup.service'; import { OpLog } from '../../core/log'; import { @@ -1432,9 +1435,9 @@ export class OperationLogSyncService { (snapshotState?.['archiveOld'] as typeof MODEL_CONFIGS.archiveOld.defaultData) ?? MODEL_CONFIGS.archiveOld.defaultData!; - let capturedBackupSavedAt: number | undefined; + let capturedBackupRef: ImportBackupRef | undefined; let replacementCommitted = false; - let backupSavedAt: number | undefined; + let backupRef: ImportBackupRef | undefined; let preservedLocalOps: Operation[] = []; // A capture racing the rebuild aborts the attempt (CaptureRacedRebuildError // from the asserts below). Retry in-call from the already-downloaded @@ -1454,13 +1457,24 @@ export class OperationLogSyncService { // so flushing while holding it deadlocks) and re-checks inside the lock — // actions dispatched while the network request and preflight were in // flight are durably written and included in the reversible safety backup. - backupSavedAt = await this.writeFlushService.flushThenRunExclusive(async () => { - let savedAt: number | undefined; + backupRef = await this.writeFlushService.flushThenRunExclusive(async () => { + let currentBackupRef: ImportBackupRef | undefined; if (isCrashResume) { // Keep the first attempt's pre-replace backup (see option JSDoc). - savedAt = (await this.opLogStore.loadImportBackup())?.savedAt; - capturedBackupSavedAt = savedAt; const marker = await this.opLogStore.loadRawRebuildIncomplete(); + const storedBackup = await this.opLogStore.loadImportBackup(); + const expectedBackupRef = marker?.backupRef ?? capturedBackupRef; + currentBackupRef = expectedBackupRef + ? storedBackup?.backupId === expectedBackupRef.backupId + ? expectedBackupRef + : undefined + : storedBackup + ? { + backupId: storedBackup.backupId, + savedAt: storedBackup.savedAt, + } + : undefined; + capturedBackupRef ??= currentBackupRef; const liveLocalOps = (await this.opLogStore.getUnsynced()).map( (entry) => entry.op, ); @@ -1470,8 +1484,8 @@ export class OperationLogSyncService { ); } else { try { - savedAt = await this.backupService.captureImportBackup(); - capturedBackupSavedAt = savedAt; + currentBackupRef = await this.backupService.captureImportBackup(); + capturedBackupRef = currentBackupRef; } catch (e) { OpLog.warn( 'OperationLogSyncService: Pre-replace safety backup failed; aborting force download.', @@ -1499,6 +1513,7 @@ export class OperationLogSyncService { archiveYoung, archiveOld, preservedLocalOps, + backupRef: currentBackupRef, }); replacementCommitted = true; @@ -1550,12 +1565,12 @@ export class OperationLogSyncService { await syncProvider.setLastServerSeq(result.latestServerSeq); } - await this._completeRawRebuild(); + const hasDurableRecovery = await this._completeRawRebuild(currentBackupRef); OpLog.normal( 'OperationLogSyncService: Force download snapshot hydration complete.', ); - return savedAt; + return hasDurableRecovery ? currentBackupRef : undefined; } // Reset live state to defaults, then replay the COMPLETE server history on @@ -1602,12 +1617,12 @@ export class OperationLogSyncService { await syncProvider.setLastServerSeq(result.latestServerSeq); } - await this._completeRawRebuild(); + const hasDurableRecovery = await this._completeRawRebuild(currentBackupRef); OpLog.normal( `OperationLogSyncService: Force download complete. Rebuilt from ${migratedRemoteOps.length} ops.`, ); - return savedAt; + return hasDurableRecovery ? currentBackupRef : undefined; }); break; } catch (e) { @@ -1627,16 +1642,16 @@ export class OperationLogSyncService { } // Final failure only — showing this per aborted attempt would churn the // single snack slot. - if (replacementCommitted && capturedBackupSavedAt !== undefined) { - this._showRestorePreviousDataSnack(capturedBackupSavedAt); + if (replacementCommitted && capturedBackupRef) { + await this._offerStrandedRebuildBackup(); } throw e; } } // On a crash resume without a surviving backup there is nothing to offer. - if (backupSavedAt !== undefined) { - this._showRestorePreviousDataSnack(backupSavedAt); + if (backupRef) { + this._showRestorePreviousDataSnack(backupRef, true); } } @@ -1746,9 +1761,9 @@ export class OperationLogSyncService { } } - private async _completeRawRebuild(): Promise { + private async _completeRawRebuild(backupRef?: ImportBackupRef): Promise { this._assertNoCaptureRacedWithRebuild(); - await this.opLogStore.clearRawRebuildIncomplete(); + return this.opLogStore.completeRawRebuild(backupRef); } /** @@ -1855,14 +1870,20 @@ export class OperationLogSyncService { * control) and no auto-dismiss timer (duration: 0) so the undo isn't lost to a * timeout. (#8107) */ - private _showRestorePreviousDataSnack(backupSavedAt: number): void { + private _showRestorePreviousDataSnack( + backupRef: ImportBackupRef, + isCompletedRecovery: boolean, + ): void { this.snackService.open({ type: 'WARNING', msg: T.F.SYNC.S.LOCAL_DATA_REPLACE_UNDO, actionStr: T.G.UNDO, actionFn: async (): Promise => { try { - const didRestore = await this.backupService.restoreImportBackup(backupSavedAt); + const didRestore = await this.backupService.restoreImportBackup(backupRef); + if (didRestore) { + await this.opLogStore.clearRawRebuildRecovery(backupRef.backupId); + } this.snackService.open({ type: didRestore ? 'SUCCESS' : 'ERROR', msg: didRestore ? T.F.SYNC.S.RESTORE_SUCCESS : T.F.SYNC.S.RESTORE_ERROR, @@ -1874,17 +1895,30 @@ export class OperationLogSyncService { this.snackService.open({ type: 'ERROR', msg: T.F.SYNC.S.RESTORE_ERROR }); } }, + ...(isCompletedRecovery + ? { + dismissFn: async (): Promise => { + try { + await this.opLogStore.retireCompletedRawRebuildRecovery( + backupRef.backupId, + ); + } catch (error) { + OpLog.err( + 'OperationLogSyncService: Failed to retire dismissed rebuild recovery', + { name: (error as Error | undefined)?.name }, + ); + } + }, + } + : {}), config: { duration: 0 }, }); } /** - * Boot-time entry point for the interrupted-rebuild affordance (see - * StartupService): the user woke up on the rebuild baseline, not their data. - * A running sync resumes the rebuild by itself, but when none will run - * (offline, or sync disabled facing the "emptied" app) the pre-replace - * backup would have no visible entry point. Non-destructive — only surfaces - * the persistent restore snack, deduped against a visible recovery action. + * Boot-time entry point for raw-rebuild recovery (see StartupService). + * Handles both an interrupted rebuild and a completed rebuild whose durable + * Undo token survived a reload. */ async offerInterruptedRebuildRecovery(): Promise { await this._offerStrandedRebuildBackup(); @@ -1902,9 +1936,46 @@ export class OperationLogSyncService { if (this.snackService.hasPendingPersistentAction()) { return; } - const backup = await this.opLogStore.loadImportBackup(); - if (backup?.savedAt !== undefined) { - this._showRestorePreviousDataSnack(backup.savedAt); + const [incomplete, recovery, backup] = await Promise.all([ + this.opLogStore.loadRawRebuildIncomplete(), + this.opLogStore.loadRawRebuildRecovery(), + this.opLogStore.loadImportBackup(), + ]); + if (!incomplete && !recovery) { + return; + } + if (!backup) { + if (recovery && !incomplete) { + await this.opLogStore.clearRawRebuildRecovery(recovery.backupId); + } + return; + } + + // While incomplete, the surviving backup slot is still the authoritative + // pre-rebuild snapshot. Completed recovery is stricter: its durable token + // must match so an unrelated later import can never be offered as Undo. + if (incomplete) { + const expectedBackup = incomplete.backupRef; + if (expectedBackup && expectedBackup.backupId !== backup.backupId) { + return; + } + this._showRestorePreviousDataSnack( + expectedBackup ?? { + backupId: backup.backupId, + savedAt: backup.savedAt, + }, + false, + ); + } else if (recovery && recovery.backupId === backup.backupId) { + this._showRestorePreviousDataSnack( + { + backupId: recovery.backupId, + savedAt: recovery.backupSavedAt, + }, + true, + ); + } else if (recovery) { + await this.opLogStore.clearRawRebuildRecovery(recovery.backupId); } } From 982f2916caa99e6ac834e169b70c8ca7e22e805f Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:00:55 +0200 Subject: [PATCH 36/47] fix(sync): serialize archive mutations and persist before dispatch Remote archive side effects must be idempotent and immune to concurrent read-modify-write races, or a retried operation can duplicate or drop archive rows. - TaskArchiveService serializes every archive mutation behind a dedicated sp_task_archive web lock (separate from OPERATION_LOG, which remote archive handlers already hold non-reentrantly). - ArchiveService.moveTasksToArchiveAndFlushArchiveIfDue writes tasks with setMany over a deduplicated sorted id set, so a retry over a partially written archive converges instead of double-adding. - TaskService._moveToArchive coalesces concurrent archive calls for the same ids and persists archive data before dispatching moveToArchive, so a full-state snapshot cannot acknowledge the op while its archive write is still in flight. --- .../features/archive/archive.service.spec.ts | 112 +++++++++++++++++ src/app/features/archive/archive.service.ts | 66 +++++++--- .../archive/task-archive.service.spec.ts | 79 +++++++++++- .../features/archive/task-archive.service.ts | 114 +++++++++++++++--- src/app/features/tasks/task.service.spec.ts | 66 ++++++++++ src/app/features/tasks/task.service.ts | 60 +++++++-- src/app/op-log/core/operation-log.const.ts | 16 ++- 7 files changed, 464 insertions(+), 49 deletions(-) diff --git a/src/app/features/archive/archive.service.spec.ts b/src/app/features/archive/archive.service.spec.ts index 730425a160..3513e49035 100644 --- a/src/app/features/archive/archive.service.spec.ts +++ b/src/app/features/archive/archive.service.spec.ts @@ -83,6 +83,86 @@ describe('ArchiveService', () => { expect(savedData.task.ids).toContain('task-1'); }); + it('should serialize concurrent task archive mutations without dropping either task', async () => { + let archiveYoung = createEmptyArchive(Date.now()); + let releaseFirstSave: (() => void) | undefined; + let firstSaveStarted: (() => void) | undefined; + const firstSaveStartedPromise = new Promise((resolve) => { + firstSaveStarted = resolve; + }); + const firstSaveGate = new Promise((resolve) => { + releaseFirstSave = resolve; + }); + + mockArchiveDbAdapter.loadArchiveYoung.and.callFake(async () => archiveYoung); + mockArchiveDbAdapter.loadArchiveOld.and.callFake(async () => + createEmptyArchive(Date.now()), + ); + mockArchiveDbAdapter.saveArchiveYoung.and.callFake(async (nextArchive) => { + if (mockArchiveDbAdapter.saveArchiveYoung.calls.count() === 1) { + firstSaveStarted?.(); + await firstSaveGate; + } + archiveYoung = nextArchive; + }); + + const first = service.moveTasksToArchiveAndFlushArchiveIfDue([ + createMockTask('task-a'), + ]); + await firstSaveStartedPromise; + const second = service.moveTasksToArchiveAndFlushArchiveIfDue([ + createMockTask('task-b'), + ]); + + // The second read must wait for the first write to commit. + await Promise.resolve(); + expect(mockArchiveDbAdapter.loadArchiveYoung).toHaveBeenCalledTimes(1); + + releaseFirstSave?.(); + await Promise.all([first, second]); + + expect(archiveYoung.task.ids).toEqual(['task-a', 'task-b']); + expect(Object.keys(archiveYoung.task.entities).sort()).toEqual([ + 'task-a', + 'task-b', + ]); + }); + + it('should replace a stale archived task on retry and keep ids deduped and sorted', async () => { + const staleTask = createMockTask('task-b', { + title: 'Stale archived title', + timeSpent: 1, + }); + const otherTask = createMockTask('task-a'); + mockArchiveDbAdapter.loadArchiveYoung.and.returnValue( + Promise.resolve({ + ...createEmptyArchive(), + task: { + ids: ['task-b', 'task-a', 'task-b'], + entities: { + [otherTask.id]: otherTask, + [staleTask.id]: staleTask, + }, + }, + }), + ); + mockArchiveDbAdapter.loadArchiveOld.and.returnValue( + Promise.resolve(createEmptyArchive(Date.now() - 1000)), + ); + const retriedTask = createMockTask('task-b', { + title: 'Newest active title', + timeSpent: 42, + }); + + await service.moveTasksToArchiveAndFlushArchiveIfDue([retriedTask]); + + const savedTaskState = + mockArchiveDbAdapter.saveArchiveYoung.calls.first().args[0].task; + expect(savedTaskState.ids).toEqual(['task-a', 'task-b']); + expect(savedTaskState.entities['task-b']!.title).toBe('Newest active title'); + expect(savedTaskState.entities['task-b']!.timeSpent).toBe(42); + }); + it('should dispatch updateWholeState for time tracking', async () => { const tasks = [createMockTask('task-1')]; @@ -344,6 +424,38 @@ describe('ArchiveService', () => { }); describe('writeTasksToArchiveForRemoteSync', () => { + it('should replace a stale archived task on remote retry and keep ids deduped and sorted', async () => { + const staleTask = createMockTask('task-b', { + title: 'Stale remote title', + timeSpent: 1, + }); + const otherTask = createMockTask('task-a'); + mockArchiveDbAdapter.loadArchiveYoung.and.returnValue( + Promise.resolve({ + ...createEmptyArchive(), + task: { + ids: ['task-b', 'task-a', 'task-b'], + entities: { + [otherTask.id]: otherTask, + [staleTask.id]: staleTask, + }, + }, + }), + ); + const retriedTask = createMockTask('task-b', { + title: 'Newest remote title', + timeSpent: 84, + }); + + await service.writeTasksToArchiveForRemoteSync([retriedTask]); + + const savedTaskState = + mockArchiveDbAdapter.saveArchiveYoung.calls.mostRecent().args[0].task; + expect(savedTaskState.ids).toEqual(['task-a', 'task-b']); + expect(savedTaskState.entities['task-b']!.title).toBe('Newest remote title'); + expect(savedTaskState.entities['task-b']!.timeSpent).toBe(84); + }); + it('should ignore malformed tasks without valid ids', async () => { const invalidTask = { title: 'Invalid task', diff --git a/src/app/features/archive/archive.service.ts b/src/app/features/archive/archive.service.ts index 705a764c3b..f2e2f8a639 100644 --- a/src/app/features/archive/archive.service.ts +++ b/src/app/features/archive/archive.service.ts @@ -1,5 +1,5 @@ import { inject, Injectable } from '@angular/core'; -import { Task, TaskWithSubTasks } from '../tasks/task.model'; +import { Task, TaskArchive, TaskWithSubTasks } from '../tasks/task.model'; import { flattenTasks } from '../tasks/store/task.selectors'; import { createEmptyEntity } from '../../util/create-empty-entity'; import { taskAdapter } from '../tasks/store/task.adapter'; @@ -19,6 +19,8 @@ import { first } from 'rxjs/operators'; import { firstValueFrom } from 'rxjs'; import { selectTimeTrackingState } from '../time-tracking/store/time-tracking.selectors'; import { isValidEntityId } from '../../op-log/validation/is-valid-entity-id'; +import { LockService } from '../../op-log/sync/lock.service'; +import { LOCK_NAMES } from '../../op-log/core/operation-log.const'; /** * Maps tasks to archive format by: @@ -60,6 +62,23 @@ const mapTasksToArchiveFormat = ( }); }; +const replaceTasksInArchive = ( + archiveTasks: Task[], + taskArchiveState: TaskArchive, +): TaskArchive => { + const updatedTaskArchive = taskAdapter.setMany(archiveTasks, taskArchiveState); + const uniqueStringIds = new Set( + updatedTaskArchive.ids.filter((id): id is string => typeof id === 'string'), + ); + + return { + ...updatedTaskArchive, + // Keep retry output deterministic even when an older archive already + // contains duplicate or unsorted IDs. + ids: [...uniqueStringIds].sort(), + }; +}; + export const sanitizeTasksForArchiving = ( tasksIn: TaskWithSubTasks[], logPrefix: string, @@ -153,10 +172,29 @@ const DEFAULT_ARCHIVE: ArchiveModel = { export class ArchiveService { private readonly _archiveDbAdapter = inject(ArchiveDbAdapter); private readonly _store = inject(Store); + private readonly _lockService = inject(LockService); + + /** + * Serializes task archive read-modify-write cycles. Separate task actions can + * arrive before either IndexedDB write finishes; without one shared queue, + * both calls can read the same archive and the last save silently drops the + * other task. A failed mutation does not poison later retries. + */ + private _runTaskArchiveMutation(mutation: () => Promise): Promise { + return this._lockService.request(LOCK_NAMES.TASK_ARCHIVE, mutation); + } // NOTE: we choose this method as trigger to check for flushing to archive, since // it is usually triggered every work-day once - async moveTasksToArchiveAndFlushArchiveIfDue(tasks: TaskWithSubTasks[]): Promise { + moveTasksToArchiveAndFlushArchiveIfDue(tasks: TaskWithSubTasks[]): Promise { + return this._runTaskArchiveMutation(() => + this._moveTasksToArchiveAndFlushArchiveIfDue(tasks), + ); + } + + private async _moveTasksToArchiveAndFlushArchiveIfDue( + tasks: TaskWithSubTasks[], + ): Promise { const now = Date.now(); const sanitizedTasks = sanitizeTasksForArchiving(tasks, 'moveToArchive'); const flatTasks = flattenTasks(sanitizedTasks); @@ -178,12 +216,7 @@ export class ArchiveService { const taskArchiveState = archiveYoung.task || createEmptyEntity(); const archiveTasks = mapTasksToArchiveFormat(flatTasks, now, 'moveToArchive'); - const newTaskArchiveUnsorted = taskAdapter.addMany(archiveTasks, taskArchiveState); - // Sort ids for deterministic ordering across clients (UUIDv7 is lexicographically sortable) - const newTaskArchive = { - ...newTaskArchiveUnsorted, - ids: [...newTaskArchiveUnsorted.ids].sort(), - }; + const newTaskArchive = replaceTasksInArchive(archiveTasks, taskArchiveState); // ------------------------------------------------ // Result A: @@ -317,7 +350,15 @@ export class ArchiveService { * only on the originating client and synced via the flushYoungToOld action. * This ensures flushes happen exactly once, not on every receiving client. */ - async writeTasksToArchiveForRemoteSync(tasks: TaskWithSubTasks[]): Promise { + writeTasksToArchiveForRemoteSync(tasks: TaskWithSubTasks[]): Promise { + return this._runTaskArchiveMutation(() => + this._writeTasksToArchiveForRemoteSync(tasks), + ); + } + + private async _writeTasksToArchiveForRemoteSync( + tasks: TaskWithSubTasks[], + ): Promise { const now = Date.now(); const sanitizedTasks = sanitizeTasksForArchiving(tasks, 'Remote sync'); const flatTasks = flattenTasks(sanitizedTasks); @@ -339,12 +380,7 @@ export class ArchiveService { const taskArchiveState = archiveYoung.task || createEmptyEntity(); const archiveTasks = mapTasksToArchiveFormat(flatTasks, now, 'Remote sync'); - const newTaskArchiveUnsorted = taskAdapter.addMany(archiveTasks, taskArchiveState); - // Sort ids for deterministic ordering across clients (UUIDv7 is lexicographically sortable) - const newTaskArchive = { - ...newTaskArchiveUnsorted, - ids: [...newTaskArchiveUnsorted.ids].sort(), - }; + const newTaskArchive = replaceTasksInArchive(archiveTasks, taskArchiveState); // Also move historical time tracking data to archiveYoung // This ensures the remote client's archive matches the originating client diff --git a/src/app/features/archive/task-archive.service.spec.ts b/src/app/features/archive/task-archive.service.spec.ts index 38f140dfa6..9eabd5c4c2 100644 --- a/src/app/features/archive/task-archive.service.spec.ts +++ b/src/app/features/archive/task-archive.service.spec.ts @@ -6,11 +6,39 @@ import { Task, TaskArchive, TaskState } from '../tasks/task.model'; import { ArchiveModel } from './archive.model'; import { Update } from '@ngrx/entity'; import { RoundTimeOption } from '../project/project.model'; +import { ArchiveService } from './archive.service'; +import { of } from 'rxjs'; +import { LockService } from '../../op-log/sync/lock.service'; + +class TestLockService { + private readonly _tails = new Map>(); + + async request(lockName: string, callback: () => Promise): Promise { + const previous = this._tails.get(lockName) ?? Promise.resolve(); + let release: (() => void) | undefined; + const current = new Promise((resolve) => { + release = resolve; + }); + const tail = previous.then(() => current); + this._tails.set(lockName, tail); + + await previous; + try { + return await callback(); + } finally { + release?.(); + if (this._tails.get(lockName) === tail) { + this._tails.delete(lockName); + } + } + } +} import { TaskSharedActions } from '../../root-store/meta/task-shared.actions'; describe('TaskArchiveService', () => { let service: TaskArchiveService; + let archiveService: ArchiveService; let storeMock: jasmine.SpyObj; let archiveDbAdapterMock: jasmine.SpyObj; @@ -53,7 +81,8 @@ describe('TaskArchiveService', () => { }); beforeEach(() => { - storeMock = jasmine.createSpyObj('Store', ['dispatch']); + storeMock = jasmine.createSpyObj('Store', ['dispatch', 'select']); + storeMock.select.and.returnValue(of({ project: {}, tag: {} })); archiveDbAdapterMock = jasmine.createSpyObj('ArchiveDbAdapter', [ 'loadArchiveYoung', @@ -75,10 +104,58 @@ describe('TaskArchiveService', () => { TaskArchiveService, { provide: ArchiveDbAdapter, useValue: archiveDbAdapterMock }, { provide: Store, useValue: storeMock }, + { provide: LockService, useClass: TestLockService }, ], }); service = TestBed.inject(TaskArchiveService); + archiveService = TestBed.inject(ArchiveService); + }); + + describe('archive mutation locking', () => { + it('should serialize a task update with an ArchiveService mutation', async () => { + const existingTask = createMockTask('existing', { title: 'Original' }); + const archivedTask = { + ...createMockTask('new-task', { + isDone: true, + doneOn: Date.now(), + }), + subTasks: [], + }; + let archiveYoung = createMockArchiveModel([existingTask]); + let releaseFirstSave: (() => void) | undefined; + let firstSaveStarted: (() => void) | undefined; + const firstSaveStartedPromise = new Promise((resolve) => { + firstSaveStarted = resolve; + }); + const firstSaveGate = new Promise((resolve) => { + releaseFirstSave = resolve; + }); + + archiveDbAdapterMock.loadArchiveYoung.and.callFake(async () => archiveYoung); + archiveDbAdapterMock.saveArchiveYoung.and.callFake(async (nextArchive) => { + if (archiveDbAdapterMock.saveArchiveYoung.calls.count() === 1) { + firstSaveStarted?.(); + await firstSaveGate; + } + archiveYoung = nextArchive; + }); + + const updatePromise = service.updateTask('existing', { title: 'Updated' }); + await firstSaveStartedPromise; + const archivePromise = archiveService.moveTasksToArchiveAndFlushArchiveIfDue([ + archivedTask, + ]); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(archiveDbAdapterMock.loadArchiveYoung).toHaveBeenCalledTimes(1); + + releaseFirstSave?.(); + await Promise.all([updatePromise, archivePromise]); + + expect(archiveYoung.task.entities['existing']!.title).toBe('Updated'); + expect(archiveYoung.task.entities['new-task']).toBeDefined(); + }); }); describe('loadYoung', () => { diff --git a/src/app/features/archive/task-archive.service.ts b/src/app/features/archive/task-archive.service.ts index 4eb53b4451..1ae352eca3 100644 --- a/src/app/features/archive/task-archive.service.ts +++ b/src/app/features/archive/task-archive.service.ts @@ -17,6 +17,8 @@ import { TAG_FEATURE_NAME, tagAdapter } from '../tag/store/tag.reducer'; import { WORK_CONTEXT_FEATURE_NAME } from '../work-context/store/work-context.selectors'; import { plannerFeatureKey } from '../planner/store/planner.reducer'; import { TODAY_TAG } from '../tag/tag.const'; +import { LockService } from '../../op-log/sync/lock.service'; +import { LOCK_NAMES } from '../../op-log/core/operation-log.const'; // Normalize timeSpentOnDay at the data boundary so all consumers can trust the // invariant: timeSpentOnDay is always a valid object, never undefined. This mirrors @@ -74,6 +76,7 @@ type TaskArchiveAction = }) export class TaskArchiveService { private _injector = inject(Injector); + private readonly _lockService = inject(LockService); private _archiveDbAdapter?: ArchiveDbAdapter; private get archiveDbAdapter(): ArchiveDbAdapter { if (!this._archiveDbAdapter) { @@ -106,6 +109,15 @@ export class TaskArchiveService { constructor() {} + private _runTaskArchiveMutation( + mutation: () => Promise, + isIgnoreDBLock: boolean = false, + ): Promise { + return isIgnoreDBLock + ? mutation() + : this._lockService.request(LOCK_NAMES.TASK_ARCHIVE, mutation); + } + async loadYoung(): Promise { const archiveYoung = (await this.archiveDbAdapter.loadArchiveYoung()) || DEFAULT_ARCHIVE; @@ -230,10 +242,17 @@ export class TaskArchiveService { return result; } - async deleteTasks( + deleteTasks( taskIdsToDelete: string[], options?: { isIgnoreDBLock?: boolean }, ): Promise { + return this._runTaskArchiveMutation( + () => this._deleteTasks(taskIdsToDelete), + options?.isIgnoreDBLock, + ); + } + + private async _deleteTasks(taskIdsToDelete: string[]): Promise { const archiveYoung = (await this.archiveDbAdapter.loadArchiveYoung()) || DEFAULT_ARCHIVE; const toDeleteInArchiveYoung = taskIdsToDelete.filter( @@ -272,7 +291,18 @@ export class TaskArchiveService { } } - async updateTask( + updateTask( + id: string, + changedFields: Partial, + options?: { isSkipDispatch?: boolean; isIgnoreDBLock?: boolean }, + ): Promise { + return this._runTaskArchiveMutation( + () => this._updateTask(id, changedFields, options), + options?.isIgnoreDBLock, + ); + } + + private async _updateTask( id: string, changedFields: Partial, options?: { isSkipDispatch?: boolean; isIgnoreDBLock?: boolean }, @@ -311,7 +341,17 @@ export class TaskArchiveService { throw new Error('Archive task to update not found'); } - async updateTasks( + updateTasks( + updates: Update[], + options?: { isSkipDispatch?: boolean; isIgnoreDBLock?: boolean }, + ): Promise { + return this._runTaskArchiveMutation( + () => this._updateTasks(updates, options), + options?.isIgnoreDBLock, + ); + } + + private async _updateTasks( updates: Update[], options?: { isSkipDispatch?: boolean; isIgnoreDBLock?: boolean }, ): Promise { @@ -364,9 +404,18 @@ export class TaskArchiveService { } // ----------------------------------------- - async removeAllArchiveTasksForProject( + removeAllArchiveTasksForProject( projectIdToDelete: string, options?: { isIgnoreDBLock?: boolean }, + ): Promise { + return this._runTaskArchiveMutation( + () => this._removeAllArchiveTasksForProject(projectIdToDelete), + options?.isIgnoreDBLock, + ); + } + + private async _removeAllArchiveTasksForProject( + projectIdToDelete: string, ): Promise { const taskArchiveState: TaskArchive = await this.load(); const archiveTaskIdsToDelete = !!taskArchiveState @@ -378,13 +427,20 @@ export class TaskArchiveService { return t.projectId === projectIdToDelete; }) : []; - await this.deleteTasks(archiveTaskIdsToDelete, options); + await this._deleteTasks(archiveTaskIdsToDelete); } - async removeTagsFromAllTasks( + removeTagsFromAllTasks( tagIdsToRemove: string[], options?: { isIgnoreDBLock?: boolean }, ): Promise { + return this._runTaskArchiveMutation( + () => this._removeTagsFromAllTasks(tagIdsToRemove), + options?.isIgnoreDBLock, + ); + } + + private async _removeTagsFromAllTasks(tagIdsToRemove: string[]): Promise { const taskArchiveState: TaskArchive = await this.load(); await this._execActionBoth( TaskSharedActions.removeTagsForAllTasks({ tagIdsToRemove }), @@ -406,16 +462,23 @@ export class TaskArchiveService { } }); // TODO check to maybe update to today tag instead - await this.deleteTasks( - [...archiveMainTaskIdsToDelete, ...archiveSubTaskIdsToDelete], - options, - ); + await this._deleteTasks([ + ...archiveMainTaskIdsToDelete, + ...archiveSubTaskIdsToDelete, + ]); } - async removeRepeatCfgFromArchiveTasks( + removeRepeatCfgFromArchiveTasks( repeatConfigId: string, options?: { isIgnoreDBLock?: boolean }, ): Promise { + return this._runTaskArchiveMutation( + () => this._removeRepeatCfgFromArchiveTasks(repeatConfigId), + options?.isIgnoreDBLock, + ); + } + + private async _removeRepeatCfgFromArchiveTasks(repeatConfigId: string): Promise { const taskArchive = await this.load(); const newState = { ...taskArchive }; @@ -435,16 +498,24 @@ export class TaskArchiveService { }, }; }); - await this.updateTasks(updates, { + await this._updateTasks(updates, { isSkipDispatch: true, - isIgnoreDBLock: options?.isIgnoreDBLock, }); } } - async unlinkIssueProviderFromArchiveTasks( + unlinkIssueProviderFromArchiveTasks( issueProviderId: string, options?: { isIgnoreDBLock?: boolean }, + ): Promise { + return this._runTaskArchiveMutation( + () => this._unlinkIssueProviderFromArchiveTasks(issueProviderId), + options?.isIgnoreDBLock, + ); + } + + private async _unlinkIssueProviderFromArchiveTasks( + issueProviderId: string, ): Promise { const taskArchive = await this.load(); @@ -466,14 +537,23 @@ export class TaskArchiveService { issuePoints: undefined, }, })); - await this.updateTasks(updates, { + await this._updateTasks(updates, { isSkipDispatch: true, - isIgnoreDBLock: options?.isIgnoreDBLock, }); } } - async roundTimeSpent({ + roundTimeSpent(params: { + day: string; + taskIds: string[]; + roundTo: RoundTimeOption; + isRoundUp: boolean; + projectId?: string | null; + }): Promise { + return this._runTaskArchiveMutation(() => this._roundTimeSpent(params)); + } + + private async _roundTimeSpent({ day, taskIds, roundTo, diff --git a/src/app/features/tasks/task.service.spec.ts b/src/app/features/tasks/task.service.spec.ts index ea9d6c4c27..e0193caad9 100644 --- a/src/app/features/tasks/task.service.spec.ts +++ b/src/app/features/tasks/task.service.spec.ts @@ -457,6 +457,72 @@ describe('TaskService', () => { }); describe('moveToArchive', () => { + it('should persist parent tasks before dispatching the archive action', async () => { + const task = createMockTaskWithSubTasks(createMockTask('task-1')); + const callOrder: string[] = []; + archiveService.moveTasksToArchiveAndFlushArchiveIfDue.and.callFake(async () => { + callOrder.push('persist-archive'); + }); + store.dispatch = jasmine + .createSpy('dispatch') + .and.callFake(() => callOrder.push('dispatch')); + + await service.moveToArchive(task); + + expect(callOrder).toEqual(['persist-archive', 'dispatch']); + }); + + it('should coalesce concurrent archive requests for the same task', async () => { + const task = createMockTaskWithSubTasks(createMockTask('task-1')); + let finishPersistence: (() => void) | undefined; + archiveService.moveTasksToArchiveAndFlushArchiveIfDue.and.returnValue( + new Promise((resolve) => { + finishPersistence = resolve; + }), + ); + + const first = service.moveToArchive(task); + const duplicate = service.moveToArchive(task); + let duplicateResolved = false; + void duplicate.then(() => { + duplicateResolved = true; + }); + + expect(archiveService.moveTasksToArchiveAndFlushArchiveIfDue).toHaveBeenCalledTimes( + 1, + ); + expect(store.dispatch).not.toHaveBeenCalledWith( + jasmine.objectContaining({ type: TaskSharedActions.moveToArchive.type }), + ); + await Promise.resolve(); + expect(duplicateResolved).toBeFalse(); + + finishPersistence?.(); + await Promise.all([first, duplicate]); + + expect(duplicateResolved).toBeTrue(); + expect(store.dispatch).toHaveBeenCalledTimes(1); + }); + + it('should release the in-flight archive guard after persistence fails', async () => { + const task = createMockTaskWithSubTasks(createMockTask('task-1')); + archiveService.moveTasksToArchiveAndFlushArchiveIfDue.and.rejectWith( + new Error('archive write failed'), + ); + + await expectAsync(service.moveToArchive(task)).toBeRejectedWithError( + 'archive write failed', + ); + + archiveService.moveTasksToArchiveAndFlushArchiveIfDue.and.resolveTo(); + await service.moveToArchive(task); + + expect(archiveService.moveTasksToArchiveAndFlushArchiveIfDue).toHaveBeenCalledTimes( + 2, + ); + expect(store.dispatch).toHaveBeenCalledTimes(1); + }); + it('should dispatch moveToArchive and call archive service for parent tasks', async () => { const task = createMockTaskWithSubTasks( createMockTask('task-1', { projectId: 'test-project' }), diff --git a/src/app/features/tasks/task.service.ts b/src/app/features/tasks/task.service.ts index 4e381c5ceb..ffeb8dd23c 100644 --- a/src/app/features/tasks/task.service.ts +++ b/src/app/features/tasks/task.service.ts @@ -123,6 +123,7 @@ export class TaskService { private readonly _taskFocusService = inject(TaskFocusService); private readonly _deletedTaskIssueSidecar = inject(DeletedTaskIssueSidecarService); private readonly _timeBlockDeleteSidecar = inject(TimeBlockDeleteSidecarService); + private readonly _archiveTaskPromisesById = new Map>(); currentTaskId$: Observable = this._store.pipe( select(selectCurrentTaskId), @@ -947,15 +948,60 @@ export class TaskService { } } - if (parentTasks.length) { - TaskLog.log('[TaskService] Dispatching moveToArchive action for parent tasks'); + const parentTasksToArchive: TaskWithSubTasks[] = []; + const reservedTaskIds = new Set(); + const existingArchivePromises = new Set>(); + for (const task of parentTasks) { + if (task.id) { + const existingArchivePromise = this._archiveTaskPromisesById.get(task.id); + if (existingArchivePromise) { + TaskLog.log('[TaskService] Archive already in progress', { id: task.id }); + existingArchivePromises.add(existingArchivePromise); + continue; + } + if (reservedTaskIds.has(task.id)) { + continue; + } + reservedTaskIds.add(task.id); + } + parentTasksToArchive.push(task); + } + + if (parentTasksToArchive.length) { // Only move parent tasks to archive, never subtasks // Note: Full task payload required for sync - see docs/archive-operation-redesign.md - this._store.dispatch(TaskSharedActions.moveToArchive({ tasks: parentTasks })); - // Only archive parent tasks to prevent orphaned subtasks - TaskLog.log('[TaskService] Calling archive service to persist tasks'); - await this._archiveService.moveTasksToArchiveAndFlushArchiveIfDue(parentTasks); - TaskLog.log('[TaskService] Archive operation completed successfully'); + // Persist first: dispatch removes the tasks from NgRx and makes the captured + // operation eligible for a full-state snapshot. If archive persistence were + // still in flight, that snapshot could acknowledge the operation while + // omitting its archived task data. + const archivePromise = (async (): Promise => { + TaskLog.log('[TaskService] Calling archive service to persist tasks'); + await this._archiveService.moveTasksToArchiveAndFlushArchiveIfDue( + parentTasksToArchive, + ); + TaskLog.log('[TaskService] Dispatching moveToArchive action for parent tasks'); + this._store.dispatch( + TaskSharedActions.moveToArchive({ tasks: parentTasksToArchive }), + ); + TaskLog.log('[TaskService] Archive operation completed successfully'); + })(); + for (const taskId of reservedTaskIds) { + this._archiveTaskPromisesById.set(taskId, archivePromise); + } + + try { + await Promise.all([...existingArchivePromises, archivePromise]); + } finally { + for (const taskId of reservedTaskIds) { + if (this._archiveTaskPromisesById.get(taskId) === archivePromise) { + this._archiveTaskPromisesById.delete(taskId); + } + } + } + } else if (existingArchivePromises.size > 0) { + // A duplicate caller observes the same success/failure and does not return + // before the durable archive write plus NgRx removal have completed. + await Promise.all(existingArchivePromises); } else { TaskLog.log('[TaskService] No parent tasks to archive'); } diff --git a/src/app/op-log/core/operation-log.const.ts b/src/app/op-log/core/operation-log.const.ts index 5355851a0a..af2eb58c69 100644 --- a/src/app/op-log/core/operation-log.const.ts +++ b/src/app/op-log/core/operation-log.const.ts @@ -49,6 +49,13 @@ import { InjectionToken } from '@angular/core'; * IMPORTANT: These names are also used in tests - update tests if renaming. */ export const LOCK_NAMES = { + /** + * Serializes archive task read-modify-write cycles across tabs. This stays + * separate from OPERATION_LOG because remote archive handlers already run + * while holding that non-reentrant lock. + */ + TASK_ARCHIVE: 'sp_task_archive', + /** * Main operation log lock. Used for: * - Writing operations to IndexedDB @@ -151,15 +158,6 @@ export const MAX_REJECTED_OPS_BEFORE_WARNING = 10; */ export const MAX_BATCH_OPERATIONS_SIZE = 50; -/** - * Maximum age for pending operations before they are quarantined as failed (milliseconds). - * If an operation has been pending for longer than this (e.g., due to data corruption - * or repeated crashes), hydration still restores its reducer history and archive recovery - * remains required. - * Default: 24 hours - enough time for legitimate recovery scenarios - */ -export const PENDING_OPERATION_EXPIRY_MS = 24 * 60 * 60 * 1000; - /** * Threshold for warning about clock drift between client and server (milliseconds). * If the client's clock differs from the server by more than this amount, From e019ef0b71f2f22d5ee090bd2df7e94b220273c2 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:01:14 +0200 Subject: [PATCH 37/47] fix(supersync): widen conflict lookups to aliased and unioned entity ids Two conflict-detection blind spots let concurrent edits slip through without a conflict: - An op carrying both a scalar entityId and an entityIds array only checked the array; the scalar and array sets are now unioned for detection while getStoredEntityIds keeps the historical storage normalization (scalar in entity_id, arrays in entity_ids). - Histories written before schema v2 keep migrated task settings under GLOBAL_CONFIG:misc. Incoming GLOBAL_CONFIG:tasks writes now consult the legacy misc row as an alias (newest of the two wins, compared by serverSeq), and legacy misc ops check the tasks key per-entity. Works for encrypted payloads since only row metadata is consulted. --- .../super-sync-server/src/sync/conflict.ts | 107 +++++++++++++++--- .../super-sync-server/src/sync/sync.types.ts | 1 + .../super-sync-server/tests/conflict.spec.ts | 23 ++++ 3 files changed, 116 insertions(+), 15 deletions(-) diff --git a/packages/super-sync-server/src/sync/conflict.ts b/packages/super-sync-server/src/sync/conflict.ts index 008733516a..6924586f0f 100644 --- a/packages/super-sync-server/src/sync/conflict.ts +++ b/packages/super-sync-server/src/sync/conflict.ts @@ -1,4 +1,5 @@ import { Prisma } from '@prisma/client'; +import { CURRENT_SCHEMA_VERSION } from '@sp/shared-schema'; import { Logger } from '../logger'; import { BatchUploadCandidate, @@ -34,11 +35,7 @@ export const detectConflict = async ( // Build list of entity IDs to check for conflicts. // Operations may have either entityId (singular) or entityIds (batch operations). - const rawEntityIdsToCheck = op.entityIds?.length - ? op.entityIds - : op.entityId - ? [op.entityId] - : []; + const rawEntityIdsToCheck = getConflictEntityIds(op); // Skip if no entity IDs (can't have entity-level conflicts) if (rawEntityIdsToCheck.length === 0) { @@ -49,6 +46,18 @@ export const detectConflict = async ( return detectConflictForEntity(userId, op, rawEntityIdsToCheck[0], tx); } + // v1 GLOBAL_CONFIG:misc contains fields that became GLOBAL_CONFIG:tasks in + // v2. This compatibility path is intentionally per-key: it is rare, keeps + // the normal multi-entity SQL unchanged, and works for encrypted legacy + // payloads whose contents the server cannot inspect. + if (isLegacyMiscConfigOperation(op)) { + for (const entityId of rawEntityIdsToCheck) { + const result = await detectConflictForEntity(userId, op, entityId, tx); + if (result.hasConflict) return result; + } + return { hasConflict: false }; + } + const entityIdsToCheck = Array.from(new Set(rawEntityIdsToCheck)); return detectConflictForEntities(userId, op, entityIdsToCheck, tx); }; @@ -210,16 +219,40 @@ export const detectConflictForEntity = async ( entityType: op.entityType, OR: [{ entityId }, { entityIds: { has: entityId } }], }, - select: { clientId: true, vectorClock: true }, + select: { clientId: true, vectorClock: true, serverSeq: true }, orderBy: { serverSeq: 'desc' }, }); + // Histories written before schema v2 persist migrated task settings under + // the raw `GLOBAL_CONFIG:misc` key. Consult that key as an alias when the + // incoming write targets `tasks`; no backfill (and no payload decryption) is + // required. Pick the newer of the canonical and legacy rows. + const legacyMiscOp = + op.entityType === 'GLOBAL_CONFIG' && entityId === 'tasks' + ? await tx.operation.findFirst({ + where: { + userId, + entityType: 'GLOBAL_CONFIG', + entityId: 'misc', + schemaVersion: { lt: CURRENT_SCHEMA_VERSION }, + }, + select: { clientId: true, vectorClock: true, serverSeq: true }, + orderBy: { serverSeq: 'desc' }, + }) + : null; + + const latestExistingOp = + legacyMiscOp && + (!existingOp || (legacyMiscOp.serverSeq ?? -1) > (existingOp.serverSeq ?? -1)) + ? legacyMiscOp + : existingOp; + // No existing operation = no conflict - if (!existingOp) { + if (!latestExistingOp) { return { hasConflict: false }; } - return resolveConflictForExistingOp(op, entityId, existingOp); + return resolveConflictForExistingOp(op, entityId, latestExistingOp); }; export const isSameDuplicateOperation = ( @@ -350,14 +383,21 @@ export const toStableJsonValue = (value: unknown): unknown => { * `entity_ids` column, which uses {@link getStoredEntityIds} (multi-entity only). */ export const getConflictEntityIds = (op: Operation): string[] => { - const rawEntityIds = op.entityIds?.length - ? op.entityIds - : op.entityId - ? [op.entityId] - : []; + const rawEntityIds = [ + ...(op.entityId ? [op.entityId] : []), + ...(op.entityIds?.length ? op.entityIds : []), + ]; + if (isLegacyMiscConfigOperation(op)) { + rawEntityIds.push('tasks'); + } return Array.from(new Set(rawEntityIds)); }; +const isLegacyMiscConfigOperation = (op: Operation): boolean => + op.schemaVersion < CURRENT_SCHEMA_VERSION && + op.entityType === 'GLOBAL_CONFIG' && + op.entityId === 'misc'; + /** * The entity_ids array to persist with an op. Ops whose touched-entity set is * already covered by the scalar entity_id store an empty array, so single-entity @@ -371,7 +411,12 @@ export const getConflictEntityIds = (op: Operation): string[] => { * would be invisible to conflict lookups (#8334). */ export const getStoredEntityIds = (op: Operation): string[] => { - const ids = getConflictEntityIds(op); + // Preserve the historical storage normalization: the scalar is persisted in + // entity_id and only the declared multi-entity set belongs in entity_ids. + // Conflict detection unions both fields via getConflictEntityIds above. + const ids = Array.from( + new Set(op.entityIds?.length ? op.entityIds : op.entityId ? [op.entityId] : []), + ); if (ids.length <= 1 && ids[0] === op.entityId) { return []; } @@ -436,7 +481,8 @@ export const prefetchLatestEntityOpsForBatch = async ( o.entity_type AS "entityType", eid AS "entityId", o.client_id AS "clientId", - o.vector_clock AS "vectorClock" + o.vector_clock AS "vectorClock", + o.server_seq AS "serverSeq" FROM operations o CROSS JOIN LATERAL unnest( o.entity_ids || CASE WHEN o.entity_id IS NULL THEN '{}'::text[] ELSE ARRAY[o.entity_id] END @@ -457,6 +503,37 @@ export const prefetchLatestEntityOpsForBatch = async ( } } + if ( + entityPairs.some( + ({ entityType, entityId }) => + entityType === 'GLOBAL_CONFIG' && entityId === 'tasks', + ) + ) { + const legacyMiscOp = await tx.operation.findFirst({ + where: { + userId, + entityType: 'GLOBAL_CONFIG', + entityId: 'misc', + schemaVersion: { lt: CURRENT_SCHEMA_VERSION }, + }, + select: { clientId: true, vectorClock: true, serverSeq: true }, + orderBy: { serverSeq: 'desc' }, + }); + const tasksKey = getEntityConflictKey('GLOBAL_CONFIG', 'tasks'); + const currentTasksOp = latestByEntity.get(tasksKey); + if ( + legacyMiscOp && + (!currentTasksOp || + (legacyMiscOp.serverSeq ?? -1) > (currentTasksOp.serverSeq ?? -1)) + ) { + latestByEntity.set(tasksKey, { + entityType: 'GLOBAL_CONFIG', + entityId: 'tasks', + ...legacyMiscOp, + }); + } + } + return latestByEntity; }; diff --git a/packages/super-sync-server/src/sync/sync.types.ts b/packages/super-sync-server/src/sync/sync.types.ts index 50950b75b6..b47b1c0a3c 100644 --- a/packages/super-sync-server/src/sync/sync.types.ts +++ b/packages/super-sync-server/src/sync/sync.types.ts @@ -222,6 +222,7 @@ export interface LatestEntityOperationRow { entityId: string; clientId: string; vectorClock: unknown; + serverSeq?: number; } export interface LatestBatchEntityOperationRow extends LatestEntityOperationRow { diff --git a/packages/super-sync-server/tests/conflict.spec.ts b/packages/super-sync-server/tests/conflict.spec.ts index d0824ddbc0..dca33fff32 100644 --- a/packages/super-sync-server/tests/conflict.spec.ts +++ b/packages/super-sync-server/tests/conflict.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { + getConflictEntityIds, isSameDuplicateOperation, isSameDuplicateTimestamp, pruneVectorClockForStorage, @@ -48,6 +49,28 @@ const duplicateCandidate = ( }); describe('conflict helpers', () => { + it('includes a divergent scalar entityId in the incoming conflict set', () => { + expect( + getConflictEntityIds(op({ entityId: 'task-scalar', entityIds: ['task-array'] })), + ).toEqual(['task-scalar', 'task-array']); + }); + + it.each([false, true])( + 'aliases legacy misc config writes to tasks when encrypted=%s', + (isPayloadEncrypted) => { + expect( + getConflictEntityIds( + op({ + entityType: 'GLOBAL_CONFIG', + entityId: 'misc', + schemaVersion: 1, + isPayloadEncrypted, + }), + ), + ).toEqual(['misc', 'tasks']); + }, + ); + it('accepts matching duplicate operations regardless of JSON key order', () => { const incoming = op({ payload: { title: 'A', nested: { b: 2, a: 1 } }, From 12145ed6b2df6595c88ab71728b26af1677432b3 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:04:23 +0200 Subject: [PATCH 38/47] fix(supersync): isolate duplicate upload ids and make clean-slate idempotent - A request repeating one operation id no longer double-charges quota or wedges the batch: the first occurrence reserves the id and later siblings are terminally rejected (DUPLICATE_OPERATION for an exact retry, INVALID_OP_ID otherwise), in both batch and serial paths. - Clean-slate snapshot uploads become durably idempotent: the request cache is process-local and expiring, so the client-supplied opId is now required (contract superRefine) and checked against the stored operation inside the per-user lock before any data is deleted. An exact retry returns the original serverSeq; a colliding opId is rejected without touching existing data. - Audit logging validates id/entityType/opType/clientId against the known-safe charsets before embedding them in log lines, and the six duplicated reject-audit blocks collapse into one rejectedUploadResult helper; the ops handler logs rejected error codes instead of whole operation objects. --- .../src/supersync-http-contract.ts | 36 +- .../tests/supersync-http-contract.spec.ts | 13 + .../sync/services/operation-upload.service.ts | 312 ++++++++++-------- .../src/sync/sync.routes.ops-handler.ts | 4 +- .../src/sync/sync.routes.snapshot-handler.ts | 59 +++- .../src/sync/sync.service.ts | 25 +- .../tests/sync-compressed-body.routes.spec.ts | 96 +++++- .../tests/sync.service.spec.ts | 270 +++++++++++++++ 8 files changed, 661 insertions(+), 154 deletions(-) diff --git a/packages/shared-schema/src/supersync-http-contract.ts b/packages/shared-schema/src/supersync-http-contract.ts index 5b7e393283..c9a0099786 100644 --- a/packages/shared-schema/src/supersync-http-contract.ts +++ b/packages/shared-schema/src/supersync-http-contract.ts @@ -112,19 +112,29 @@ export const SuperSyncDownloadOpsQuerySchema = z.object({ excludeClient: SuperSyncClientIdSchema.optional(), }); -export const SuperSyncUploadSnapshotRequestSchema = z.object({ - state: z.unknown(), - clientId: SuperSyncClientIdSchema, - reason: z.enum(SUPER_SYNC_SNAPSHOT_REASONS), - vectorClock: SuperSyncVectorClockSchema, - schemaVersion: z.number().int().min(1).max(100).optional(), - isPayloadEncrypted: z.boolean().optional(), - syncImportReason: z.enum(SUPER_SYNC_IMPORT_REASONS).optional(), - opId: z.string().uuid().optional(), - isCleanSlate: z.boolean().optional(), - snapshotOpType: z.enum(SUPER_SYNC_SNAPSHOT_OP_TYPES).optional(), - requestId: SuperSyncRequestIdSchema.optional(), -}); +export const SuperSyncUploadSnapshotRequestSchema = z + .object({ + state: z.unknown(), + clientId: SuperSyncClientIdSchema, + reason: z.enum(SUPER_SYNC_SNAPSHOT_REASONS), + vectorClock: SuperSyncVectorClockSchema, + schemaVersion: z.number().int().min(1).max(100).optional(), + isPayloadEncrypted: z.boolean().optional(), + syncImportReason: z.enum(SUPER_SYNC_IMPORT_REASONS).optional(), + opId: z.string().uuid().optional(), + isCleanSlate: z.boolean().optional(), + snapshotOpType: z.enum(SUPER_SYNC_SNAPSHOT_OP_TYPES).optional(), + requestId: SuperSyncRequestIdSchema.optional(), + }) + .superRefine((request, context) => { + if (request.isCleanSlate && !request.opId) { + context.addIssue({ + code: 'custom', + path: ['opId'], + message: 'opId is required for clean-slate snapshot idempotency', + }); + } + }); export const SuperSyncOperationResponseSchema = SuperSyncOperationSchema.passthrough(); diff --git a/packages/shared-schema/tests/supersync-http-contract.spec.ts b/packages/shared-schema/tests/supersync-http-contract.spec.ts index c8ff1a9582..34048852f8 100644 --- a/packages/shared-schema/tests/supersync-http-contract.spec.ts +++ b/packages/shared-schema/tests/supersync-http-contract.spec.ts @@ -188,6 +188,19 @@ describe('SuperSync HTTP contract schemas', () => { expect(parsed.requestId).toBe('snapshot-v1-request'); }); + it('requires an operation ID for destructive clean-slate snapshots', () => { + expect(() => + SuperSyncUploadSnapshotRequestSchema.parse({ + state: {}, + clientId: 'client_1', + reason: 'recovery', + vectorClock: { client_1: 1 }, + schemaVersion: 1, + isCleanSlate: true, + }), + ).toThrow(); + }); + it('rejects requestIds containing characters outside the safe-log charset', () => { // Control character — would be unsafe to embed in server log lines. expect(() => diff --git a/packages/super-sync-server/src/sync/services/operation-upload.service.ts b/packages/super-sync-server/src/sync/services/operation-upload.service.ts index 5b7750ae4c..6e366ae0f3 100644 --- a/packages/super-sync-server/src/sync/services/operation-upload.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-upload.service.ts @@ -1,6 +1,10 @@ import { Prisma } from '@prisma/client'; import { Logger } from '../../logger'; -import { computeOpStorageBytes } from '../sync.const'; +import { + CLIENT_ID_REGEX, + computeOpStorageBytes, + MAX_CLIENT_ID_LENGTH, +} from '../sync.const'; import { AcceptedBatchOperation, BatchUploadCandidate, @@ -11,6 +15,7 @@ import { isFullStateOpType, limitVectorClockSize, Operation, + OP_TYPES, ProcessOperationResult, SyncConfig, SYNC_ERROR_CODES, @@ -29,13 +34,58 @@ import { pruneVectorClockForStorage, resolveConflictForExistingOp, } from '../conflict'; -import { ValidationService, type ValidationResult } from './validation.service'; +import { + ALLOWED_ENTITY_TYPES, + ValidationService, + type ValidationResult, +} from './validation.service'; // Observability threshold: log a warning when the full-state op aggregate scan // exceeds this duration. Mirrors the threshold used by the legacy snapshot // vector-clock aggregate in OperationDownloadService so production logs use a // consistent slow-aggregate signal. const SLOW_FULL_STATE_AGGREGATE_MS = 5_000; +const INVALID_AUDIT_FIELD = '[invalid]'; +const SAFE_AUDIT_ID_REGEX = /^[A-Za-z0-9_-]+$/; + +const isSafeAuditIdentifier = (value: unknown, maxLength: number): value is string => + typeof value === 'string' && + value.length > 0 && + value.length <= maxLength && + SAFE_AUDIT_ID_REGEX.test(value); + +const getSafeAuditOperationMetadata = ( + op: Operation, +): { opId: string; entityType: string; entityId?: string; opType: string } => { + const rawOp = op as unknown as Record; + const rawOpId = rawOp['id']; + const rawEntityType = rawOp['entityType']; + const rawEntityId = rawOp['entityId']; + const rawOpType = rawOp['opType']; + + return { + opId: isSafeAuditIdentifier(rawOpId, 255) ? rawOpId : INVALID_AUDIT_FIELD, + entityType: + typeof rawEntityType === 'string' && ALLOWED_ENTITY_TYPES.has(rawEntityType) + ? rawEntityType + : INVALID_AUDIT_FIELD, + entityId: + rawEntityId === undefined || rawEntityId === null + ? undefined + : isSafeAuditIdentifier(rawEntityId, 255) + ? rawEntityId + : INVALID_AUDIT_FIELD, + opType: + typeof rawOpType === 'string' && OP_TYPES.includes(rawOpType as Operation['opType']) + ? rawOpType + : INVALID_AUDIT_FIELD, + }; +}; + +const getSafeAuditClientId = (clientId: string): string => + clientId.length <= MAX_CLIENT_ID_LENGTH && CLIENT_ID_REGEX.test(clientId) + ? clientId + : INVALID_AUDIT_FIELD; const toSafeServerSeq = (value: number | bigint | undefined, userId: number): number => { if (typeof value === 'bigint') { @@ -75,9 +125,8 @@ export class OperationUploadService { Logger.audit({ event: 'TIMESTAMP_CLAMPED', userId, - clientId, - opId: op.id, - entityType: op.entityType, + clientId: getSafeAuditClientId(clientId), + ...getSafeAuditOperationMetadata(op), originalTimestamp, clampedTo: maxAllowedTimestamp, driftMs: originalTimestamp - now, @@ -97,13 +146,10 @@ export class OperationUploadService { Logger.audit({ event: 'OP_REJECTED', userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, + clientId: getSafeAuditClientId(clientId), + ...getSafeAuditOperationMetadata(op), errorCode, - reason: error, - opType: op.opType, + reason: errorCode ?? 'OP_REJECTED', }); return { @@ -281,8 +327,9 @@ export class OperationUploadService { } /** - * Stage 1: clamp future timestamps and validate every op in memory (no DB). - * Invalid ops get a terminal rejection written into `results` by index. + * Stage 1: reserve each transport-level operation ID, clamp timestamps, and + * validate every first occurrence in memory (no DB). Invalid first ops and + * every later same-ID sibling get terminal results by index. */ private validateAndClampBatch( userId: number, @@ -293,9 +340,36 @@ export class OperationUploadService { prevalidatedResults?: ReadonlyMap, ): BatchUploadCandidate[] { const validatedCandidates: BatchUploadCandidate[] = []; + const firstOperationById = new Map< + string, + { op: Operation; originalTimestamp: number } + >(); for (let i = 0; i < ops.length; i++) { const op = ops[i]; const originalTimestamp = this.clampFutureTimestamp(userId, clientId, op, now); + const firstOperation = firstOperationById.get(op.id); + if (firstOperation) { + const isExactRetry = isSameIncomingOperation( + firstOperation.op, + op, + firstOperation.originalTimestamp, + originalTimestamp, + ); + results[i] = this.rejectedUploadResult( + userId, + clientId, + op, + isExactRetry + ? 'Duplicate operation ID' + : 'Operation ID already belongs to a different operation', + isExactRetry + ? SYNC_ERROR_CODES.DUPLICATE_OPERATION + : SYNC_ERROR_CODES.INVALID_OP_ID, + ); + continue; + } + firstOperationById.set(op.id, { op, originalTimestamp }); + const validation = prevalidatedResults?.get(op) ?? this.validationService.validateOp(op, clientId); @@ -634,6 +708,7 @@ export class OperationUploadService { tx: Prisma.TransactionClient, prevalidatedResult?: ValidationResult, wasOccupiedAtRequestStart?: boolean, + firstRequestOperation?: { op: Operation; originalTimestamp: number }, ): Promise { // Rejected ops have no storage cost; the caller only reads storageBytes when // result.accepted is true. @@ -650,25 +725,40 @@ export class OperationUploadService { // Validate operation (including clientId match) const validation = prevalidatedResult ?? this.validationService.validateOp(op, clientId); - if (!validation.valid) { - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode: validation.errorCode, - reason: validation.error, - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: validation.error, - errorCode: validation.errorCode, - }); + if (firstRequestOperation) { + const isExactRetry = isSameIncomingOperation( + firstRequestOperation.op, + op, + firstRequestOperation.originalTimestamp, + originalTimestamp, + ); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + isExactRetry + ? 'Duplicate operation ID' + : 'Operation ID already belongs to a different operation', + isExactRetry + ? SYNC_ERROR_CODES.DUPLICATE_OPERATION + : SYNC_ERROR_CODES.INVALID_OP_ID, + ), + ); } + + if (!validation.valid) { + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + validation.error, + validation.errorCode, + ), + ); + } + // Capture the *unpruned* vector clock for full-state ops. The op row stores // the pruned clock (see `limitVectorClockSize` call below); persisting the // unpruned copy on `user_sync_state` lets the download path re-prune at @@ -696,42 +786,26 @@ export class OperationUploadService { originalTimestamp, ) ) { - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, - reason: 'Operation ID already belongs to a different operation', - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: 'Operation ID already belongs to a different operation', - errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, - }); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + 'Operation ID already belongs to a different operation', + SYNC_ERROR_CODES.INVALID_OP_ID, + ), + ); } - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode: SYNC_ERROR_CODES.DUPLICATE_OPERATION, - reason: 'Duplicate operation ID (pre-check)', - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: 'Duplicate operation ID', - errorCode: SYNC_ERROR_CODES.DUPLICATE_OPERATION, - }); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + 'Duplicate operation ID', + SYNC_ERROR_CODES.DUPLICATE_OPERATION, + ), + ); } if (wasOccupiedAtRequestStart) { @@ -754,24 +828,16 @@ export class OperationUploadService { conflict.conflictType === 'equal_different_client' ? SYNC_ERROR_CODES.CONFLICT_CONCURRENT : SYNC_ERROR_CODES.CONFLICT_SUPERSEDED; - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode, - reason: conflict.reason, - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: conflict.reason, - errorCode, - existingClock: conflict.existingClock, - }); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + conflict.reason, + errorCode, + conflict.existingClock, + ), + ); } // Get next sequence number @@ -797,24 +863,16 @@ export class OperationUploadService { finalConflict.conflictType === 'equal_different_client' ? SYNC_ERROR_CODES.CONFLICT_CONCURRENT : SYNC_ERROR_CODES.CONFLICT_SUPERSEDED; - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode, - reason: `[RACE] ${finalConflict.reason}`, - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: finalConflict.reason, - errorCode, - existingClock: finalConflict.existingClock, - }); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + finalConflict.reason, + errorCode, + finalConflict.existingClock, + ), + ); } // Prune vector clock AFTER conflict detection but BEFORE storage. @@ -896,42 +954,26 @@ export class OperationUploadService { originalTimestamp, ) ) { - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, - reason: 'Operation ID already belongs to a different operation', - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: 'Operation ID already belongs to a different operation', - errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, - }); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + 'Operation ID already belongs to a different operation', + SYNC_ERROR_CODES.INVALID_OP_ID, + ), + ); } - Logger.audit({ - event: 'OP_REJECTED', - userId, - clientId, - opId: op.id, - entityType: op.entityType, - entityId: op.entityId, - errorCode: SYNC_ERROR_CODES.DUPLICATE_OPERATION, - reason: 'Duplicate operation ID (insert race)', - opType: op.opType, - }); - return reject({ - opId: op.id, - accepted: false, - error: 'Duplicate operation ID', - errorCode: SYNC_ERROR_CODES.DUPLICATE_OPERATION, - }); + return reject( + this.rejectedUploadResult( + userId, + clientId, + op, + 'Duplicate operation ID', + SYNC_ERROR_CODES.DUPLICATE_OPERATION, + ), + ); } if (fullStateVectorClock) { diff --git a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts index 568ad1c689..feda3e9be7 100644 --- a/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.ops-handler.ts @@ -250,8 +250,8 @@ export const uploadOpsHandler = async ( if (rejected > 0) { Logger.debug( - `[user:${userId}] Rejected ops:`, - results.filter((r) => !r.accepted), + `[user:${userId}] Rejected op error codes:`, + results.filter((r) => !r.accepted).map((r) => r.errorCode ?? 'UNKNOWN'), ); } diff --git a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts index 281e695406..d5b718887c 100644 --- a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts @@ -6,7 +6,13 @@ import { Logger } from '../logger'; import { getAuthUser } from '../middleware'; import { getSyncService } from './sync.service'; import { getWsConnectionService } from './services/websocket-connection.service'; -import { SYNC_ERROR_CODES, UploadResult } from './sync.types'; +import { + DUPLICATE_OP_SELECT, + Operation, + SYNC_ERROR_CODES, + UploadResult, +} from './sync.types'; +import { isSameIncomingOperation } from './conflict'; import { isSingleTokenGzipEncoding, parseCompressedJsonBody, @@ -222,6 +228,57 @@ export const uploadSnapshotHandler = async ( const result = await syncService.runWithStorageUsageLock( userId, async () => { + // Clean-slate uploads are destructive, so request-cache deduplication is + // not sufficient: it is process-local and expires. The client-supplied + // opId is the durable idempotency key. Check it inside the per-user lock + // before quota work or uploadOps can delete any existing data. + if (isCleanSlate && opId) { + const existingCleanSlateOp = await prisma.operation.findUnique({ + where: { id: opId }, + select: { ...DUPLICATE_OP_SELECT, serverSeq: true }, + }); + if (existingCleanSlateOp) { + const existingOperation: Operation = { + id: existingCleanSlateOp.id, + clientId: existingCleanSlateOp.clientId, + actionType: existingCleanSlateOp.actionType, + opType: existingCleanSlateOp.opType as Operation['opType'], + entityType: existingCleanSlateOp.entityType, + entityId: existingCleanSlateOp.entityId ?? undefined, + entityIds: existingCleanSlateOp.entityIds, + payload: existingCleanSlateOp.payload, + vectorClock: existingCleanSlateOp.vectorClock as Operation['vectorClock'], + timestamp: op.timestamp, + schemaVersion: existingCleanSlateOp.schemaVersion, + isPayloadEncrypted: existingCleanSlateOp.isPayloadEncrypted, + syncImportReason: existingCleanSlateOp.syncImportReason ?? undefined, + }; + const isExactRetry = + existingCleanSlateOp.userId === userId && + (existingCleanSlateOp.opType === 'SYNC_IMPORT' || + existingCleanSlateOp.opType === 'BACKUP_IMPORT' || + existingCleanSlateOp.opType === 'REPAIR') && + isSameIncomingOperation(existingOperation, op, 0, 0); + if (isExactRetry) { + Logger.info( + `[user:${userId}] Idempotent clean-slate retry from client ${clientId} ` + + `for existing op seq=${existingCleanSlateOp.serverSeq}`, + ); + return { + opId, + accepted: true, + serverSeq: existingCleanSlateOp.serverSeq, + }; + } + return { + opId, + accepted: false, + error: 'Operation ID already belongs to a different operation', + errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, + }; + } + } + if (reason === 'initial' && !isCleanSlate) { const existingImport = await findExistingSyncImport(userId, opId); diff --git a/packages/super-sync-server/src/sync/sync.service.ts b/packages/super-sync-server/src/sync/sync.service.ts index a1d3697e7c..81b3f4ba1d 100644 --- a/packages/super-sync-server/src/sync/sync.service.ts +++ b/packages/super-sync-server/src/sync/sync.service.ts @@ -111,10 +111,13 @@ export class SyncService { * and block otherwise valid operations in the same request. */ filterValidOpsForQuota(ops: Operation[], clientId: string): Operation[] { + const seenOperationIds = new Set(); return ops.filter((op) => { + const isFirstOccurrence = !seenOperationIds.has(op.id); + seenOperationIds.add(op.id); const validation = this.validationService.validateOp(op, clientId); this.prevalidatedOps.set(op, validation); - return validation.valid; + return isFirstOccurrence && validation.valid; }); } @@ -218,7 +221,24 @@ export class SyncService { }); uploadDbRoundtrips++; + const firstOperationById = new Map< + string, + { op: Operation; originalTimestamp: number } + >(); + for (const op of ops) { + const firstRequestOperation = firstOperationById.get(op.id); + if (!firstRequestOperation) { + firstOperationById.set(op.id, { + op, + originalTimestamp: op.timestamp, + }); + } + const validation = + prevalidatedResults.get(op) ?? + this.validationService.validateOp(op, clientId); + prevalidatedResults.set(op, validation); + const { result, storageBytes, fallback } = await this.operationUploadService.processOperation( userId, @@ -226,8 +246,9 @@ export class SyncService { op, now, tx, - prevalidatedResults.get(op), + validation, requestStartOccupiedIds?.has(op.id), + firstRequestOperation, ); results.push(result); if (result.accepted) { diff --git a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts index b4a112d193..ce8a9d1ee3 100644 --- a/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts +++ b/packages/super-sync-server/tests/sync-compressed-body.routes.spec.ts @@ -70,7 +70,7 @@ vi.mock('../src/db', () => ({ import { syncRoutes } from '../src/sync/sync.routes'; import { SYNC_ERROR_CODES } from '../src/sync/sync.types'; import { computeOpStorageBytes } from '../src/sync/sync.const'; -import { SUPER_SYNC_MAX_OPS_PER_UPLOAD } from '@sp/shared-schema'; +import { CURRENT_SCHEMA_VERSION, SUPER_SYNC_MAX_OPS_PER_UPLOAD } from '@sp/shared-schema'; const gzipAsync = promisify(zlib.gzip); @@ -170,6 +170,7 @@ describe('Sync compressed body routes', () => { }); mocks.syncService.getCachedSnapshotBytes.mockResolvedValue(0); mocks.prisma.operation.findFirst.mockResolvedValue(null); + mocks.prisma.operation.findUnique.mockReset().mockResolvedValue(null); mocks.prisma.operation.findMany.mockResolvedValue([]); app = Fastify(); @@ -905,6 +906,7 @@ describe('Sync compressed body routes', () => { clientId, reason: 'initial', vectorClock: {}, + opId: '018f2f0b-1c2d-7a1b-8c3d-123456789abc', isCleanSlate: true, }, }); @@ -914,6 +916,98 @@ describe('Sync compressed body routes', () => { expect(mocks.syncService.uploadOps).not.toHaveBeenCalled(); }); + it('should return persistent success for a clean-slate retry before deleting data', async () => { + const opId = '018f2f0b-1c2d-7a1b-8c3d-123456789abc'; + const clientId = 'clean-slate-retry-client'; + const state = { TASK: { 'task-1': { id: 'task-1' } } }; + const vectorClock = { [clientId]: 1 }; + mocks.prisma.operation.findUnique.mockResolvedValueOnce({ + id: opId, + userId: 1, + clientId, + actionType: '[SP_ALL] Load(import) all data', + opType: 'SYNC_IMPORT', + entityType: 'ALL', + entityId: null, + entityIds: [], + payload: state, + vectorClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + clientTimestamp: BigInt(1), + receivedAt: BigInt(1), + isPayloadEncrypted: false, + syncImportReason: null, + serverSeq: 77, + }); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/snapshot', + headers: { authorization: `Bearer ${authToken}` }, + payload: { + state, + clientId, + reason: 'recovery', + vectorClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + opId, + isCleanSlate: true, + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ accepted: true, serverSeq: 77 }); + expect(mocks.syncService.uploadOps).not.toHaveBeenCalled(); + expect(mocks.syncService.checkStorageQuota).not.toHaveBeenCalled(); + }); + + it('should reject a clean-slate retry whose opId belongs to different content', async () => { + const opId = '018f2f0b-1c2d-7a1b-8c3d-123456789abc'; + const clientId = 'clean-slate-collision-client'; + const vectorClock = { [clientId]: 1 }; + mocks.prisma.operation.findUnique.mockResolvedValueOnce({ + id: opId, + userId: 1, + clientId, + actionType: '[SP_ALL] Load(import) all data', + opType: 'SYNC_IMPORT', + entityType: 'ALL', + entityId: null, + entityIds: [], + payload: { TASK: { existing: { id: 'existing' } } }, + vectorClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + clientTimestamp: BigInt(1), + receivedAt: BigInt(1), + isPayloadEncrypted: false, + syncImportReason: null, + serverSeq: 77, + }); + + const response = await app.inject({ + method: 'POST', + url: '/api/sync/snapshot', + headers: { authorization: `Bearer ${authToken}` }, + payload: { + state: { TASK: { replacement: { id: 'replacement' } } }, + clientId, + reason: 'recovery', + vectorClock, + schemaVersion: CURRENT_SCHEMA_VERSION, + opId, + isCleanSlate: true, + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ + accepted: false, + error: 'Operation ID already belongs to a different operation', + }); + expect(mocks.syncService.uploadOps).not.toHaveBeenCalled(); + expect(mocks.syncService.checkStorageQuota).not.toHaveBeenCalled(); + }); + it('should repeat initial snapshot duplicate detection inside the user lock', async () => { const clientId = 'initial-race-client'; mocks.prisma.operation.findFirst diff --git a/packages/super-sync-server/tests/sync.service.spec.ts b/packages/super-sync-server/tests/sync.service.spec.ts index 3dd887c608..67a3704433 100644 --- a/packages/super-sync-server/tests/sync.service.spec.ts +++ b/packages/super-sync-server/tests/sync.service.spec.ts @@ -79,6 +79,23 @@ vi.mock('../src/db', async () => { } } } + if (args.where?.entityType && Array.isArray(args.where?.OR)) { + const targetEntityId = + args.where.OR.find((condition: any) => condition.entityId !== undefined) + ?.entityId ?? + args.where.OR.find((condition: any) => condition.entityIds?.has !== undefined) + ?.entityIds.has; + const ops = Array.from(state.operations.values()) + .filter( + (op: any) => + op.userId === args.where.userId && + op.entityType === args.where.entityType && + (op.entityId === targetEntityId || + op.entityIds?.includes(targetEntityId)), + ) + .sort((a: any, b: any) => b.serverSeq - a.serverSeq); + return applyOperationSelect(ops[0], args.select) || null; + } if (args.where?.entityId && args.where?.entityType) { const ops = Array.from(state.operations.values()) .filter( @@ -359,6 +376,7 @@ vi.mock('../src/db', async () => { entityId: op.entityId, clientId: op.clientId, vectorClock: op.vectorClock, + serverSeq: op.serverSeq, })); } if (sql.includes('jsonb_each_text(vector_clock)')) { @@ -605,6 +623,8 @@ import { DeviceService } from '../src/sync/services/device.service'; import { OperationDownloadService } from '../src/sync/services/operation-download.service'; import { Operation, DEFAULT_SYNC_CONFIG, SYNC_ERROR_CODES } from '../src/sync/sync.types'; import { prisma } from '../src/db'; +import { Logger } from '../src/logger'; +import { CURRENT_SCHEMA_VERSION } from '@sp/shared-schema'; describe('SyncService', () => { const userId = 1; @@ -628,6 +648,19 @@ describe('SyncService', () => { ...overrides, }); + const makeGlobalConfigOp = (overrides: Partial = {}): Operation => + makeOp({ + actionType: '[GLOBAL_CONFIG] Update section', + opType: 'UPD', + entityType: 'GLOBAL_CONFIG', + entityId: 'misc', + payload: { + sectionKey: 'misc', + sectionCfg: { defaultProjectId: 'project-1' }, + }, + ...overrides, + }); + beforeEach(() => { // Reset all test data stores resetTestState(); @@ -676,6 +709,23 @@ describe('SyncService', () => { expect(result).toEqual([validOp]); }); + + it('does not charge a later valid sibling when an invalid op reserved its ID', () => { + const service = new SyncService(); + const invalidFirst = makeOp({ + id: 'reserved-by-invalid-op', + entityType: 'INVALID_ENTITY_TYPE', + }); + const laterLargeSibling = makeOp({ + id: invalidFirst.id, + entityId: 'fresh-task', + payload: { data: 'x'.repeat(10_000) }, + }); + + expect( + service.filterValidOpsForQuota([invalidFirst, laterLargeSibling], clientId), + ).toEqual([]); + }); }); describe('uploadOps', () => { @@ -826,6 +876,114 @@ describe('SyncService', () => { ); }); + it('terminally rejects a later serial same-ID sibling when the first one conflicts', async () => { + const service = new SyncService({ batchUpload: false }); + const otherClientId = 'other-device'; + const existing = makeOp({ + id: 'existing-op', + clientId: otherClientId, + entityId: 'blocked-task', + vectorClock: { [otherClientId]: 1 }, + }); + expect( + (await service.uploadOps(userId, otherClientId, [existing]))[0].accepted, + ).toBe(true); + + const repeatedId = 'repeated-request-id'; + const first = makeOp({ + id: repeatedId, + entityId: 'blocked-task', + payload: { title: 'small' }, + vectorClock: { [clientId]: 1 }, + }); + const laterLargeSibling = makeOp({ + id: repeatedId, + entityId: 'fresh-task', + payload: { data: 'x'.repeat(10_000) }, + vectorClock: { [clientId]: 2 }, + timestamp: first.timestamp + 1, + }); + + const results = await service.uploadOps(userId, clientId, [ + first, + laterLargeSibling, + ]); + + expect(results[0]).toEqual( + expect.objectContaining({ + accepted: false, + errorCode: SYNC_ERROR_CODES.CONFLICT_CONCURRENT, + }), + ); + expect(results[1]).toEqual( + expect.objectContaining({ + accepted: false, + errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, + }), + ); + expect(testState.operations.has(repeatedId)).toBe(false); + }); + + it('redacts malformed operation metadata from audit logs', async () => { + const service = new SyncService({ batchUpload: false }); + const privateText = 'private task title that must not be logged'; + const auditSpy = vi.spyOn(Logger, 'audit').mockImplementation(() => undefined); + const malformed = makeOp({ + id: privateText, + entityType: privateText, + }); + + const result = await service.uploadOps(userId, clientId, [malformed]); + + expect(result[0].accepted).toBe(false); + const rejection = auditSpy.mock.calls + .map(([entry]) => entry) + .find((entry) => entry.event === 'OP_REJECTED'); + expect(rejection).toBeDefined(); + expect(rejection?.opId).toBe('[invalid]'); + expect(rejection?.entityType).toBe('[invalid]'); + expect(rejection?.reason).toBe(SYNC_ERROR_CODES.INVALID_ENTITY_TYPE); + expect(JSON.stringify(rejection)).not.toContain(privateText); + }); + + it.each([ + ['serial', false], + ['batch', true], + ])( + 'terminally rejects a valid %s sibling whose ID was reserved by an invalid op', + async (_label, batchUpload) => { + const service = new SyncService({ batchUpload }); + const invalidFirst = makeOp({ + id: 'invalid-first-shared-id', + entityType: 'INVALID_ENTITY_TYPE', + }); + const laterLargeSibling = makeOp({ + id: invalidFirst.id, + entityId: 'fresh-task', + payload: { data: 'x'.repeat(10_000) }, + }); + + const results = await service.uploadOps(userId, clientId, [ + invalidFirst, + laterLargeSibling, + ]); + + expect(results[0]).toEqual( + expect.objectContaining({ + accepted: false, + errorCode: SYNC_ERROR_CODES.INVALID_ENTITY_TYPE, + }), + ); + expect(results[1]).toEqual( + expect.objectContaining({ + accepted: false, + errorCode: SYNC_ERROR_CODES.INVALID_OP_ID, + }), + ); + expect(testState.operations.has(invalidFirst.id)).toBe(false); + }, + ); + it('should reject intra-batch entity conflicts in order', async () => { const service = new SyncService({ batchUpload: true }); const ops: Operation[] = [ @@ -868,6 +1026,118 @@ describe('SyncService', () => { expect(testState.operations.size).toBe(1); }); + it.each([ + ['serial', false], + ['batch', true], + ])( + 'rejects a v2 tasks write against an already-stored raw v1 misc row in the %s path', + async (_label, batchUpload) => { + const legacyClientId = 'legacy-client'; + testState.userSyncStates.set(userId, { userId, lastSeq: 1 }); + testState.serverSeqCounter = 1; + testState.operations.set('stored-legacy-misc', { + id: 'stored-legacy-misc', + userId, + clientId: legacyClientId, + serverSeq: 1, + actionType: '[GLOBAL_CONFIG] Update section', + opType: 'UPD', + entityType: 'GLOBAL_CONFIG', + entityId: 'misc', + entityIds: [], + payload: { + sectionKey: 'misc', + sectionCfg: { defaultProjectId: 'legacy-project' }, + }, + payloadBytes: BigInt(10), + vectorClock: { [legacyClientId]: 1 }, + schemaVersion: 1, + clientTimestamp: BigInt(Date.now() - 1_000), + receivedAt: BigInt(Date.now() - 1_000), + isPayloadEncrypted: false, + syncImportReason: null, + }); + + const service = new SyncService({ batchUpload }); + const result = await service.uploadOps(userId, clientId, [ + makeGlobalConfigOp({ + id: 'current-tasks-write', + entityId: 'tasks', + payload: { + sectionKey: 'tasks', + sectionCfg: { defaultProjectId: 'current-project' }, + }, + vectorClock: { [clientId]: 1 }, + schemaVersion: CURRENT_SCHEMA_VERSION, + }), + ]); + + expect(result).toEqual([ + expect.objectContaining({ + opId: 'current-tasks-write', + accepted: false, + errorCode: SYNC_ERROR_CODES.CONFLICT_CONCURRENT, + existingClock: { [legacyClientId]: 1 }, + }), + ]); + expect(testState.operations.size).toBe(1); + }, + ); + + it.each([ + ['serial', false], + ['batch', true], + ])( + 'atomically rejects a new mixed v1 misc upload that conflicts with v2 tasks in the %s path', + async (_label, batchUpload) => { + const currentClientId = 'current-client'; + const service = new SyncService({ batchUpload }); + const currentResult = await service.uploadOps(userId, currentClientId, [ + makeGlobalConfigOp({ + id: 'existing-current-tasks', + clientId: currentClientId, + entityId: 'tasks', + payload: { + sectionKey: 'tasks', + sectionCfg: { defaultProjectId: 'current-project' }, + }, + vectorClock: { [currentClientId]: 1 }, + schemaVersion: CURRENT_SCHEMA_VERSION, + }), + ]); + expect(currentResult[0].accepted).toBe(true); + + const sourceId = 'incoming-legacy-mixed'; + const legacyResult = await service.uploadOps(userId, clientId, [ + makeGlobalConfigOp({ + id: sourceId, + payload: { + sectionKey: 'misc', + sectionCfg: { + defaultProjectId: 'legacy-project', + isMinimizeToTray: true, + }, + }, + vectorClock: { [clientId]: 1 }, + schemaVersion: 1, + }), + ]); + + expect(legacyResult).toEqual([ + expect.objectContaining({ + opId: sourceId, + accepted: false, + errorCode: SYNC_ERROR_CODES.CONFLICT_CONCURRENT, + existingClock: { [currentClientId]: 1 }, + }), + ]); + expect(testState.operations.has(`${sourceId}_misc`)).toBe(false); + expect(testState.operations.has(`${sourceId}_tasks`)).toBe(false); + expect(testState.operations.has(sourceId)).toBe(false); + expect(testState.operations.size).toBe(1); + }, + ); + it('should use entityIds when prefetching batch conflicts', async () => { const service = new SyncService({ batchUpload: true }); testState.userSyncStates.set(userId, { userId, lastSeq: 1 }); From 307fda3584924630ec21b057203bc3b4b3c0a66a Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:04:54 +0200 Subject: [PATCH 39/47] fix(shared-schema): let existing tasks settings win in v1-to-v2 merge The misc-to-tasks migration spread the transformed legacy misc values over an already-populated tasks section, so a stale v1 misc copy could overwrite settings a v2 client had since changed. Flip the merge order (existing tasks section wins) and drop the already-migrated early return, which made the outcome depend on which duplicate arrived first. --- .../misc-to-tasks-settings-migration-v1-to-v2.ts | 9 +++------ .../misc-to-tasks-settings-migration-v1-to-v2.spec.ts | 7 ++++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts b/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts index 9a0d35b9b9..317f9e4134 100644 --- a/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts +++ b/packages/shared-schema/src/migrations/misc-to-tasks-settings-migration-v1-to-v2.ts @@ -30,17 +30,14 @@ export const MiscToTasksSettingsMigration_v1v2: SchemaMigration = { const tasks = state.globalConfig?.tasks ?? {}; - // Skip if already migrated (tasks has new fields AND misc has no migrated fields) - if (tasks.isConfirmBeforeDelete !== undefined && !hasMigratedFields(misc)) { - return state; - } - return { ...state, globalConfig: { ...state.globalConfig, misc: removeMigratedFields(misc), - tasks: { ...tasks, ...transformMiscToTasks(misc) }, + // Existing target values come from a newer/partial migration and must + // not be overwritten by stale legacy copies left in misc. + tasks: { ...transformMiscToTasks(misc), ...tasks }, }, }; }, diff --git a/packages/shared-schema/tests/migrations/misc-to-tasks-settings-migration-v1-to-v2.spec.ts b/packages/shared-schema/tests/migrations/misc-to-tasks-settings-migration-v1-to-v2.spec.ts index 82b3fb406b..57b69fcfa7 100644 --- a/packages/shared-schema/tests/migrations/misc-to-tasks-settings-migration-v1-to-v2.spec.ts +++ b/packages/shared-schema/tests/migrations/misc-to-tasks-settings-migration-v1-to-v2.spec.ts @@ -64,7 +64,7 @@ describe('Migrate MiscConfig to TasksConfig', () => { expect(migratedState).toEqual(initialState); }); - it('should complete migration even if tasks.isConfirmBeforeDelete exists but misc still has migrated fields', () => { + it('should preserve target choices while migrating remaining legacy fields', () => { const initialState = { globalConfig: { misc: { @@ -86,8 +86,9 @@ describe('Migrate MiscConfig to TasksConfig', () => { }; }; - // Should migrate remaining fields from misc - expect(migratedState.globalConfig.tasks.isConfirmBeforeDelete).toBe(true); // Migrated value overwrites + // A populated target field belongs to the newer/partial migration and wins + // over the stale legacy copy. + expect(migratedState.globalConfig.tasks.isConfirmBeforeDelete).toBe(false); expect(migratedState.globalConfig.tasks.defaultProjectId).toBe('proj-123'); // Migrated expect(migratedState.globalConfig.tasks.existingField).toBe('existing'); // Preserved expect(migratedState.globalConfig.misc.isConfirmBeforeTaskDelete).toBeUndefined(); // Removed From 582929e375d791dfde366b6289be4f42b925deda Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:06:20 +0200 Subject: [PATCH 40/47] docs(sync): document atomic checkpoint, db v8 barrier and rebuild undo Update the archive-operation lifecycle docs to the atomic reducer-checkpoint + clock-merge design, record the IndexedDB v8 downgrade barrier, drop the removed PENDING_OPERATION_EXPIRY_MS constant, and describe the durable Use-Server-Data Undo across reloads. --- .../diagrams/06-archive-operations.md | 8 ++++---- .../operation-log-architecture.md | 12 ++++++------ docs/sync-and-op-log/operation-rules.md | 17 ++++++++--------- docs/sync-and-op-log/supersync-scenarios.md | 7 ++++--- docs/wiki/3.06-User-Data.md | 6 +++--- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docs/sync-and-op-log/diagrams/06-archive-operations.md b/docs/sync-and-op-log/diagrams/06-archive-operations.md index d6b25e71ec..0e27773472 100644 --- a/docs/sync-and-op-log/diagrams/06-archive-operations.md +++ b/docs/sync-and-op-log/diagrams/06-archive-operations.md @@ -88,11 +88,11 @@ flowchart TD subgraph RemoteOp["REMOTE Operation (Sync)"] R1[Download operation
from sync] --> R2["Append remote row
status: pending"] R2 --> R3["Bulk reducer dispatch
meta.isRemote=true"] - R3 --> R4["Reducer-commit checkpoint:
status archive_pending
merge vector clocks"] + R3 --> R4["Atomic reducer checkpoint:
archive_pending + vector clocks"] R4 --> R5["ArchiveOperationHandler
.handleOperation"] R5 --> R6{"Archive side effect
succeeded?"} R6 -->|Yes| R7["Mark applied"] - R6 -->|No| R8["Mark attempted row failed;
successors stay archive_pending"] + R6 -->|No| R8["Bump attempted row retryCount;
successors stay quarantined"] NoEffect["❌ Regular effects DON'T run
(action has meta.isRemote=true)"] end @@ -116,14 +116,14 @@ flowchart TD ## ArchiveOperationHandler Integration -The `OperationApplierService` applies operations in the order they arrive from the sync server, which preserves causal ordering (each client uploads its ops in causal order, and the server assigns sequence numbers in upload order). Incoming rows are first stored as `pending`. Immediately after the single bulk reducer dispatch, the reducer-commit callback checkpoints the entire batch as `archive_pending` and merges its vector clocks; only then do archive side effects run. Successful rows become `applied`. The attempted row becomes `failed` if an archive side effect throws, while unattempted successors remain `archive_pending`. On startup, hydration restores reducer state and retries `archive_pending`/`failed` rows with reducer dispatch disabled, so additive reducers are never applied twice. +The `OperationApplierService` applies operations in the order they arrive from the sync server, which preserves causal ordering (each client uploads its ops in causal order, and the server assigns sequence numbers in upload order). Incoming rows are first stored as `pending`. Immediately after the single bulk reducer dispatch, one transaction changes the whole reducer-committed batch to `archive_pending` and merges its vector clocks; only then do archive side effects run. Successful rows become `applied`. If an archive side effect throws, only the attempted row becomes `failed` and consumes `retryCount`; unattempted successors remain `archive_pending`. On startup, hydration restores reducer state and retries `archive_pending`/`failed` rows with reducer dispatch disabled, so additive reducers are never applied twice. ```mermaid flowchart TD subgraph OperationApplierService["OperationApplierService (Bulk Dispatch)"] OA1["Receive rows already stored
as pending"] --> OA3[convertOpToAction] OA3 --> OA4["Single bulk dispatch
with meta.isRemote=true"] - OA4 --> OA4b["Reducer-commit callback:
mark archive_pending
merge vector clocks"] + OA4 --> OA4b["Atomic reducer checkpoint:
archive_pending + vector clocks"] OA4b --> OA5["archiveOperationHandler
.handleOperation(action)"] end diff --git a/docs/sync-and-op-log/operation-log-architecture.md b/docs/sync-and-op-log/operation-log-architecture.md index 57592e02b3..0c7d7b2461 100644 --- a/docs/sync-and-op-log/operation-log-architecture.md +++ b/docs/sync-and-op-log/operation-log-architecture.md @@ -170,7 +170,7 @@ interface OperationLogEntry { source: 'local' | 'remote'; syncedAt?: number; // For server sync (Part C) rejectedAt?: number; // When rejected during conflict resolution - applicationStatus?: 'pending' | 'archive_pending' | 'applied' | 'failed'; + applicationStatus?: 'pending' | 'archive_pending' | 'failed' | 'applied'; } // state_cache table - periodic snapshots @@ -211,11 +211,11 @@ interface StateCache { Downloaded operations use a durable status transition so reducer state, archive IndexedDB side effects, vector clocks, and the server cursor cannot disagree after a crash: 1. `pending` — the remote op is stored, but no reducer-commit checkpoint exists yet. -2. `archive_pending` — the complete batch was bulk-dispatched to reducers and its vector clocks were merged; archive side effects are still outstanding. -3. `applied` — reducer and archive work both completed. -4. `failed` — an attempted archive side effect failed. Later operations in the same batch remain `archive_pending` without consuming retry budget. +2. `archive_pending` — reducers committed and the op's vector clock was merged atomically; archive side effects have not yet completed. +3. `failed` — an archive side-effect attempt failed. `retryCount` is charged only to the attempted row, so later rows in the same batch do not consume retry budget. +4. `applied` — reducer and archive work both completed. -Startup hydration replays the persisted state history, converts surviving `pending` rows to the archive checkpoint, and retries `archive_pending`/`failed` rows with reducer dispatch disabled. Ordinary sync refuses to download, upload, or advance its cursor while any of these incomplete rows remain. +Startup hydration replays persisted state history, quarantines surviving `pending` rows, and retries `archive_pending`/`failed` rows with reducer dispatch disabled. Ordinary sync refuses to download, upload, or advance its cursor while any incomplete rows remain. Database version 8 is a downgrade barrier: released readers that do not understand the distinct reducer checkpoint cannot open the newer store and silently overlook outstanding archive work. Local actions buffered during a remote-apply window stay ordered until each operation is durable. Transient persistence failures keep the failed suffix queued and block the current sync so a later sync can retry. A deterministically invalid buffered action also remains queued, but requires reload: its reducer already changed live state, so discarding it would let live state diverge from the durable operation log. @@ -2324,7 +2324,7 @@ When adding new entities or relationships: > - **Archive validation**: archiveOld tasks now validated for project/tag references, null-safety added > - **Lock service robustness**: Handle NaN timestamps and invalid lock formats in fallback lock > - **Array payload rejection**: Explicit check to reject arrays (which bypass `typeof === 'object'`) -> - **Pending operation expiry**: Operations pending >24h are quarantined as failed; hydration still restores their reducer history, while archive recovery and the sync gate remain active (`PENDING_OPERATION_EXPIRY_MS`). +> - **Remote apply checkpoints**: reducer commit and vector-clock merge are atomic; `archive_pending` distinguishes unattempted archive work from attempted `failed` rows, while hydration and the sync gate keep incomplete work visible. --- diff --git a/docs/sync-and-op-log/operation-rules.md b/docs/sync-and-op-log/operation-rules.md index 455cf1d3c2..3fa400feda 100644 --- a/docs/sync-and-op-log/operation-rules.md +++ b/docs/sync-and-op-log/operation-rules.md @@ -193,15 +193,14 @@ export class MyEffects { See `operation-log.const.ts` for all configurable values: -| Constant | Value | Description | -| ----------------------------------- | -------- | -------------------------------------------------------------------------- | -| `COMPACTION_TRIGGER` | 500 ops | Operations before automatic compaction | -| `COMPACTION_RETENTION_MS` | 7 days | Synced ops older than this may be deleted | -| `EMERGENCY_COMPACTION_RETENTION_MS` | 1 day | Shorter retention for quota exceeded | -| `MAX_COMPACTION_FAILURES` | 3 | Failures before user notification | -| `MAX_DOWNLOAD_OPS_IN_MEMORY` | 50,000 | Bounds memory during API download | -| `REMOTE_OP_FILE_RETENTION_MS` | 14 days | Server-side operation file retention | -| `PENDING_OPERATION_EXPIRY_MS` | 24 hours | Pending ops older than this are quarantined as failed for archive recovery | +| Constant | Value | Description | +| ----------------------------------- | ------- | ----------------------------------------- | +| `COMPACTION_TRIGGER` | 500 ops | Operations before automatic compaction | +| `COMPACTION_RETENTION_MS` | 7 days | Synced ops older than this may be deleted | +| `EMERGENCY_COMPACTION_RETENTION_MS` | 1 day | Shorter retention for quota exceeded | +| `MAX_COMPACTION_FAILURES` | 3 | Failures before user notification | +| `MAX_DOWNLOAD_OPS_IN_MEMORY` | 50,000 | Bounds memory during API download | +| `REMOTE_OP_FILE_RETENTION_MS` | 14 days | Server-side operation file retention | ## 7. Quick Reference Checklist diff --git a/docs/sync-and-op-log/supersync-scenarios.md b/docs/sync-and-op-log/supersync-scenarios.md index 5bb3d45ebf..7b7340b3c4 100644 --- a/docs/sync-and-op-log/supersync-scenarios.md +++ b/docs/sync-and-op-log/supersync-scenarios.md @@ -154,7 +154,8 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat 3. Throw `LocalDataConflictError` 4. Show full conflict dialog: USE_LOCAL / USE_REMOTE / CANCEL 5. USE_LOCAL → `forceUploadLocalState()` (creates SYNC_IMPORT) -6. USE_REMOTE → `forceDownloadRemoteState()` (clears local ops) +6. USE_REMOTE → capture a single-slot pre-replace backup, then `forceDownloadRemoteState()` (clears local ops) +7. The replacement transaction sets a raw-rebuild resume marker. Completion atomically replaces it with a durable Undo provenance token; startup re-offers Undo after reload while the same backup remains. **User sees:** Full conflict resolution dialog. @@ -201,10 +202,10 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat 2. Check pending local ops. Any pending work triggers the dialog except onboarding example-task creates and the never-synced `GLOBAL_CONFIG:sync` provider-setup write. Other GLOBAL_CONFIG sections remain protected. 3. **Show conflict dialog BEFORE processing** with `scenario: 'INCOMING_IMPORT'` and `syncImportReason` 4. USE_LOCAL → `forceUploadLocalState()` (overrides remote with local data) -5. USE_REMOTE → `forceDownloadRemoteState()` (clears local ops, downloads from seq 0) +5. USE_REMOTE → capture a pre-replace backup, then `forceDownloadRemoteState()` (clears local ops, downloads from seq 0) 6. CANCEL → return with `cancelled: true`, skip upload phase -**User sees:** Conflict dialog explaining remote import detected with local changes at risk. "Use Server Data" recommended. +**User sees:** Conflict dialog explaining remote import detected with local changes at risk. "Use Server Data" recommended. After replacement, a persistent Undo action remains recoverable across reloads while its matching backup exists. ### D.3: Remote Ops Filtered by Stored Local SYNC_IMPORT ✓ diff --git a/docs/wiki/3.06-User-Data.md b/docs/wiki/3.06-User-Data.md index 1baea56684..a51620d446 100644 --- a/docs/wiki/3.06-User-Data.md +++ b/docs/wiki/3.06-User-Data.md @@ -77,7 +77,7 @@ The Windows Store build uses a different path that includes the Windows Store pa ### IndexedDB Databases -- **SUP_OPS** (current, version 4): Main database. Object stores: `ops`, `state_cache`, `import_backup`, `vector_clock`, `archive_young`, `archive_old`. Holds operations log, state snapshots, vector clocks, and archives. +- **SUP_OPS** (current, version 8): Main database. Object stores: `ops`, `state_cache`, `import_backup`, `vector_clock`, `archive_young`, `archive_old`. Holds operations log, state snapshots, vector clocks, and archives. - **pf:** Legacy database used for migration and recovery. - **SUPPluginCache:** Plugin cache. @@ -107,7 +107,7 @@ On mobile (Android/iOS), if the app launches with no local data but a usable on- **Manual backups:** (1) **Create manual backup** in Settings → Sync & Export creates a backup using the same mechanism as automatic backups (platform-dependent location). (2) **Safety backups** are created automatically before certain sync operations; the app keeps a limited number of slots (e.g. two most recent, one first from today, one first from day before). (3) **Export data** downloads a complete backup JSON file to a path you choose (or the browser download folder on web). -**What is included:** All application data is included in backups and exports; nothing is excluded. When archives are included (e.g. when using "Export data" with full backup), both `archive_young` and `archive_old` are part of the file. A separate, single-slot **import backup** is stored locally in IndexedDB (`import_backup`). The app captures it before replacing local state through a file import or the sync conflict action **Use Server Data**. A successful Use Server Data replacement shows a persistent **Undo** action that restores this backup; interrupted replacements offer the same recovery action when sync resumes. The slot is internal recovery state and is not included in exports or the user-visible automatic/manual backup lists. +**What is included:** All application data is included in backups and exports; nothing is excluded. When archives are included (e.g. when using "Export data" with full backup), both `archive_young` and `archive_old` are part of the file. A separate, single-slot **import backup** is stored locally in IndexedDB (`import_backup`). The app captures it before replacing local state through a file import or the sync conflict action **Use Server Data**. A successful Use Server Data replacement shows a persistent **Undo** action that restores this backup; interrupted replacements offer the same recovery action when sync resumes. The Undo remains available after an app reload while that same backup is still present. The slot is internal recovery state and is not included in exports or the user-visible automatic/manual backup lists. ## Import and Export @@ -176,7 +176,7 @@ On mobile (Android/iOS), if the app launches with no local data but a usable on- ## Configuration and Versioning - There are no separate config files in the User Data Folder. Configuration is stored in IndexedDB (`SUP_OPS`, `state_cache`) and in localStorage (`SUP_*` and related keys). -- **Database version:** `SUP_OPS` is at schema version 4. +- **Database version:** `SUP_OPS` is at schema version 8. - **Application version:** Defined in the build (e.g. `src/environments/versions.ts`), not stored in the User Data Folder. - **Backup files:** Each JSON file carries full state; desktop automatic backup naming includes date and time. From 9d18d02f961aa11b7859bcdfad4e18649613613e Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:14:27 +0200 Subject: [PATCH 41/47] test(sync): spy the locked-internal archive paths, not the public wrappers The lock-serialization refactor routes internal archive calls through _updateTasks/_deleteTasks so a held sp_task_archive lock is never re-acquired; five assertions still spied the public wrappers and never fired. --- .../archive/task-archive.service.spec.ts | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/app/features/archive/task-archive.service.spec.ts b/src/app/features/archive/task-archive.service.spec.ts index 9eabd5c4c2..91ab523001 100644 --- a/src/app/features/archive/task-archive.service.spec.ts +++ b/src/app/features/archive/task-archive.service.spec.ts @@ -652,17 +652,17 @@ describe('TaskArchiveService', () => { ); archiveDbAdapterMock.loadArchiveOld.and.returnValue(Promise.resolve(oldArchive)); - // Mock the updateTasks method to track calls - spyOn(service, 'updateTasks').and.returnValue(Promise.resolve()); + // Track the internal (already-locked) update path + spyOn(service as any, '_updateTasks').and.returnValue(Promise.resolve()); await service.removeRepeatCfgFromArchiveTasks('repeat1'); - expect(service.updateTasks).toHaveBeenCalledWith( + expect((service as any)._updateTasks).toHaveBeenCalledWith( [ { id: 'task1', changes: { repeatCfgId: undefined } }, { id: 'task2', changes: { repeatCfgId: undefined } }, ], - { isSkipDispatch: true, isIgnoreDBLock: undefined }, + { isSkipDispatch: true }, ); }); @@ -677,11 +677,11 @@ describe('TaskArchiveService', () => { Promise.resolve(createMockArchiveModel([])), ); - spyOn(service, 'updateTasks').and.returnValue(Promise.resolve()); + spyOn(service as any, '_updateTasks').and.returnValue(Promise.resolve()); await service.removeRepeatCfgFromArchiveTasks('repeat1'); - expect(service.updateTasks).not.toHaveBeenCalled(); + expect((service as any)._updateTasks).not.toHaveBeenCalled(); }); }); @@ -711,11 +711,11 @@ describe('TaskArchiveService', () => { ); archiveDbAdapterMock.loadArchiveOld.and.returnValue(Promise.resolve(oldArchive)); - spyOn(service, 'updateTasks').and.returnValue(Promise.resolve()); + spyOn(service as any, '_updateTasks').and.returnValue(Promise.resolve()); await service.unlinkIssueProviderFromArchiveTasks('provider1'); - expect(service.updateTasks).toHaveBeenCalledWith( + expect((service as any)._updateTasks).toHaveBeenCalledWith( [ { id: 'task1', @@ -744,7 +744,7 @@ describe('TaskArchiveService', () => { }, }, ], - { isSkipDispatch: true, isIgnoreDBLock: undefined }, + { isSkipDispatch: true }, ); }); @@ -759,11 +759,11 @@ describe('TaskArchiveService', () => { Promise.resolve(createMockArchiveModel([])), ); - spyOn(service, 'updateTasks').and.returnValue(Promise.resolve()); + spyOn(service as any, '_updateTasks').and.returnValue(Promise.resolve()); await service.unlinkIssueProviderFromArchiveTasks('provider1'); - expect(service.updateTasks).not.toHaveBeenCalled(); + expect((service as any)._updateTasks).not.toHaveBeenCalled(); }); }); @@ -785,11 +785,11 @@ describe('TaskArchiveService', () => { }; spyOn(service, 'load').and.returnValue(Promise.resolve(mockArchive)); - spyOn(service, 'deleteTasks').and.returnValue(Promise.resolve()); + spyOn(service as any, '_deleteTasks').and.returnValue(Promise.resolve()); await service.removeAllArchiveTasksForProject('project1'); - expect(service.deleteTasks).toHaveBeenCalledWith(['task1', 'task2'], undefined); + expect((service as any)._deleteTasks).toHaveBeenCalledWith(['task1', 'task2']); }); }); @@ -899,8 +899,8 @@ describe('TaskArchiveService', () => { ); archiveDbAdapterMock.loadArchiveOld.and.returnValue(Promise.resolve(oldArchive)); - // Spy on deleteTasks to verify it's called with the right parameters - spyOn(service, 'deleteTasks').and.returnValue(Promise.resolve()); + // Spy on the internal (already-locked) delete path + spyOn(service as any, '_deleteTasks').and.returnValue(Promise.resolve()); await service.removeTagsFromAllTasks(['tag1']); @@ -909,10 +909,11 @@ describe('TaskArchiveService', () => { // parent1 has no tags and no project after removal, so it and its subtasks are orphaned // task1 still has other tags, so it's not orphaned // task3 has a project, so it's not orphaned - expect(service.deleteTasks).toHaveBeenCalledWith( - ['task2', 'parent1', 'sub1'], - undefined, - ); + expect((service as any)._deleteTasks).toHaveBeenCalledWith([ + 'task2', + 'parent1', + 'sub1', + ]); }); }); From fe544b1f796e3a57f20264416130d122b9d4edef Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:55:51 +0200 Subject: [PATCH 42/47] test(sync): mock the mixed-source batch port in the disjoint-merge spec The #8874 spec predates the atomic mixed-source batch and reducer checkpoint on the store port; its two createSpyObj sites lacked the methods, so every flow through the local-wins write path threw. --- ...conflict-resolution.disjoint-merge.spec.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts b/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts index 884513b3cb..5b0f20e2e1 100644 --- a/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts +++ b/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts @@ -82,14 +82,27 @@ describe('ConflictResolutionService — SPAP-14 disjoint-field merge', () => { mockOpLogStore = jasmine.createSpyObj('OperationLogStoreService', [ 'appendBatchSkipDuplicates', + 'appendMixedSourceBatchSkipDuplicates', 'appendWithVectorClockUpdate', 'markApplied', 'markRejected', 'markFailed', 'getUnsyncedByEntity', 'mergeRemoteOpClocks', + 'markReducersCommittedAndMergeClocks', ]); mockOpLogStore.mergeRemoteOpClocks.and.resolveTo(undefined); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.resolveTo(undefined); + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.and.callFake(async (batches) => ({ + written: batches.flatMap((batch) => + batch.ops.map((batchOp, index) => ({ + seq: index + 1, + op: batchOp, + source: batch.source, + })), + ), + skippedCount: 0, + })); mockOpLogStore.getUnsyncedByEntity.and.resolveTo(new Map()); mockOpLogStore.markRejected.and.resolveTo(undefined); mockOpLogStore.markApplied.and.resolveTo(undefined); @@ -634,14 +647,27 @@ describe('ConflictResolutionService — SPAP-14 disjoint-field merge', () => { const opLogStore = jasmine.createSpyObj('OperationLogStoreService', [ 'appendBatchSkipDuplicates', + 'appendMixedSourceBatchSkipDuplicates', 'appendWithVectorClockUpdate', 'markApplied', 'markRejected', 'markFailed', 'getUnsyncedByEntity', 'mergeRemoteOpClocks', + 'markReducersCommittedAndMergeClocks', ]); opLogStore.mergeRemoteOpClocks.and.resolveTo(undefined); + opLogStore.markReducersCommittedAndMergeClocks.and.resolveTo(undefined); + opLogStore.appendMixedSourceBatchSkipDuplicates.and.callFake(async (batches) => ({ + written: batches.flatMap((batch) => + batch.ops.map((batchOp, index) => ({ + seq: index + 1, + op: batchOp, + source: batch.source, + })), + ), + skippedCount: 0, + })); opLogStore.getUnsyncedByEntity.and.resolveTo(new Map()); opLogStore.markRejected.and.resolveTo(undefined); opLogStore.markApplied.and.resolveTo(undefined); From a026b2017f7caf88cc6ad9b3a505c9104f072e69 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 18:59:27 +0200 Subject: [PATCH 43/47] test(sync): mock the mixed-source batch port in the journal-hook spec Same #8874-vintage store mock as the disjoint-merge spec: without the atomic mixed-source batch method every local-wins flow through autoResolveConflictsLWW threw before journaling. --- .../sync/conflict-journal-hook.integration.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/app/op-log/sync/conflict-journal-hook.integration.spec.ts b/src/app/op-log/sync/conflict-journal-hook.integration.spec.ts index 0984abe60b..e249f23db7 100644 --- a/src/app/op-log/sync/conflict-journal-hook.integration.spec.ts +++ b/src/app/op-log/sync/conflict-journal-hook.integration.spec.ts @@ -81,14 +81,27 @@ describe('ConflictResolution → ConflictJournal hook (integration)', () => { mockOpLogStore = jasmine.createSpyObj('OperationLogStoreService', [ 'appendBatchSkipDuplicates', + 'appendMixedSourceBatchSkipDuplicates', 'appendWithVectorClockUpdate', 'markApplied', 'markRejected', 'markFailed', 'getUnsyncedByEntity', 'mergeRemoteOpClocks', + 'markReducersCommittedAndMergeClocks', ]); mockOpLogStore.mergeRemoteOpClocks.and.resolveTo(undefined); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.resolveTo(undefined); + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.and.callFake(async (batches) => ({ + written: batches.flatMap((batch) => + batch.ops.map((batchOp, index) => ({ + seq: index + 1, + op: batchOp, + source: batch.source, + })), + ), + skippedCount: 0, + })); mockOpLogStore.getUnsyncedByEntity.and.resolveTo(new Map()); mockOpLogStore.appendWithVectorClockUpdate.and.resolveTo(undefined); mockOpLogStore.markRejected.and.resolveTo(undefined); From f572b6d077528e4a9b986cb4c3313858f50f6e69 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 20:14:35 +0200 Subject: [PATCH 44/47] fix(sync): persist disjoint merges atomically and exempt them from checkpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the #8874 x #8900 merge seam found two defects in STEP 3b: - The synthesized merged op rode in the apply batch, so the reducer checkpoint's pending-only assertion threw on it: every real disjoint auto-merge aborted the checkpoint transaction and wedged sync behind IncompleteRemoteOperationsError until app restart. Checkpoint now covers only pending-appended remote rows; synthetic local ops are exempt (their durability contract is the append + upload path). - appendWithVectorClockUpdate replaced the durable vector clock with a clock computed only from the conflict's own ops, regressing the client counter and enabling silent cross-device op drops. Merge writes now go through appendMixedSourceBatchSkipDuplicates, which rebases the merged op on the durable clock in the same transaction — also closing the crash window between the remote originals and their superseding merged op, and turning duplicate re-appends into skips instead of ConstraintErrors. Two regression tests enforce the coordinator's whole-batch reducer- commit contract and the pending-only checkpoint against the resolution flow; the journal keeps recording merges only after a durable append. --- ...conflict-resolution.disjoint-merge.spec.ts | 120 +++++++++++++++++- .../sync/conflict-resolution.service.ts | 87 +++++++++---- 2 files changed, 179 insertions(+), 28 deletions(-) diff --git a/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts b/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts index 5b0f20e2e1..026678bcf2 100644 --- a/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts +++ b/src/app/op-log/sync/conflict-resolution.disjoint-merge.spec.ts @@ -66,9 +66,11 @@ describe('ConflictResolutionService — SPAP-14 disjoint-field merge', () => { }); const mergedOpArgs = (): Operation | undefined => - mockOpLogStore.appendWithVectorClockUpdate.calls + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.calls .allArgs() - .map(([o]) => o as Operation) + .flatMap(([batches]) => batches) + .filter((batch) => batch.source === 'local') + .flatMap((batch) => [...batch.ops]) .find((o) => o.entityId === 'task-1' && o.opType === OpType.Update); beforeEach(() => { @@ -149,6 +151,110 @@ describe('ConflictResolutionService — SPAP-14 disjoint-field merge', () => { journal = TestBed.inject(ConflictJournalService); }); + // ── regression: checkpoint contract vs synthetic merged ops (#8900 seam) ─── + it('resolves without checkpointing the synthetic merged op when the applier reports reducer commit', async () => { + mockStore.select.and.returnValue( + of({ id: 'task-1', title: 'Local title', notes: 'base notes' }), + ); + // Honor the coordinator contract: the reducer-commit callback receives the + // ENTIRE apply batch (including the synthetic merged local op). + mockOperationApplier.applyOperations.and.callFake(async (ops, options) => { + await options?.onReducersCommitted?.(ops); + return { appliedOps: ops }; + }); + // Enforce the real store's pending-only checkpoint assertion: only rows + // appended with pendingApply may be checkpointed. + const pendingAppendedIds = new Set(); + mockOpLogStore.appendBatchSkipDuplicates.and.callFake( + (ops: Operation[], _source, options) => { + if (options?.pendingApply) { + ops.forEach((o) => pendingAppendedIds.add(o.id)); + } + return Promise.resolve({ + seqs: ops.map((_, i) => i + 1), + writtenOps: ops, + skippedCount: 0, + }); + }, + ); + mockOpLogStore.markReducersCommittedAndMergeClocks.and.callFake( + async (_seqs, ops) => { + for (const o of ops) { + if (!pendingAppendedIds.has(o.id)) { + throw new Error( + `Reducer checkpoint requires pending remote operation (${o.id}).`, + ); + } + } + }, + ); + + const localOp = op({ + id: 'local-cp', + clientId: 'A', + vectorClock: { A: 1 }, + timestamp: 2000, + payload: { task: { id: 'task-1', changes: { title: 'Local title' } } }, + }); + const remoteOp = op({ + id: 'remote-cp', + clientId: 'B', + vectorClock: { B: 1 }, + timestamp: 1000, + payload: { task: { id: 'task-1', changes: { notes: 'Remote notes' } } }, + }); + + await expectAsync( + service.autoResolveConflictsLWW([conflictOf([localOp], [remoteOp])]), + ).toBeResolved(); + + // The merged op reached the reducers… + const appliedOps = mockOperationApplier.applyOperations.calls.mostRecent() + .args[0] as Operation[]; + expect( + appliedOps.some((o) => o.opType === OpType.Update && o.entityId === 'task-1'), + ).toBeTrue(); + // …but only pending-appended rows were ever checkpointed. + const checkpointedOps = mockOpLogStore.markReducersCommittedAndMergeClocks.calls + .allArgs() + .flatMap(([, ops]) => ops); + expect(checkpointedOps.every((o) => pendingAppendedIds.has(o.id))).toBeTrue(); + }); + + it('persists merge writes through the atomic mixed-source batch, never the clock-overwriting append', async () => { + mockStore.select.and.returnValue( + of({ id: 'task-1', title: 'Local title', notes: 'base notes' }), + ); + + const localOp = op({ + id: 'local-mb', + clientId: 'A', + vectorClock: { A: 1 }, + timestamp: 2000, + payload: { task: { id: 'task-1', changes: { title: 'Local title' } } }, + }); + const remoteOp = op({ + id: 'remote-mb', + clientId: 'B', + vectorClock: { B: 1 }, + timestamp: 1000, + payload: { task: { id: 'task-1', changes: { notes: 'Remote notes' } } }, + }); + + await service.autoResolveConflictsLWW([conflictOf([localOp], [remoteOp])]); + + // appendWithVectorClockUpdate REPLACES the durable clock with the caller's + // clock (built only from the conflict's ops) — the batch rebases instead. + expect(mockOpLogStore.appendWithVectorClockUpdate).not.toHaveBeenCalled(); + const batches = + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.calls.mostRecent().args[0]; + const remoteBatch = batches.find((b) => b.source === 'remote'); + const localBatch = batches.find((b) => b.source === 'local'); + expect(remoteBatch!.ops.map((o) => o.id)).toEqual(['remote-mb']); + expect(localBatch!.ops.length).toBe(1); + expect(localBatch!.ops[0].opType).toBe(OpType.Update); + }); + // ── (a) title vs notes → merge both ──────────────────────────────────────── it('(a) merges concurrent title-vs-notes edits into one op keeping BOTH', async () => { mockStore.select.and.returnValue( @@ -367,7 +473,9 @@ describe('ConflictResolutionService — SPAP-14 disjoint-field merge', () => { mockStore.select.and.returnValue( of({ id: 'task-1', title: 'Local title', notes: 'base' }), ); - mockOpLogStore.appendWithVectorClockUpdate.and.rejectWith(new Error('append failed')); + mockOpLogStore.appendMixedSourceBatchSkipDuplicates.and.rejectWith( + new Error('append failed'), + ); const localOp = op({ id: 'local-1', @@ -714,9 +822,11 @@ describe('ConflictResolutionService — SPAP-14 disjoint-field merge', () => { const svc = TestBed.inject(ConflictResolutionService); await svc.autoResolveConflictsLWW([conflict]); - const synthesized = opLogStore.appendWithVectorClockUpdate.calls + const synthesized = opLogStore.appendMixedSourceBatchSkipDuplicates.calls .allArgs() - .map(([o]) => o as Operation) + .flatMap(([batches]) => batches) + .filter((batch) => batch.source === 'local') + .flatMap((batch) => [...batch.ops]) .find((o) => o.entityId === 'task-1' && o.opType === OpType.Update); return { synthesized }; }; diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index aa67eaf828..ea3df18780 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -343,6 +343,9 @@ export class ConflictResolutionService { const allOpsToApply: Operation[] = []; const allStoredOps: Array<{ id: string; seq: number }> = []; + // Synthetic local ops (disjoint merges) ride in the apply batch but are NOT + // pending remote rows — the reducer-commit checkpoint must never see them. + const checkpointExemptOpIds = new Set(); const lwwPartitions = partitionLwwResolutions( resolutions, @@ -474,32 +477,63 @@ export class ConflictResolutionService { // client's state picks up the remote side's fields — local's are already // optimistically applied). The op stays unsynced+not-rejected → it uploads. // ───────────────────────────────────────────────────────────────────────── - for (const merged of mergedResolutions) { - for (const op of merged.conflict.localOps) { - if (!localOpsToRejectSet.has(op.id)) { - localOpsToReject.push(op.id); - localOpsToRejectSet.add(op.id); + if (mergedResolutions.length > 0) { + for (const merged of mergedResolutions) { + for (const op of merged.conflict.localOps) { + if (!localOpsToRejectSet.has(op.id)) { + localOpsToReject.push(op.id); + localOpsToRejectSet.add(op.id); + } } - } - if (merged.conflict.remoteOps.length > 0) { - await this._filterAndAppendOpsWithRetry(merged.conflict.remoteOps, 'remote'); remoteOpsToReject.push(...merged.conflict.remoteOps.map((op) => op.id)); } - const seq = await this.opLogStore.appendWithVectorClockUpdate( - merged.mergedOp, - 'local', - ); - allStoredOps.push({ id: merged.mergedOp.id, seq }); - allOpsToApply.push(merged.mergedOp); + // ONE atomic mixed-source batch for all merge writes: an original remote + // loser must never be durable without its superseding merged op (crash + // safety), and the batch rebases each merged op on the durable clock so a + // synthetic op cannot reuse or regress this client's counter. The rebased + // clock still dominates both original sides. + const mergeBatch = await this.opLogStore.appendMixedSourceBatchSkipDuplicates([ + { + ops: mergedResolutions.flatMap((merged) => merged.conflict.remoteOps), + source: 'remote', + }, + { + ops: mergedResolutions.map((merged) => merged.mergedOp), + source: 'local', + }, + ]); + if (mergeBatch.skippedCount > 0) { + OpLog.verbose( + `ConflictResolutionService: Skipped ${mergeBatch.skippedCount} duplicate merge-resolution op(s)`, + ); + } + + const writtenMergedOpIds = new Set(); + for (const entry of mergeBatch.written) { + if (entry.source !== 'local') { + continue; + } + // Apply/upload the WRITTEN op — it carries the rebased vector clock. + allStoredOps.push({ id: entry.op.id, seq: entry.seq }); + allOpsToApply.push(entry.op); + checkpointExemptOpIds.add(entry.op.id); + writtenMergedOpIds.add(entry.op.id); + OpLog.normal( + `ConflictResolutionService: Appended disjoint-merge op ${entry.op.id} for ` + + `${entry.op.entityType}:${entry.op.entityId}`, + ); + } + // Journal ONLY after the append: once persisted as a pending local op the // merge is durable (it applies/uploads even across a crash), so a `merged` - // ("kept both") entry can never describe a merge that didn't happen. - await this._journalMergedResolution(merged.plan); - OpLog.normal( - `ConflictResolutionService: Appended disjoint-merge op ${merged.mergedOp.id} for ` + - `${merged.mergedOp.entityType}:${merged.mergedOp.entityId}`, - ); + // ("kept both") entry can never describe a merge that didn't happen. A + // skipped (already-persisted) merged op was journaled when first written. + for (const merged of mergedResolutions) { + if (writtenMergedOpIds.has(merged.mergedOp.id)) { + await this._journalMergedResolution(merged.plan); + } + } } // ───────────────────────────────────────────────────────────────────────── @@ -539,17 +573,24 @@ export class ConflictResolutionService { const applyResult = await this.operationApplier.applyOperations(allOpsToApply, { skipDeferredLocalActions: true, onReducersCommitted: async (reducerCommittedOps) => { - const reducerCommittedSeqs = reducerCommittedOps + // Disjoint-merge ops are synthetic LOCAL rows in the apply batch; + // their durability contract is the mixed-source append + upload + // path. The checkpoint's pending-only assertion must only see rows + // appended with pendingApply. + const checkpointOps = reducerCommittedOps.filter( + (op) => !checkpointExemptOpIds.has(op.id), + ); + const reducerCommittedSeqs = checkpointOps .map((op) => opIdToSeq.get(op.id)) .filter((seq): seq is number => seq !== undefined); - if (reducerCommittedSeqs.length !== reducerCommittedOps.length) { + if (reducerCommittedSeqs.length !== checkpointOps.length) { throw new Error( 'ConflictResolutionService: reducer commit contained an unknown operation.', ); } await this.opLogStore.markReducersCommittedAndMergeClocks( reducerCommittedSeqs, - reducerCommittedOps, + checkpointOps, ); }, }); From 867d84a3d1c931fb9df0b6fe0fb2a16499a8399a Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 20:21:00 +0200 Subject: [PATCH 45/47] fix(sync): serialize remote archive side effects behind the archive mutex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isIgnoreDBLock historically meant "sync already holds the op-log DB lock", but _runTaskArchiveMutation also treated it as permission to skip the new TASK_ARCHIVE mutex — so every remote archive side effect except moveToArchive ran unserialized against locked local mutations, and a concurrent read-modify-write could silently drop one side's archive write (the exact race the mutex was added to close). TASK_ARCHIVE is deliberately separate from OPERATION_LOG, so acquiring it while sync holds the op-log lock is safe — the remote moveToArchive path has always done exactly that. The mutex is now unconditional, and the remote flushYoungToOld handler wraps its two-archive read-modify- write in the same lock instead of writing through the adapter bare. The isIgnoreDBLock option is retained as inert API surface; removing the threading is tracked as a follow-up. --- .../features/archive/task-archive.service.ts | 53 +++++++-------- .../archive-operation-handler.service.ts | 66 +++++++++++-------- 2 files changed, 61 insertions(+), 58 deletions(-) diff --git a/src/app/features/archive/task-archive.service.ts b/src/app/features/archive/task-archive.service.ts index 1ae352eca3..6d2cad4dbc 100644 --- a/src/app/features/archive/task-archive.service.ts +++ b/src/app/features/archive/task-archive.service.ts @@ -109,13 +109,17 @@ export class TaskArchiveService { constructor() {} - private _runTaskArchiveMutation( - mutation: () => Promise, - isIgnoreDBLock: boolean = false, - ): Promise { - return isIgnoreDBLock - ? mutation() - : this._lockService.request(LOCK_NAMES.TASK_ARCHIVE, mutation); + /** + * Every archive mutation is serialized behind the cross-tab TASK_ARCHIVE + * lock — including remote sync side effects: TASK_ARCHIVE is deliberately a + * separate lock from OPERATION_LOG, so acquiring it while sync holds the + * op-log lock is safe (and the remote moveToArchive path already does). + * The legacy `isIgnoreDBLock` option no longer bypasses this mutex; a + * bypassed remote mutation could interleave with a locked local one and + * silently drop one side's archive write. + */ + private _runTaskArchiveMutation(mutation: () => Promise): Promise { + return this._lockService.request(LOCK_NAMES.TASK_ARCHIVE, mutation); } async loadYoung(): Promise { @@ -246,10 +250,7 @@ export class TaskArchiveService { taskIdsToDelete: string[], options?: { isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._deleteTasks(taskIdsToDelete), - options?.isIgnoreDBLock, - ); + return this._runTaskArchiveMutation(() => this._deleteTasks(taskIdsToDelete)); } private async _deleteTasks(taskIdsToDelete: string[]): Promise { @@ -296,9 +297,8 @@ export class TaskArchiveService { changedFields: Partial, options?: { isSkipDispatch?: boolean; isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._updateTask(id, changedFields, options), - options?.isIgnoreDBLock, + return this._runTaskArchiveMutation(() => + this._updateTask(id, changedFields, options), ); } @@ -345,10 +345,7 @@ export class TaskArchiveService { updates: Update[], options?: { isSkipDispatch?: boolean; isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._updateTasks(updates, options), - options?.isIgnoreDBLock, - ); + return this._runTaskArchiveMutation(() => this._updateTasks(updates, options)); } private async _updateTasks( @@ -408,9 +405,8 @@ export class TaskArchiveService { projectIdToDelete: string, options?: { isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._removeAllArchiveTasksForProject(projectIdToDelete), - options?.isIgnoreDBLock, + return this._runTaskArchiveMutation(() => + this._removeAllArchiveTasksForProject(projectIdToDelete), ); } @@ -434,9 +430,8 @@ export class TaskArchiveService { tagIdsToRemove: string[], options?: { isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._removeTagsFromAllTasks(tagIdsToRemove), - options?.isIgnoreDBLock, + return this._runTaskArchiveMutation(() => + this._removeTagsFromAllTasks(tagIdsToRemove), ); } @@ -472,9 +467,8 @@ export class TaskArchiveService { repeatConfigId: string, options?: { isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._removeRepeatCfgFromArchiveTasks(repeatConfigId), - options?.isIgnoreDBLock, + return this._runTaskArchiveMutation(() => + this._removeRepeatCfgFromArchiveTasks(repeatConfigId), ); } @@ -508,9 +502,8 @@ export class TaskArchiveService { issueProviderId: string, options?: { isIgnoreDBLock?: boolean }, ): Promise { - return this._runTaskArchiveMutation( - () => this._unlinkIssueProviderFromArchiveTasks(issueProviderId), - options?.isIgnoreDBLock, + return this._runTaskArchiveMutation(() => + this._unlinkIssueProviderFromArchiveTasks(issueProviderId), ); } diff --git a/src/app/op-log/apply/archive-operation-handler.service.ts b/src/app/op-log/apply/archive-operation-handler.service.ts index b7c98ac393..2ff5d14421 100644 --- a/src/app/op-log/apply/archive-operation-handler.service.ts +++ b/src/app/op-log/apply/archive-operation-handler.service.ts @@ -22,6 +22,8 @@ import { loadAllData } from '../../root-store/meta/load-all-data.action'; import { ArchiveModel } from '../../features/archive/archive.model'; import { ArchiveDbAdapter } from '../../core/persistence/archive-db-adapter.service'; import { OpType } from '../core/operation.types'; +import { LockService } from '../sync/lock.service'; +import { LOCK_NAMES } from '../core/operation-log.const'; import { confirmDialog } from '../../util/native-dialogs'; /** @@ -104,7 +106,7 @@ export const isArchiveAffectingAction = (action: Action): action is PersistentAc * * ## Important Notes * - * - For remote operations, uses `isIgnoreDBLock: true` because sync processing has the DB locked + * - Remote archive mutations serialize behind the TASK_ARCHIVE mutex (separate from the OPERATION_LOG lock sync holds) * - All operations are idempotent - safe to run multiple times * - Use `isArchiveAffectingAction()` helper to check if an action needs archive handling */ @@ -128,6 +130,7 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort { const task = (action as ReturnType).task; @@ -313,27 +317,33 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort).timestamp; - // Load current state using ArchiveDbAdapter - // Default to empty archives if they don't exist (first-time usage) - const currentArchiveYoung = - (await this._archiveDbAdapter.loadArchiveYoung()) ?? createEmptyArchiveModel(); - const currentArchiveOld = - (await this._archiveDbAdapter.loadArchiveOld()) ?? createEmptyArchiveModel(); + // Serialize against local archive mutations: this is a read-modify-write + // over BOTH archives, so an unlocked run could silently drop a concurrent + // locked local write. TASK_ARCHIVE is separate from OPERATION_LOG, so + // acquiring it while sync holds the op-log lock is safe. + await this._lockService.request(LOCK_NAMES.TASK_ARCHIVE, async () => { + // Load current state using ArchiveDbAdapter + // Default to empty archives if they don't exist (first-time usage) + const currentArchiveYoung = + (await this._archiveDbAdapter.loadArchiveYoung()) ?? createEmptyArchiveModel(); + const currentArchiveOld = + (await this._archiveDbAdapter.loadArchiveOld()) ?? createEmptyArchiveModel(); - const newSorted = sortTimeTrackingAndTasksFromArchiveYoungToOld({ - archiveYoung: currentArchiveYoung, - archiveOld: currentArchiveOld, - threshold: ARCHIVE_TASK_YOUNG_TO_OLD_THRESHOLD, - now: timestamp, + const newSorted = sortTimeTrackingAndTasksFromArchiveYoungToOld({ + archiveYoung: currentArchiveYoung, + archiveOld: currentArchiveOld, + threshold: ARCHIVE_TASK_YOUNG_TO_OLD_THRESHOLD, + now: timestamp, + }); + + // Atomic write: both archives written in a single IndexedDB transaction. + // If either write fails, the entire transaction is rolled back automatically. + await this._archiveDbAdapter.saveArchivesAtomic( + { ...newSorted.archiveYoung, lastTimeTrackingFlush: timestamp }, + { ...newSorted.archiveOld, lastTimeTrackingFlush: timestamp }, + ); }); - // Atomic write: both archives written in a single IndexedDB transaction. - // If either write fails, the entire transaction is rolled back automatically. - await this._archiveDbAdapter.saveArchivesAtomic( - { ...newSorted.archiveYoung, lastTimeTrackingFlush: timestamp }, - { ...newSorted.archiveOld, lastTimeTrackingFlush: timestamp }, - ); - OpLog.log( '______________________\nFLUSHED ALL FROM ARCHIVE YOUNG TO OLD (via remote op handler)\n_______________________', ); @@ -363,7 +373,7 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort { const projectId = (action as ReturnType) @@ -380,7 +390,7 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort { const tagIdsToRemove = @@ -403,7 +413,7 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort { const repeatCfgId = ( @@ -420,7 +430,7 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort { const issueProviderId = ( @@ -437,7 +447,7 @@ export class ArchiveOperationHandler implements ArchiveSideEffectPort { const ids = (action as ReturnType).ids; From 02c84df821fc710577d9c174d7af37c4b6a3df94 Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 20:25:49 +0200 Subject: [PATCH 46/47] refactor(sync): apply multi-review cleanups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Delete the unreachable stage-2 intra-batch duplicate pass: stage 1 (validateAndClampBatch) reserves every op id including invalid first siblings, so validated candidates are unique by construction. - Delete clearRawRebuildIncomplete: superseded by completeRawRebuild, which retires the marker atomically with the recovery token; only specs still called it. - Clear the conflict journal when a USE_REMOTE rebuild completes — the documented "cleared whenever the full dataset is replaced" contract previously had a single caller (backup import), leaving stale badge counts and review entries describing replaced history. - _notifyResolutionOutcome: drop the win-count parameters left over from the removed count snack and gate on resolutions.length. - Snapshot handler: keep the clean-slate opId invariant local with a defense-in-depth 400 instead of relying on the contract superRefine in another package. - Document that the legacy misc->tasks conflict alias only covers the per-entity path (GLOBAL_CONFIG writes are single-entity today). --- .../super-sync-server/src/sync/conflict.ts | 5 ++ .../sync/services/operation-upload.service.ts | 51 ++----------------- .../src/sync/sync.routes.snapshot-handler.ts | 9 ++++ .../operation-log-store.service.spec.ts | 2 +- .../operation-log-store.service.ts | 6 --- .../sync/conflict-resolution.service.ts | 14 ++--- .../sync/operation-log-sync.service.spec.ts | 12 ++++- .../op-log/sync/operation-log-sync.service.ts | 11 +++- 8 files changed, 43 insertions(+), 67 deletions(-) diff --git a/packages/super-sync-server/src/sync/conflict.ts b/packages/super-sync-server/src/sync/conflict.ts index 6924586f0f..488837f8e2 100644 --- a/packages/super-sync-server/src/sync/conflict.ts +++ b/packages/super-sync-server/src/sync/conflict.ts @@ -227,6 +227,11 @@ export const detectConflictForEntity = async ( // the raw `GLOBAL_CONFIG:misc` key. Consult that key as an alias when the // incoming write targets `tasks`; no backfill (and no payload decryption) is // required. Pick the newer of the canonical and legacy rows. + // + // NOTE: the alias exists only on this per-entity path. A v2 MULTI-entity op + // whose entityIds include 'tasks' goes through detectConflictForEntities and + // skips the legacy lookup — acceptable today because GLOBAL_CONFIG writes are + // single-entity; revisit if a batch path ever carries config entities. const legacyMiscOp = op.entityType === 'GLOBAL_CONFIG' && entityId === 'tasks' ? await tx.operation.findFirst({ diff --git a/packages/super-sync-server/src/sync/services/operation-upload.service.ts b/packages/super-sync-server/src/sync/services/operation-upload.service.ts index 6e366ae0f3..5d923b1784 100644 --- a/packages/super-sync-server/src/sync/services/operation-upload.service.ts +++ b/packages/super-sync-server/src/sync/services/operation-upload.service.ts @@ -260,12 +260,10 @@ export class OperationUploadService { prevalidatedResults, ); - const uniqueCandidates = this.rejectIntraBatchDuplicates( - userId, - clientId, - validatedCandidates, - results, - ); + // Intra-batch duplicate ids were already terminally rejected in stage 1: + // validateAndClampBatch reserves each id on first occurrence (including + // invalid first siblings), so validatedCandidates is unique by op id. + const uniqueCandidates = validatedCandidates; if (uniqueCandidates.length === 0) { return { @@ -397,47 +395,6 @@ export class OperationUploadService { return validatedCandidates; } - /** - * Stage 2: within a single batch, accept the first op for an id. Exact retries - * are DUPLICATE_OPERATION; same-id operations with different complete identity - * are INVALID_OP_ID. Must run before sequence reservation. - */ - private rejectIntraBatchDuplicates( - userId: number, - clientId: string, - validatedCandidates: BatchUploadCandidate[], - results: UploadResult[], - ): BatchUploadCandidate[] { - const firstCandidateByOpId = new Map(); - const uniqueCandidates: BatchUploadCandidate[] = []; - for (const candidate of validatedCandidates) { - const firstCandidate = firstCandidateByOpId.get(candidate.op.id); - if (firstCandidate) { - const isExactRetry = isSameIncomingOperation( - firstCandidate.op, - candidate.op, - firstCandidate.originalTimestamp, - candidate.originalTimestamp, - ); - results[candidate.resultIndex] = this.rejectedUploadResult( - userId, - clientId, - candidate.op, - isExactRetry - ? 'Duplicate operation ID' - : 'Operation ID already belongs to a different operation', - isExactRetry - ? SYNC_ERROR_CODES.DUPLICATE_OPERATION - : SYNC_ERROR_CODES.INVALID_OP_ID, - ); - continue; - } - firstCandidateByOpId.set(candidate.op.id, candidate); - uniqueCandidates.push(candidate); - } - return uniqueCandidates; - } - /** * Stage 3: prefetch any currently persisted ops sharing an incoming id (one * query) and classify each as an idempotent retry (DUPLICATE_OPERATION) or diff --git a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts index d5b718887c..69bb5567d4 100644 --- a/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts +++ b/packages/super-sync-server/src/sync/sync.routes.snapshot-handler.ts @@ -140,6 +140,15 @@ export const uploadSnapshotHandler = async ( } } + // Defense in depth: the contract superRefine already requires opId for + // clean-slate uploads, but the destructive wipe below must never depend on + // a schema in another package staying strict. Keep the invariant local. + if (isCleanSlate && !opId) { + return reply.code(400).send({ + error: 'opId is required for clean-slate snapshot idempotency', + }); + } + // Cheap pre-quota gate BEFORE prepareSnapshotCache so quota-exhausted // clients can't burn CPU on JSON.stringify + zlib.gzipSync. Uses only // the cached counter; if it says we're already at quota, reconcile diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 15e6bb460e..6a7b83ed47 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -2752,7 +2752,7 @@ describe('OperationLogStoreService', () => { // the marker set so the next sync redoes the raw rebuild. expect(await service.isRawRebuildIncomplete()).toBe(true); - await service.clearRawRebuildIncomplete(); + await service.completeRawRebuild(); expect(await service.isRawRebuildIncomplete()).toBe(false); }); diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index 5e2b759ccb..1f57a6d278 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -2006,12 +2006,6 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { - await this._ensureInit(); - await this._adapter.delete(STORE_NAMES.META, RAW_REBUILD_INCOMPLETE_META_KEY); - } - /** * Atomically transitions a raw rebuild from resumable/incomplete to complete. * When a pre-replace backup exists, its provenance token remains durable so diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index ea3df18780..4bee429901 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -359,8 +359,6 @@ export class ConflictResolutionService { ); const { - localWinsCount, - remoteWinsCount, remoteWinsOps, localWinsRemoteOps, remoteOpsToReject, @@ -671,8 +669,8 @@ export class ConflictResolutionService { // existing transient count; genuine content loss gets a dismissible banner // naming the affected task(s) so the user can double-check. (#8694) // ───────────────────────────────────────────────────────────────────────── - if (localWinsCount > 0 || remoteWinsCount > 0) { - await this._notifyResolutionOutcome(resolutions, localWinsCount, remoteWinsCount); + if (resolutions.length > 0) { + await this._notifyResolutionOutcome(resolutions); } // ───────────────────────────────────────────────────────────────────────── @@ -688,7 +686,7 @@ export class ConflictResolutionService { // caller uses this count to trigger the immediate re-upload // (immediate-upload.service.ts) — omitting merges lets a merge-only sync // report IN_SYNC while its merged op sits unsynced until a later cycle. - // Mirrors the rejection-handler path (operation-log-sync.service.ts:361). + // Mirrors the rejection-handler accumulation in operation-log-sync.service. // writtenLocalWinOps (not newLocalWinOps) is the post-dedupe set the atomic // mixed-source batch actually persisted. return { @@ -707,11 +705,7 @@ export class ConflictResolutionService { * Purely a read of the already-decided resolutions — it never influences which * ops were applied or rejected. */ - private async _notifyResolutionOutcome( - resolutions: LWWResolution[], - localWinsCount: number, - remoteWinsCount: number, - ): Promise { + private async _notifyResolutionOutcome(resolutions: LWWResolution[]): Promise { const contentConflicts = findLwwContentConflicts(resolutions, (entityType) => this._resolvePayloadKey(entityType as EntityType), ); diff --git a/src/app/op-log/sync/operation-log-sync.service.spec.ts b/src/app/op-log/sync/operation-log-sync.service.spec.ts index d0373ff6ac..7747c7d827 100644 --- a/src/app/op-log/sync/operation-log-sync.service.spec.ts +++ b/src/app/op-log/sync/operation-log-sync.service.spec.ts @@ -23,6 +23,7 @@ import { SyncImportFilterService } from './sync-import-filter.service'; import { ServerMigrationService } from './server-migration.service'; import { SupersededOperationResolverService } from './superseded-operation-resolver.service'; import { RemoteOpsProcessingService } from './remote-ops-processing.service'; +import { ConflictJournalService } from './conflict-journal.service'; import { RejectedOpsHandlerService } from './rejected-ops-handler.service'; import { OperationWriteFlushService } from './operation-write-flush.service'; import { SuperSyncStatusService } from './super-sync-status.service'; @@ -57,6 +58,7 @@ describe('OperationLogSyncService', () => { let opLogStoreSpy: jasmine.SpyObj; let serverMigrationServiceSpy: jasmine.SpyObj; let remoteOpsProcessingServiceSpy: jasmine.SpyObj; + let conflictJournalServiceSpy: jasmine.SpyObj; let rejectedOpsHandlerServiceSpy: jasmine.SpyObj; let writeFlushServiceSpy: jasmine.SpyObj; let superSyncStatusServiceSpy: jasmine.SpyObj; @@ -115,7 +117,6 @@ describe('OperationLogSyncService', () => { 'runRemoteStateReplacement', 'isRawRebuildIncomplete', 'loadRawRebuildIncomplete', - 'clearRawRebuildIncomplete', 'completeRawRebuild', 'loadRawRebuildRecovery', 'clearRawRebuildRecovery', @@ -138,7 +139,6 @@ describe('OperationLogSyncService', () => { opLogStoreSpy.runRemoteStateReplacement.and.resolveTo(); opLogStoreSpy.isRawRebuildIncomplete.and.resolveTo(false); opLogStoreSpy.loadRawRebuildIncomplete.and.resolveTo(null); - opLogStoreSpy.clearRawRebuildIncomplete.and.resolveTo(); opLogStoreSpy.completeRawRebuild.and.resolveTo(true); opLogStoreSpy.loadRawRebuildRecovery.and.resolveTo(null); opLogStoreSpy.clearRawRebuildRecovery.and.resolveTo(); @@ -191,6 +191,10 @@ describe('OperationLogSyncService', () => { remoteOpsProcessingServiceSpy = jasmine.createSpyObj('RemoteOpsProcessingService', [ 'processRemoteOps', ]); + conflictJournalServiceSpy = jasmine.createSpyObj('ConflictJournalService', [ + 'clearAll', + ]); + conflictJournalServiceSpy.clearAll.and.resolveTo(); remoteOpsProcessingServiceSpy.processRemoteOps.and.resolveTo({ localWinOpsCreated: 0, allOpsFilteredBySyncImport: false, @@ -315,6 +319,10 @@ describe('OperationLogSyncService', () => { ]), }, { provide: RemoteOpsProcessingService, useValue: remoteOpsProcessingServiceSpy }, + { + provide: ConflictJournalService, + useValue: conflictJournalServiceSpy, + }, { provide: RejectedOpsHandlerService, useValue: rejectedOpsHandlerServiceSpy }, { provide: OperationWriteFlushService, useValue: writeFlushServiceSpy }, { provide: SuperSyncStatusService, useValue: superSyncStatusServiceSpy }, diff --git a/src/app/op-log/sync/operation-log-sync.service.ts b/src/app/op-log/sync/operation-log-sync.service.ts index 472901bab3..1d7d8a1ab5 100644 --- a/src/app/op-log/sync/operation-log-sync.service.ts +++ b/src/app/op-log/sync/operation-log-sync.service.ts @@ -33,6 +33,7 @@ import { SuperSyncStatusService } from './super-sync-status.service'; import { ServerMigrationService } from './server-migration.service'; import { OperationWriteFlushService } from './operation-write-flush.service'; import { RemoteOpsProcessingService } from './remote-ops-processing.service'; +import { ConflictJournalService } from './conflict-journal.service'; import { VectorClockService } from './vector-clock.service'; import { DownloadResultForRejection, @@ -157,6 +158,7 @@ export class OperationLogSyncService { // Extracted services private remoteOpsProcessingService = inject(RemoteOpsProcessingService); + private conflictJournalService = inject(ConflictJournalService); private vectorClockService = inject(VectorClockService); private rejectedOpsHandlerService = inject(RejectedOpsHandlerService); private syncHydrationService = inject(SyncHydrationService); @@ -1763,7 +1765,14 @@ export class OperationLogSyncService { private async _completeRawRebuild(backupRef?: ImportBackupRef): Promise { this._assertNoCaptureRacedWithRebuild(); - return this.opLogStore.completeRawRebuild(backupRef); + const hasDurableRecovery = await this.opLogStore.completeRawRebuild(backupRef); + // The conflict journal describes conflicts in the op history that was JUST + // replaced (documented contract: cleared whenever the full dataset is + // replaced — see BackupService.importCompleteBackup). Stale entries would + // keep the badge count and offer review actions against replaced state. + // clearAll swallows its own errors and must not fail the rebuild. + await this.conflictJournalService.clearAll(); + return hasDurableRecovery; } /** From ed0861be79de825016f2a04fa8632ab727de5ffc Mon Sep 17 00:00:00 2001 From: Johannes Millan Date: Sat, 11 Jul 2026 21:05:56 +0200 Subject: [PATCH 47/47] docs(sync): correct two overclaiming comments from the fix review - The archive-mutex comment claimed every archive mutation is locked; compressArchive, the remote loadAllData import and the time-tracking cleanups still write outside it (tracked in #8941). Name them. - The merge-journal comment described an impossible skipped-duplicate path (merged ids are fresh per run); describe the real accepted window instead: durable merge, crash before journaling, entry stays absent (observe-only log). --- .../features/archive/task-archive.service.ts | 19 ++++++++++++------- .../sync/conflict-resolution.service.ts | 7 +++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/app/features/archive/task-archive.service.ts b/src/app/features/archive/task-archive.service.ts index 6d2cad4dbc..e69aa7ccc9 100644 --- a/src/app/features/archive/task-archive.service.ts +++ b/src/app/features/archive/task-archive.service.ts @@ -110,13 +110,18 @@ export class TaskArchiveService { constructor() {} /** - * Every archive mutation is serialized behind the cross-tab TASK_ARCHIVE - * lock — including remote sync side effects: TASK_ARCHIVE is deliberately a - * separate lock from OPERATION_LOG, so acquiring it while sync holds the - * op-log lock is safe (and the remote moveToArchive path already does). - * The legacy `isIgnoreDBLock` option no longer bypasses this mutex; a - * bypassed remote mutation could interleave with a locked local one and - * silently drop one side's archive write. + * Every mutation in THIS service is serialized behind the cross-tab + * TASK_ARCHIVE lock — including remote sync side effects: TASK_ARCHIVE is + * deliberately a separate lock from OPERATION_LOG, so acquiring it while + * sync holds the op-log lock is safe (and the remote moveToArchive path + * already does). The legacy `isIgnoreDBLock` option no longer bypasses this + * mutex; a bypassed remote mutation could interleave with a locked local one + * and silently drop one side's archive write. + * + * Known paths still OUTSIDE the lock (tracked in #8941): + * ArchiveCompressionService.compressArchive, the remote loadAllData archive + * import (holds no lock across its user-confirmation guard by design), and + * TimeTrackingService's project/tag archive cleanups. */ private _runTaskArchiveMutation(mutation: () => Promise): Promise { return this._lockService.request(LOCK_NAMES.TASK_ARCHIVE, mutation); diff --git a/src/app/op-log/sync/conflict-resolution.service.ts b/src/app/op-log/sync/conflict-resolution.service.ts index 4bee429901..3381d92fc3 100644 --- a/src/app/op-log/sync/conflict-resolution.service.ts +++ b/src/app/op-log/sync/conflict-resolution.service.ts @@ -525,8 +525,11 @@ export class ConflictResolutionService { // Journal ONLY after the append: once persisted as a pending local op the // merge is durable (it applies/uploads even across a crash), so a `merged` - // ("kept both") entry can never describe a merge that didn't happen. A - // skipped (already-persisted) merged op was journaled when first written. + // ("kept both") entry can never describe a merge that didn't happen. The + // inverse window is accepted: batch committed but crash before this loop + // means a durable merge without a journal entry (observe-only log; the + // remote originals are recorded-as-seen, so it never re-enters + // resolution and the entry stays absent). for (const merged of mergedResolutions) { if (writtenMergedOpIds.has(merged.mergedOp.id)) { await this._journalMergedResolution(merged.plan);