fix(sync): prevent infinite LWW auto-resolve loop for TAG:TODAY

Three root causes addressed:

1. Pruning asymmetry: Different clients preserve their own clientId
   during vector clock pruning, producing different clock shapes.
   When compared, pruned-away keys default to 0, causing false
   CONCURRENT verdicts and infinite rejection loops. Fix: when both
   clocks are at MAX_VECTOR_CLOCK_SIZE, compare only shared keys
   (intersection) instead of the union.

2. Timing gap: Between endApplyingRemoteOps() and
   startPostSyncCooldown(), isInSyncWindow() returns false, allowing
   selector-based effects to fire and create TAG:TODAY operations.
   Fix: start cooldown BEFORE ending remote ops flag.

3. No retry limit: LWW re-upload had no cap, enabling infinite loops.
   Fix: cap at 3 retries, defer remaining to next sync cycle.

Also moves MAX_VECTOR_CLOCK_SIZE and limitVectorClockSize into
@sp/shared-schema so client and server share the same pruning
constant and algorithm.
This commit is contained in:
Johannes Millan 2026-01-30 18:24:29 +01:00
parent cba5ba3f12
commit cb36c09538
8 changed files with 135 additions and 57 deletions

View file

@ -29,7 +29,12 @@ export { MIGRATIONS } from './migrations/index';
// Vector clock types and comparison (shared between client and server)
export type { VectorClock, VectorClockComparison } from './vector-clock';
export { compareVectorClocks, mergeVectorClocks } from './vector-clock';
export {
compareVectorClocks,
mergeVectorClocks,
limitVectorClockSize,
MAX_VECTOR_CLOCK_SIZE,
} from './vector-clock';
// Entity types (shared between client and server)
export type { EntityType } from './entity-types';

View file

@ -28,12 +28,23 @@ export interface VectorClock {
*/
export type VectorClockComparison = 'EQUAL' | 'LESS_THAN' | 'GREATER_THAN' | 'CONCURRENT';
/**
* Maximum number of entries in a vector clock.
* Shared between client and server to ensure consistent pruning.
*/
export const MAX_VECTOR_CLOCK_SIZE = 10;
/**
* Compare two vector clocks to determine their relationship.
*
* CRITICAL: This algorithm must produce identical results on client and server.
* Both implementations import from this shared module to ensure consistency.
*
* Pruning-aware mode: When both clocks are at MAX_VECTOR_CLOCK_SIZE, they may
* have been pruned by different clients (each preserving its own clientId).
* Missing keys could mean "pruned away" rather than "genuinely zero". Comparing
* only shared keys avoids false CONCURRENT from cross-client pruning asymmetry.
*
* @param a First vector clock
* @param b Second vector clock
* @returns The comparison result
@ -42,12 +53,28 @@ export const compareVectorClocks = (
a: VectorClock,
b: VectorClock,
): VectorClockComparison => {
const allKeys = new Set([...Object.keys(a), ...Object.keys(b)]);
const aKeys = Object.keys(a);
const bKeys = Object.keys(b);
// When both clocks are at MAX_VECTOR_CLOCK_SIZE, they may have been pruned
// by different clients (each preserving its own clientId). Missing keys could
// mean "pruned away" rather than "genuinely zero". Comparing only shared keys
// avoids false CONCURRENT from cross-client pruning asymmetry.
const bothPossiblyPruned =
aKeys.length >= MAX_VECTOR_CLOCK_SIZE && bKeys.length >= MAX_VECTOR_CLOCK_SIZE;
let keysToCompare: Set<string>;
if (bothPossiblyPruned) {
const bKeySet = new Set(bKeys);
keysToCompare = new Set(aKeys.filter((k) => bKeySet.has(k)));
} else {
keysToCompare = new Set([...aKeys, ...bKeys]);
}
let aGreater = false;
let bGreater = false;
for (const key of allKeys) {
for (const key of keysToCompare) {
const aVal = a[key] ?? 0;
const bVal = b[key] ?? 0;
@ -78,3 +105,43 @@ export const mergeVectorClocks = (a: VectorClock, b: VectorClock): VectorClock =
return merged;
};
/**
* Limits vector clock size by keeping only the most active clients.
* Used by both client (when creating ops) and server (when storing ops).
*
* @param clock The vector clock to limit
* @param preserveClientIds Client IDs to always keep (e.g., current client, protected IDs)
* @returns A clock with at most MAX_VECTOR_CLOCK_SIZE entries
*/
export const limitVectorClockSize = (
clock: VectorClock,
preserveClientIds: string[] = [],
): VectorClock => {
const entries = Object.entries(clock);
if (entries.length <= MAX_VECTOR_CLOCK_SIZE) {
return clock;
}
const alwaysPreserve = new Set(preserveClientIds);
// Sort by value descending to keep most active clients
entries.sort(([, a], [, b]) => b - a);
const limited: VectorClock = {};
for (const id of alwaysPreserve) {
if (clock[id] !== undefined) {
limited[id] = clock[id];
}
}
let count = Object.keys(limited).length;
for (const [clientId, value] of entries) {
if (!alwaysPreserve.has(clientId) && count < MAX_VECTOR_CLOCK_SIZE) {
limited[clientId] = value;
count++;
}
}
return limited;
};

View file

@ -3,10 +3,18 @@ import {
VectorClock,
VectorClockComparison,
compareVectorClocks,
limitVectorClockSize,
MAX_VECTOR_CLOCK_SIZE,
} from '@sp/shared-schema';
// Re-export for consumers of this module
export { VectorClock, VectorClockComparison, compareVectorClocks };
export {
VectorClock,
VectorClockComparison,
compareVectorClocks,
limitVectorClockSize,
MAX_VECTOR_CLOCK_SIZE,
};
// Structured error codes for client handling
export const SYNC_ERROR_CODES = {

View file

@ -3,11 +3,10 @@ import {
VectorClock as SharedVectorClock,
compareVectorClocks as sharedCompareVectorClocks,
mergeVectorClocks as sharedMergeVectorClocks,
} from '@sp/shared-schema';
import {
limitVectorClockSize as sharedLimitVectorClockSize,
MAX_VECTOR_CLOCK_SIZE,
MIN_CLIENT_ID_LENGTH,
} from '../../op-log/core/operation-log.const';
} from '@sp/shared-schema';
import { MIN_CLIENT_ID_LENGTH } from '../../op-log/core/operation-log.const';
/**
* Vector Clock implementation for distributed synchronization
@ -295,8 +294,6 @@ export const hasVectorClockChanges = (
return false;
};
// MAX_VECTOR_CLOCK_SIZE imported from operation-log.const.ts
/**
* Metrics for vector clock operations
*/
@ -307,7 +304,9 @@ export interface VectorClockMetrics {
}
/**
* Limits the size of a vector clock by keeping only the most active clients
* Limits the size of a vector clock by keeping only the most active clients.
* Wraps the shared implementation from @sp/shared-schema with client-side logging.
*
* @param clock The vector clock to limit
* @param currentClientId The current client's ID (always preserved)
* @param protectedClientIds Additional client IDs to always preserve (e.g., from latest SYNC_IMPORT).
@ -321,14 +320,10 @@ export const limitVectorClockSize = (
protectedClientIds: string[] = [],
): VectorClock => {
const entries = Object.entries(clock);
if (entries.length <= MAX_VECTOR_CLOCK_SIZE) {
return clock;
}
// Build set of clients to always preserve
const alwaysPreserve = new Set([currentClientId, ...protectedClientIds]);
PFLog.info('Vector clock pruning triggered', {
originalSize: entries.length,
maxSize: MAX_VECTOR_CLOCK_SIZE,
@ -337,27 +332,7 @@ export const limitVectorClockSize = (
pruned: entries.length - MAX_VECTOR_CLOCK_SIZE,
});
// Sort by value (descending) to keep most active clients
entries.sort(([, a], [, b]) => b - a);
// Always keep preserved clients first
const limited: VectorClock = {};
for (const id of alwaysPreserve) {
if (clock[id] !== undefined) {
limited[id] = clock[id];
}
}
// Add top active non-preserved clients up to limit
let count = Object.keys(limited).length;
for (const [clientId, value] of entries) {
if (!alwaysPreserve.has(clientId) && count < MAX_VECTOR_CLOCK_SIZE) {
limited[clientId] = value;
count++;
}
}
return limited;
return sharedLimitVectorClockSize(clock, [currentClientId, ...protectedClientIds]);
};
/**

View file

@ -299,15 +299,27 @@ export class SyncWrapperService {
);
}
// 3. If LWW created local-win ops, upload them
const totalLocalWinOps =
// 3. If LWW created local-win ops, upload them (with retry limit to prevent infinite loops)
const MAX_LWW_REUPLOAD_RETRIES = 3;
let lwwRetries = 0;
let pendingLwwOps =
(downloadResult.localWinOpsCreated ?? 0) +
(uploadResult?.localWinOpsCreated ?? 0);
if (totalLocalWinOps > 0) {
while (pendingLwwOps > 0 && lwwRetries < MAX_LWW_REUPLOAD_RETRIES) {
lwwRetries++;
SyncLog.log(
`SyncWrapperService: Re-uploading ${totalLocalWinOps} local-win op(s) from LWW...`,
`SyncWrapperService: Re-uploading ${pendingLwwOps} local-win op(s) from LWW ` +
`(attempt ${lwwRetries}/${MAX_LWW_REUPLOAD_RETRIES})...`,
);
const reuploadResult =
await this._opLogSyncService.uploadPendingOps(syncCapableProvider);
pendingLwwOps = reuploadResult?.localWinOpsCreated ?? 0;
}
if (pendingLwwOps > 0) {
SyncLog.warn(
`SyncWrapperService: LWW re-upload still has ${pendingLwwOps} pending ops after ` +
`${MAX_LWW_REUPLOAD_RETRIES} retries. Will retry on next sync.`,
);
await this._opLogSyncService.uploadPendingOps(syncCapableProvider);
}
// 4. Check for permanent rejection failures - these are critical failures that should

View file

@ -562,22 +562,32 @@ describe('OperationApplierService', () => {
expect(callOrder).toEqual(['endApplyingRemoteOps', 'processDeferredActions']);
});
it('should call processDeferredActions before startPostSyncCooldown', async () => {
it('should call startPostSyncCooldown before endApplyingRemoteOps to close timing gap', async () => {
const callOrder: string[] = [];
mockHydrationState.startPostSyncCooldown.and.callFake(() => {
callOrder.push('startPostSyncCooldown');
});
mockHydrationState.endApplyingRemoteOps.and.callFake(() => {
callOrder.push('endApplyingRemoteOps');
});
mockOperationLogEffects.processDeferredActions.and.callFake(() => {
callOrder.push('processDeferredActions');
return Promise.resolve();
});
mockHydrationState.startPostSyncCooldown.and.callFake(() => {
callOrder.push('startPostSyncCooldown');
});
const op = createMockOperation('op-1');
await service.applyOperations([op]);
expect(callOrder).toEqual(['processDeferredActions', 'startPostSyncCooldown']);
// Cooldown starts BEFORE ending remote ops to prevent the timing gap
// where isInSyncWindow() returns false and selector-based effects can fire.
expect(callOrder).toEqual([
'startPostSyncCooldown',
'endApplyingRemoteOps',
'processDeferredActions',
]);
});
it('should call processDeferredActions even when archive handling fails', async () => {

View file

@ -108,19 +108,18 @@ export class OperationApplierService {
}
}
} finally {
this.hydrationState.endApplyingRemoteOps();
// Process any user actions that were buffered during sync replay.
// These get fresh vector clocks that include the newly-applied remote ops.
// Do this before cooldown starts so deferred actions are persisted promptly.
await this.injector.get(OperationLogEffects).processDeferredActions();
// Start post-sync cooldown to suppress selector-based effects
// that might fire due to freshly-synced state changes.
// Start cooldown BEFORE ending remote ops flag to eliminate the timing gap
// where isInSyncWindow() returns false and selector-based effects can fire.
// Only needed for remote ops - local hydration doesn't cause the timing gap issue.
if (!isLocalHydration) {
this.hydrationState.startPostSyncCooldown();
}
this.hydrationState.endApplyingRemoteOps();
// Process any user actions that were buffered during sync replay.
// These get fresh vector clocks that include the newly-applied remote ops.
await this.injector.get(OperationLogEffects).processDeferredActions();
}
OpLog.normal('OperationApplierService: Finished applying operations.');

View file

@ -205,8 +205,10 @@ export const DOWNLOAD_PAGE_SIZE = 500;
* Maximum number of clients to track in a vector clock.
* When exceeded, pruning keeps the most active clients (highest counter values).
* The current client is always preserved regardless of activity level.
*
* Re-exported from @sp/shared-schema to ensure client and server use the same value.
*/
export const MAX_VECTOR_CLOCK_SIZE = 10;
export { MAX_VECTOR_CLOCK_SIZE } from '@sp/shared-schema';
/**
* Minimum length for client IDs in vector clocks.