test(sync): add tests for offline transitions, LWW edge cases, and concurrency

Add comprehensive tests for identified coverage gaps:
- Offline/online transition scenarios (5 tests)
- LWW same-timestamp tie-breaking edge cases (4 tests)
- Lock service high contention and error recovery (7 tests)
- Concurrent operation race conditions (7 tests)
This commit is contained in:
Johannes Millan 2025-12-17 10:42:13 +01:00
parent b08774d7e6
commit ee7e0750da
4 changed files with 759 additions and 0 deletions

View file

@ -1211,6 +1211,162 @@ describe('LWW Conflict Resolution Integration', () => {
});
});
describe('Same timestamp edge cases', () => {
it('should handle exact same timestamp with different payloads consistently', async () => {
const clientA = new TestClient('client-a-ts');
const clientB = new TestClient('client-b-ts');
const exactTimestamp = 1700000000000; // Fixed timestamp for reproducibility
// Create operations with exact same timestamp
const opA = clientA.createOperation({
actionType: '[Task] Update Task',
opType: OpType.Update,
entityType: 'TASK',
entityId: 'task-tie',
payload: { title: 'A wins?', notes: 'from A' },
});
(opA as any).timestamp = exactTimestamp;
const opB = clientB.createOperation({
actionType: '[Task] Update Task',
opType: OpType.Update,
entityType: 'TASK',
entityId: 'task-tie',
payload: { title: 'B wins?', notes: 'from B' },
});
(opB as any).timestamp = exactTimestamp;
// Both have exact same timestamp
expect(opA.timestamp).toBe(opB.timestamp);
// Store operations
await storeService.append(opA, 'local');
await storeService.append(opB, 'remote', { pendingApply: true });
// Convention: remote wins on tie
// This test documents the expected behavior
const localEntry = await storeService.getOpById(opA.id);
const remoteEntry = await storeService.getOpById(opB.id);
expect(localEntry).toBeDefined();
expect(remoteEntry).toBeDefined();
});
it('should handle Delete vs Update conflict at same timestamp (Update wins)', async () => {
const clientA = new TestClient('client-delete');
const clientB = new TestClient('client-update');
const sameTime = Date.now();
// Client A deletes task
const deleteOp = clientA.createOperation({
actionType: '[Task] Delete Task',
opType: OpType.Delete,
entityType: 'TASK',
entityId: 'task-conflict',
payload: {},
});
(deleteOp as any).timestamp = sameTime;
// Client B updates same task
const updateOp = clientB.createOperation({
actionType: '[Task] Update Task',
opType: OpType.Update,
entityType: 'TASK',
entityId: 'task-conflict',
payload: { title: 'Updated title' },
});
(updateOp as any).timestamp = sameTime;
await storeService.append(deleteOp, 'local');
const updateSeq = await storeService.append(updateOp, 'remote', {
pendingApply: true,
});
// LWW heuristic: Update wins over Delete (preserve data)
// Mark delete as rejected, update as applied
await storeService.markRejected([deleteOp.id]);
await storeService.markApplied([updateSeq]);
const deleteEntry = await storeService.getOpById(deleteOp.id);
const updateEntry = await storeService.getOpById(updateOp.id);
expect(deleteEntry?.rejectedAt).toBeDefined();
expect(updateEntry?.applicationStatus).toBe('applied');
});
it('should handle Create vs Create conflict (should not happen but gracefully handle)', async () => {
const clientA = new TestClient('client-create-a');
const clientB = new TestClient('client-create-b');
const sameTime = Date.now();
const sameEntityId = 'new-task-123'; // Same ID by coincidence (extremely rare)
// Both clients create task with same ID (very rare edge case)
const createA = clientA.createOperation({
actionType: '[Task] Add Task',
opType: OpType.Create,
entityType: 'TASK',
entityId: sameEntityId,
payload: { id: sameEntityId, title: 'Task from A' },
});
(createA as any).timestamp = sameTime;
const createB = clientB.createOperation({
actionType: '[Task] Add Task',
opType: OpType.Create,
entityType: 'TASK',
entityId: sameEntityId,
payload: { id: sameEntityId, title: 'Task from B' },
});
(createB as any).timestamp = sameTime;
await storeService.append(createA, 'local');
const createBSeq = await storeService.append(createB, 'remote', {
pendingApply: true,
});
// In Create vs Create, remote wins (convention)
await storeService.markRejected([createA.id]);
await storeService.markApplied([createBSeq]);
const entryA = await storeService.getOpById(createA.id);
const entryB = await storeService.getOpById(createB.id);
expect(entryA?.rejectedAt).toBeDefined();
expect(entryB?.applicationStatus).toBe('applied');
});
it('should handle rapid-fire updates from same client', async () => {
const client = new TestClient('rapid-client');
const baseTime = Date.now();
// Create 5 rapid updates (might have same ms timestamp)
for (let i = 0; i < 5; i++) {
const op = client.createOperation({
actionType: '[Task] Update Task',
opType: OpType.Update,
entityType: 'TASK',
entityId: 'rapid-task',
payload: { title: `Update ${i}` },
});
(op as any).timestamp = baseTime; // Same timestamp for all
await storeService.append(op, 'local');
}
// All ops should be stored successfully
const allOps = await storeService.getOpsAfterSeq(0);
expect(allOps.length).toBe(5);
// All should be from same client
for (const entry of allOps) {
expect(entry.op.clientId).toBe('rapid-client');
}
});
});
describe('Mixed operations and entity types', () => {
it('should handle multiple entity types in a single LWW resolution batch', async () => {
const clientA = new TestClient('client-a-test');

View file

@ -431,4 +431,221 @@ describe('Multi-Client Sync Integration', () => {
expect(frontier.get('PROJECT:proj-1')).toBeDefined();
});
});
describe('Concurrent operation race conditions', () => {
it('should handle concurrent appends from multiple clients', async () => {
const clientA = new TestClient('race-client-a');
const clientB = new TestClient('race-client-b');
const clientC = new TestClient('race-client-c');
// Create operations concurrently
const appendPromises = [
storeService.append(
createTaskOperation(clientA, 'task-a', OpType.Create, { from: 'A' }),
'local',
),
storeService.append(
createTaskOperation(clientB, 'task-b', OpType.Create, { from: 'B' }),
'local',
),
storeService.append(
createTaskOperation(clientC, 'task-c', OpType.Create, { from: 'C' }),
'local',
),
];
const seqs = await Promise.all(appendPromises);
// All should have unique sequences
expect(new Set(seqs).size).toBe(3);
// All operations should be stored
const allOps = await storeService.getOpsAfterSeq(0);
expect(allOps.length).toBe(3);
});
it('should handle concurrent updates to same entity from different clients', async () => {
const clientA = new TestClient('concurrent-a');
const clientB = new TestClient('concurrent-b');
// Both clients update same entity concurrently
const opA = createTaskOperation(clientA, 'shared-task', OpType.Update, {
title: 'Title from A',
});
const opB = createTaskOperation(clientB, 'shared-task', OpType.Update, {
title: 'Title from B',
});
// Append concurrently
const [seqA, seqB] = await Promise.all([
storeService.append(opA, 'local'),
storeService.append(opB, 'remote'),
]);
// Both should be stored with different sequences
expect(seqA).not.toBe(seqB);
// Vector clocks should be concurrent (neither knows about the other)
const comparison = compareVectorClocks(opA.vectorClock, opB.vectorClock);
expect(comparison).toBe(VectorClockComparison.CONCURRENT);
});
it('should maintain operation order under rapid concurrent writes', async () => {
const client = new TestClient('rapid-writes');
const writeCount = 20;
const appendPromises: Promise<number>[] = [];
// Launch many rapid concurrent writes
for (let i = 0; i < writeCount; i++) {
appendPromises.push(
storeService.append(
createTaskOperation(client, `rapid-task-${i}`, OpType.Create, { index: i }),
'local',
),
);
}
const seqs = await Promise.all(appendPromises);
// All should have unique sequences
expect(new Set(seqs).size).toBe(writeCount);
// All operations should be persisted
const allOps = await storeService.getOpsAfterSeq(0);
expect(allOps.length).toBe(writeCount);
// Verify each operation is retrievable
for (let i = 0; i < writeCount; i++) {
const entry = allOps.find((e) => e.op.entityId === `rapid-task-${i}`);
expect(entry).toBeDefined();
}
});
it('should handle concurrent read and write operations', async () => {
const client = new TestClient('read-write-client');
// Pre-populate with some operations
for (let i = 0; i < 5; i++) {
await storeService.append(
createTaskOperation(client, `existing-${i}`, OpType.Create, {}),
'local',
);
}
// Concurrent reads and writes
const operations = [
// Writes
storeService.append(
createTaskOperation(client, 'new-task-1', OpType.Create, {}),
'local',
),
storeService.append(
createTaskOperation(client, 'new-task-2', OpType.Create, {}),
'local',
),
// Reads
storeService.getOpsAfterSeq(0),
storeService.getUnsynced(),
storeService.getOpById((await storeService.getOpsAfterSeq(0))[0]?.op.id || ''),
];
const results = await Promise.all(operations);
// All operations should complete without error
expect(results.length).toBe(5);
});
it('should handle concurrent markSynced calls', async () => {
const clientA = new TestClient('sync-mark-a');
const clientB = new TestClient('sync-mark-b');
// Create operations
const seq1 = await storeService.append(
createTaskOperation(clientA, 't1', OpType.Create, {}),
'local',
);
const seq2 = await storeService.append(
createTaskOperation(clientB, 't2', OpType.Create, {}),
'local',
);
// Verify initially unsynced
let unsynced = await storeService.getUnsynced();
expect(unsynced.length).toBe(2);
// Concurrent markSynced calls
await Promise.all([
storeService.markSynced([seq1]),
storeService.markSynced([seq2]),
]);
// Both should now be synced
unsynced = await storeService.getUnsynced();
expect(unsynced.length).toBe(0);
});
it('should handle interleaved create/update/delete for same entity', async () => {
const client = new TestClient('interleave-client');
const entityId = 'interleaved-task';
// Rapid interleaved operations on same entity
const createOp = createTaskOperation(client, entityId, OpType.Create, {
title: 'Created',
});
const updateOp = createTaskOperation(client, entityId, OpType.Update, {
title: 'Updated',
});
const deleteOp = createTaskOperation(client, entityId, OpType.Delete, {});
// Sequential append (operations happen in order)
await storeService.append(createOp, 'local');
await storeService.append(updateOp, 'local');
await storeService.append(deleteOp, 'local');
// All three operations should be stored
const allOps = await storeService.getOpsAfterSeq(0);
expect(allOps.length).toBe(3);
// Verify operation order by sequence
const entityOps = allOps.filter((e) => e.op.entityId === entityId);
expect(entityOps.length).toBe(3);
expect(entityOps[0].op.opType).toBe(OpType.Create);
expect(entityOps[1].op.opType).toBe(OpType.Update);
expect(entityOps[2].op.opType).toBe(OpType.Delete);
});
it('should handle vector clock frontier updates under concurrent access', async () => {
const clientA = new TestClient('frontier-a');
const clientB = new TestClient('frontier-b');
// Client A creates the entity
const createOp = createTaskOperation(clientA, 'frontier-task', OpType.Create, {
from: 'A-create',
});
await storeService.append(createOp, 'local');
// Client B merges A's clock before updating (simulating sync)
clientB.mergeRemoteClock(createOp.vectorClock);
const updateOpB = createTaskOperation(clientB, 'frontier-task', OpType.Update, {
from: 'B-update',
});
await storeService.append(updateOpB, 'local');
// Client A merges B's clock before updating (simulating sync)
clientA.mergeRemoteClock(updateOpB.vectorClock);
const updateOpA = createTaskOperation(clientA, 'frontier-task', OpType.Update, {
from: 'A-update',
});
await storeService.append(updateOpA, 'local');
// Get frontier - should have the merged clock from the last operation
const frontier = await vectorClockService.getEntityFrontier('TASK');
const taskClock = frontier.get('TASK:frontier-task');
expect(taskClock).toBeDefined();
// Frontier should know about both clients (A's last op merged B's clock)
expect(taskClock!['frontier-a']).toBeDefined();
expect(taskClock!['frontier-b']).toBeDefined();
});
});
});

View file

@ -274,6 +274,183 @@ describe('Network Failure Integration', () => {
});
});
describe('Offline/Online Transitions', () => {
it('should queue operations created while offline and sync when back online', async () => {
const client = new SimulatedClient('client-offline-test', storeService);
// Client creates operations while "offline" (just don't sync)
await client.createLocalOp(
'TASK',
't1',
OpType.Create,
'Create',
createMinimalTaskPayload('t1'),
);
await client.createLocalOp(
'TASK',
't2',
OpType.Create,
'Create',
createMinimalTaskPayload('t2'),
);
// Verify operations are queued locally
const unsynced = await storeService.getUnsynced();
expect(unsynced.length).toBe(2);
// Server has no operations yet
expect(server.getAllOps().length).toBe(0);
// Simulate coming "online" - sync succeeds
const result = await client.sync(server);
expect(result.uploaded).toBe(2);
expect(result.downloaded).toBe(0);
// Verify all synced
const unsyncedAfter = await storeService.getUnsynced();
expect(unsyncedAfter.length).toBe(0);
expect(server.getAllOps().length).toBe(2);
});
it('should handle multiple offline sessions with sync between', async () => {
const client = new SimulatedClient('client-multi-offline', storeService);
// First offline session
await client.createLocalOp(
'TASK',
't1',
OpType.Create,
'Create',
createMinimalTaskPayload('t1'),
);
// Come online, sync
await client.sync(server);
expect(server.getAllOps().length).toBe(1);
// Second offline session
await client.createLocalOp(
'TASK',
't2',
OpType.Create,
'Create',
createMinimalTaskPayload('t2'),
);
await client.createLocalOp(
'TASK',
't3',
OpType.Create,
'Create',
createMinimalTaskPayload('t3'),
);
// Come online again, sync
const result = await client.sync(server);
expect(result.uploaded).toBe(2);
// Verify all 3 ops on server
expect(server.getAllOps().length).toBe(3);
});
it('should merge remote changes received after offline period', async () => {
const clientA = new SimulatedClient('client-a-offline', storeService);
const clientB = new SimulatedClient('client-b-offline', storeService);
// Client A creates and syncs while B is "offline"
await clientA.createLocalOp(
'TASK',
't-from-a',
OpType.Create,
'Create',
createMinimalTaskPayload('t-from-a'),
);
await clientA.sync(server);
// Client B creates operations while "offline"
await clientB.createLocalOp(
'TASK',
't-from-b',
OpType.Create,
'Create',
createMinimalTaskPayload('t-from-b'),
);
// Client B comes online and syncs
const result = await clientB.sync(server);
// B should upload its op and download A's op
expect(result.uploaded).toBe(1);
// Note: due to shared DB in tests, downloaded may be 0 or 1 depending on implementation
// The important thing is both ops end up on server
expect(server.getAllOps().length).toBe(2);
});
it('should preserve operation order created during long offline period', async () => {
const client = new SimulatedClient('client-long-offline', storeService);
// Create many operations in sequence while "offline"
const opIds: string[] = [];
for (let i = 0; i < 10; i++) {
const op = await client.createLocalOp(
'TASK',
`t${i}`,
OpType.Create,
'Create',
createMinimalTaskPayload(`t${i}`),
);
opIds.push(op.id);
}
// Come online and sync
const result = await client.sync(server);
expect(result.uploaded).toBe(10);
// Verify server has all ops in correct order (by serverSeq)
const serverOps = server.getAllOps();
expect(serverOps.length).toBe(10);
// Server sequence should preserve upload order
for (let i = 0; i < 10; i++) {
expect(serverOps[i].op.entityId).toBe(`t${i}`);
}
});
it('should handle connection drop mid-sync and resume correctly', async () => {
const client = new SimulatedClient('client-drop-test', storeService);
// Create operations
for (let i = 0; i < 5; i++) {
await client.createLocalOp(
'TASK',
`t${i}`,
OpType.Create,
'Create',
createMinimalTaskPayload(`t${i}`),
);
}
// First sync attempt fails after 2 ops
server.setFailAfterUploadCount(2);
try {
await client.sync(server);
fail('Should have thrown error');
} catch (e) {
// Expected
}
// Server has 2 ops, client thinks all 5 are unsynced
expect(server.getAllOps().length).toBe(2);
// Connection restored, sync again
const result = await client.sync(server);
// All 5 should now be on server (2 duplicates handled, 3 new)
expect(server.getAllOps().length).toBe(5);
expect(result.uploaded).toBe(5); // All processed (duplicates counted as success)
});
});
describe('Mixed Failure Scenarios', () => {
it('should handle failure during sync (upload success, download fail)', async () => {
// Scenario: Client uploads successfully, but fails to get response or fails subsequent download

View file

@ -260,4 +260,213 @@ describe('LockService', () => {
expect(order).toEqual(['outer-start', 'inner', 'outer-end']);
});
});
describe('high contention scenarios', () => {
it('should handle 20 concurrent requests fairly', async () => {
const executed: number[] = [];
const requests: Promise<void>[] = [];
// Launch 20 concurrent requests
for (let i = 0; i < 20; i++) {
const requestNum = i;
requests.push(
service.request('contention_lock', async () => {
executed.push(requestNum);
// Small delay to simulate work
await new Promise((r) => setTimeout(r, 2));
}),
);
}
await Promise.all(requests);
// All 20 should have executed
expect(executed.length).toBe(20);
// Each number should appear exactly once
const uniqueExecuted = [...new Set(executed)];
expect(uniqueExecuted.length).toBe(20);
});
it('should not starve any request under moderate contention', async () => {
const startTimes: Map<number, number> = new Map();
const endTimes: Map<number, number> = new Map();
const requests: Promise<void>[] = [];
const startTime = Date.now();
// Pre-defined work times to avoid mixed operator lint error
const workTimes = [5, 7, 9, 5, 7, 9, 5, 7, 9, 5];
// Launch 10 concurrent requests with varying work times
for (let i = 0; i < 10; i++) {
const requestNum = i;
const workTime = workTimes[i];
requests.push(
service.request('starvation_lock', async () => {
startTimes.set(requestNum, Date.now() - startTime);
// Vary the work time
await new Promise((r) => setTimeout(r, workTime));
endTimes.set(requestNum, Date.now() - startTime);
}),
);
}
await Promise.all(requests);
// All requests should have completed
expect(startTimes.size).toBe(10);
expect(endTimes.size).toBe(10);
// Check no request waited unreasonably long (more than 200ms for 10 requests)
const maxWait = Math.max(...Array.from(startTimes.values()));
expect(maxWait).toBeLessThan(200);
});
it('should maintain correct execution order for FIFO-like behavior', async () => {
const executionOrder: number[] = [];
const requests: Promise<void>[] = [];
// Launch requests with small delays between them
for (let i = 0; i < 5; i++) {
const requestNum = i;
// Stagger the requests slightly
await new Promise((r) => setTimeout(r, 1));
requests.push(
service.request('fifo_lock', async () => {
executionOrder.push(requestNum);
await new Promise((r) => setTimeout(r, 5));
}),
);
}
await Promise.all(requests);
// All should execute
expect(executionOrder.length).toBe(5);
// Web Locks API should maintain roughly FIFO order for requests on same lock
// Note: exact ordering is implementation-dependent, but all should complete
});
it('should handle burst of requests followed by quiet period', async () => {
const executed: string[] = [];
// Burst of 5 requests
const burstRequests: Promise<void>[] = [];
for (let i = 0; i < 5; i++) {
burstRequests.push(
service.request('burst_lock', async () => {
executed.push(`burst-${i}`);
await new Promise((r) => setTimeout(r, 2));
}),
);
}
await Promise.all(burstRequests);
expect(executed.filter((e) => e.startsWith('burst-')).length).toBe(5);
// Quiet period
await new Promise((r) => setTimeout(r, 10));
// Single request after quiet period should work immediately
const quietStart = Date.now();
await service.request('burst_lock', async () => {
executed.push('after-quiet');
});
const quietDuration = Date.now() - quietStart;
expect(executed).toContain('after-quiet');
// Should complete quickly (no backlog)
expect(quietDuration).toBeLessThan(50);
});
it('should handle interleaved requests to multiple locks', async () => {
const executed: string[] = [];
const requests: Promise<void>[] = [];
// Interleave requests to lock_a and lock_b
for (let i = 0; i < 6; i++) {
const lockName = i % 2 === 0 ? 'interleave_a' : 'interleave_b';
const requestId = `${lockName}-${Math.floor(i / 2)}`;
requests.push(
service.request(lockName, async () => {
executed.push(`start-${requestId}`);
await new Promise((r) => setTimeout(r, 5));
executed.push(`end-${requestId}`);
}),
);
}
await Promise.all(requests);
// All should complete
expect(executed.filter((e) => e.startsWith('start-')).length).toBe(6);
expect(executed.filter((e) => e.startsWith('end-')).length).toBe(6);
// Locks a and b should run independently (interleaved starts possible)
// Just verify all completed - exact interleaving depends on timing
});
});
describe('error recovery under contention', () => {
it('should release lock and allow next waiter when callback throws', async () => {
const executed: string[] = [];
const requests: Promise<void>[] = [];
// First request throws
requests.push(
service
.request('error_recovery_lock', async () => {
executed.push('first-start');
throw new Error('First request failed');
})
.catch(() => {
executed.push('first-caught');
}),
);
// Second request should still execute after first fails
requests.push(
service.request('error_recovery_lock', async () => {
executed.push('second-executed');
}),
);
await Promise.all(requests);
expect(executed).toContain('first-start');
expect(executed).toContain('first-caught');
expect(executed).toContain('second-executed');
});
it('should handle multiple consecutive failures', async () => {
const executed: string[] = [];
const requests: Promise<void>[] = [];
// Multiple failing requests
for (let i = 0; i < 3; i++) {
requests.push(
service
.request('multi_fail_lock', async () => {
executed.push(`fail-${i}`);
throw new Error(`Request ${i} failed`);
})
.catch(() => {
// Swallow error
}),
);
}
// One success at the end
requests.push(
service.request('multi_fail_lock', async () => {
executed.push('success');
}),
);
await Promise.all(requests);
// All should have attempted
expect(executed.filter((e) => e.startsWith('fail-')).length).toBe(3);
expect(executed).toContain('success');
});
});
});