diff --git a/src/app/core/persistence/operation-log/integration/lww-conflict-resolution.integration.spec.ts b/src/app/core/persistence/operation-log/integration/lww-conflict-resolution.integration.spec.ts index 88008fcb9b..c28c70b827 100644 --- a/src/app/core/persistence/operation-log/integration/lww-conflict-resolution.integration.spec.ts +++ b/src/app/core/persistence/operation-log/integration/lww-conflict-resolution.integration.spec.ts @@ -1211,6 +1211,162 @@ describe('LWW Conflict Resolution Integration', () => { }); }); + describe('Same timestamp edge cases', () => { + it('should handle exact same timestamp with different payloads consistently', async () => { + const clientA = new TestClient('client-a-ts'); + const clientB = new TestClient('client-b-ts'); + + const exactTimestamp = 1700000000000; // Fixed timestamp for reproducibility + + // Create operations with exact same timestamp + const opA = clientA.createOperation({ + actionType: '[Task] Update Task', + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-tie', + payload: { title: 'A wins?', notes: 'from A' }, + }); + (opA as any).timestamp = exactTimestamp; + + const opB = clientB.createOperation({ + actionType: '[Task] Update Task', + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-tie', + payload: { title: 'B wins?', notes: 'from B' }, + }); + (opB as any).timestamp = exactTimestamp; + + // Both have exact same timestamp + expect(opA.timestamp).toBe(opB.timestamp); + + // Store operations + await storeService.append(opA, 'local'); + await storeService.append(opB, 'remote', { pendingApply: true }); + + // Convention: remote wins on tie + // This test documents the expected behavior + const localEntry = await storeService.getOpById(opA.id); + const remoteEntry = await storeService.getOpById(opB.id); + + expect(localEntry).toBeDefined(); + expect(remoteEntry).toBeDefined(); + }); + + it('should handle Delete vs Update conflict at same timestamp (Update wins)', async () => { + const clientA = new TestClient('client-delete'); + const clientB = new TestClient('client-update'); + + const sameTime = Date.now(); + + // Client A deletes task + const deleteOp = clientA.createOperation({ + actionType: '[Task] Delete Task', + opType: OpType.Delete, + entityType: 'TASK', + entityId: 'task-conflict', + payload: {}, + }); + (deleteOp as any).timestamp = sameTime; + + // Client B updates same task + const updateOp = clientB.createOperation({ + actionType: '[Task] Update Task', + opType: OpType.Update, + entityType: 'TASK', + entityId: 'task-conflict', + payload: { title: 'Updated title' }, + }); + (updateOp as any).timestamp = sameTime; + + await storeService.append(deleteOp, 'local'); + const updateSeq = await storeService.append(updateOp, 'remote', { + pendingApply: true, + }); + + // LWW heuristic: Update wins over Delete (preserve data) + // Mark delete as rejected, update as applied + await storeService.markRejected([deleteOp.id]); + await storeService.markApplied([updateSeq]); + + const deleteEntry = await storeService.getOpById(deleteOp.id); + const updateEntry = await storeService.getOpById(updateOp.id); + + expect(deleteEntry?.rejectedAt).toBeDefined(); + expect(updateEntry?.applicationStatus).toBe('applied'); + }); + + it('should handle Create vs Create conflict (should not happen but gracefully handle)', async () => { + const clientA = new TestClient('client-create-a'); + const clientB = new TestClient('client-create-b'); + + const sameTime = Date.now(); + const sameEntityId = 'new-task-123'; // Same ID by coincidence (extremely rare) + + // Both clients create task with same ID (very rare edge case) + const createA = clientA.createOperation({ + actionType: '[Task] Add Task', + opType: OpType.Create, + entityType: 'TASK', + entityId: sameEntityId, + payload: { id: sameEntityId, title: 'Task from A' }, + }); + (createA as any).timestamp = sameTime; + + const createB = clientB.createOperation({ + actionType: '[Task] Add Task', + opType: OpType.Create, + entityType: 'TASK', + entityId: sameEntityId, + payload: { id: sameEntityId, title: 'Task from B' }, + }); + (createB as any).timestamp = sameTime; + + await storeService.append(createA, 'local'); + const createBSeq = await storeService.append(createB, 'remote', { + pendingApply: true, + }); + + // In Create vs Create, remote wins (convention) + await storeService.markRejected([createA.id]); + await storeService.markApplied([createBSeq]); + + const entryA = await storeService.getOpById(createA.id); + const entryB = await storeService.getOpById(createB.id); + + expect(entryA?.rejectedAt).toBeDefined(); + expect(entryB?.applicationStatus).toBe('applied'); + }); + + it('should handle rapid-fire updates from same client', async () => { + const client = new TestClient('rapid-client'); + + const baseTime = Date.now(); + + // Create 5 rapid updates (might have same ms timestamp) + for (let i = 0; i < 5; i++) { + const op = client.createOperation({ + actionType: '[Task] Update Task', + opType: OpType.Update, + entityType: 'TASK', + entityId: 'rapid-task', + payload: { title: `Update ${i}` }, + }); + (op as any).timestamp = baseTime; // Same timestamp for all + await storeService.append(op, 'local'); + } + + // All ops should be stored successfully + const allOps = await storeService.getOpsAfterSeq(0); + expect(allOps.length).toBe(5); + + // All should be from same client + for (const entry of allOps) { + expect(entry.op.clientId).toBe('rapid-client'); + } + }); + }); + describe('Mixed operations and entity types', () => { it('should handle multiple entity types in a single LWW resolution batch', async () => { const clientA = new TestClient('client-a-test'); diff --git a/src/app/core/persistence/operation-log/integration/multi-client-sync.integration.spec.ts b/src/app/core/persistence/operation-log/integration/multi-client-sync.integration.spec.ts index 8fa7c1ee5d..8e34b415cb 100644 --- a/src/app/core/persistence/operation-log/integration/multi-client-sync.integration.spec.ts +++ b/src/app/core/persistence/operation-log/integration/multi-client-sync.integration.spec.ts @@ -431,4 +431,221 @@ describe('Multi-Client Sync Integration', () => { expect(frontier.get('PROJECT:proj-1')).toBeDefined(); }); }); + + describe('Concurrent operation race conditions', () => { + it('should handle concurrent appends from multiple clients', async () => { + const clientA = new TestClient('race-client-a'); + const clientB = new TestClient('race-client-b'); + const clientC = new TestClient('race-client-c'); + + // Create operations concurrently + const appendPromises = [ + storeService.append( + createTaskOperation(clientA, 'task-a', OpType.Create, { from: 'A' }), + 'local', + ), + storeService.append( + createTaskOperation(clientB, 'task-b', OpType.Create, { from: 'B' }), + 'local', + ), + storeService.append( + createTaskOperation(clientC, 'task-c', OpType.Create, { from: 'C' }), + 'local', + ), + ]; + + const seqs = await Promise.all(appendPromises); + + // All should have unique sequences + expect(new Set(seqs).size).toBe(3); + + // All operations should be stored + const allOps = await storeService.getOpsAfterSeq(0); + expect(allOps.length).toBe(3); + }); + + it('should handle concurrent updates to same entity from different clients', async () => { + const clientA = new TestClient('concurrent-a'); + const clientB = new TestClient('concurrent-b'); + + // Both clients update same entity concurrently + const opA = createTaskOperation(clientA, 'shared-task', OpType.Update, { + title: 'Title from A', + }); + const opB = createTaskOperation(clientB, 'shared-task', OpType.Update, { + title: 'Title from B', + }); + + // Append concurrently + const [seqA, seqB] = await Promise.all([ + storeService.append(opA, 'local'), + storeService.append(opB, 'remote'), + ]); + + // Both should be stored with different sequences + expect(seqA).not.toBe(seqB); + + // Vector clocks should be concurrent (neither knows about the other) + const comparison = compareVectorClocks(opA.vectorClock, opB.vectorClock); + expect(comparison).toBe(VectorClockComparison.CONCURRENT); + }); + + it('should maintain operation order under rapid concurrent writes', async () => { + const client = new TestClient('rapid-writes'); + const writeCount = 20; + const appendPromises: Promise[] = []; + + // Launch many rapid concurrent writes + for (let i = 0; i < writeCount; i++) { + appendPromises.push( + storeService.append( + createTaskOperation(client, `rapid-task-${i}`, OpType.Create, { index: i }), + 'local', + ), + ); + } + + const seqs = await Promise.all(appendPromises); + + // All should have unique sequences + expect(new Set(seqs).size).toBe(writeCount); + + // All operations should be persisted + const allOps = await storeService.getOpsAfterSeq(0); + expect(allOps.length).toBe(writeCount); + + // Verify each operation is retrievable + for (let i = 0; i < writeCount; i++) { + const entry = allOps.find((e) => e.op.entityId === `rapid-task-${i}`); + expect(entry).toBeDefined(); + } + }); + + it('should handle concurrent read and write operations', async () => { + const client = new TestClient('read-write-client'); + + // Pre-populate with some operations + for (let i = 0; i < 5; i++) { + await storeService.append( + createTaskOperation(client, `existing-${i}`, OpType.Create, {}), + 'local', + ); + } + + // Concurrent reads and writes + const operations = [ + // Writes + storeService.append( + createTaskOperation(client, 'new-task-1', OpType.Create, {}), + 'local', + ), + storeService.append( + createTaskOperation(client, 'new-task-2', OpType.Create, {}), + 'local', + ), + // Reads + storeService.getOpsAfterSeq(0), + storeService.getUnsynced(), + storeService.getOpById((await storeService.getOpsAfterSeq(0))[0]?.op.id || ''), + ]; + + const results = await Promise.all(operations); + + // All operations should complete without error + expect(results.length).toBe(5); + }); + + it('should handle concurrent markSynced calls', async () => { + const clientA = new TestClient('sync-mark-a'); + const clientB = new TestClient('sync-mark-b'); + + // Create operations + const seq1 = await storeService.append( + createTaskOperation(clientA, 't1', OpType.Create, {}), + 'local', + ); + const seq2 = await storeService.append( + createTaskOperation(clientB, 't2', OpType.Create, {}), + 'local', + ); + + // Verify initially unsynced + let unsynced = await storeService.getUnsynced(); + expect(unsynced.length).toBe(2); + + // Concurrent markSynced calls + await Promise.all([ + storeService.markSynced([seq1]), + storeService.markSynced([seq2]), + ]); + + // Both should now be synced + unsynced = await storeService.getUnsynced(); + expect(unsynced.length).toBe(0); + }); + + it('should handle interleaved create/update/delete for same entity', async () => { + const client = new TestClient('interleave-client'); + const entityId = 'interleaved-task'; + + // Rapid interleaved operations on same entity + const createOp = createTaskOperation(client, entityId, OpType.Create, { + title: 'Created', + }); + const updateOp = createTaskOperation(client, entityId, OpType.Update, { + title: 'Updated', + }); + const deleteOp = createTaskOperation(client, entityId, OpType.Delete, {}); + + // Sequential append (operations happen in order) + await storeService.append(createOp, 'local'); + await storeService.append(updateOp, 'local'); + await storeService.append(deleteOp, 'local'); + + // All three operations should be stored + const allOps = await storeService.getOpsAfterSeq(0); + expect(allOps.length).toBe(3); + + // Verify operation order by sequence + const entityOps = allOps.filter((e) => e.op.entityId === entityId); + expect(entityOps.length).toBe(3); + expect(entityOps[0].op.opType).toBe(OpType.Create); + expect(entityOps[1].op.opType).toBe(OpType.Update); + expect(entityOps[2].op.opType).toBe(OpType.Delete); + }); + + it('should handle vector clock frontier updates under concurrent access', async () => { + const clientA = new TestClient('frontier-a'); + const clientB = new TestClient('frontier-b'); + + // Client A creates the entity + const createOp = createTaskOperation(clientA, 'frontier-task', OpType.Create, { + from: 'A-create', + }); + await storeService.append(createOp, 'local'); + + // Client B merges A's clock before updating (simulating sync) + clientB.mergeRemoteClock(createOp.vectorClock); + const updateOpB = createTaskOperation(clientB, 'frontier-task', OpType.Update, { + from: 'B-update', + }); + await storeService.append(updateOpB, 'local'); + + // Client A merges B's clock before updating (simulating sync) + clientA.mergeRemoteClock(updateOpB.vectorClock); + const updateOpA = createTaskOperation(clientA, 'frontier-task', OpType.Update, { + from: 'A-update', + }); + await storeService.append(updateOpA, 'local'); + + // Get frontier - should have the merged clock from the last operation + const frontier = await vectorClockService.getEntityFrontier('TASK'); + const taskClock = frontier.get('TASK:frontier-task'); + + expect(taskClock).toBeDefined(); + // Frontier should know about both clients (A's last op merged B's clock) + expect(taskClock!['frontier-a']).toBeDefined(); + expect(taskClock!['frontier-b']).toBeDefined(); + }); + }); }); diff --git a/src/app/core/persistence/operation-log/integration/network-failure.integration.spec.ts b/src/app/core/persistence/operation-log/integration/network-failure.integration.spec.ts index 74e06fe808..2bd923ac3c 100644 --- a/src/app/core/persistence/operation-log/integration/network-failure.integration.spec.ts +++ b/src/app/core/persistence/operation-log/integration/network-failure.integration.spec.ts @@ -274,6 +274,183 @@ describe('Network Failure Integration', () => { }); }); + describe('Offline/Online Transitions', () => { + it('should queue operations created while offline and sync when back online', async () => { + const client = new SimulatedClient('client-offline-test', storeService); + + // Client creates operations while "offline" (just don't sync) + await client.createLocalOp( + 'TASK', + 't1', + OpType.Create, + 'Create', + createMinimalTaskPayload('t1'), + ); + await client.createLocalOp( + 'TASK', + 't2', + OpType.Create, + 'Create', + createMinimalTaskPayload('t2'), + ); + + // Verify operations are queued locally + const unsynced = await storeService.getUnsynced(); + expect(unsynced.length).toBe(2); + + // Server has no operations yet + expect(server.getAllOps().length).toBe(0); + + // Simulate coming "online" - sync succeeds + const result = await client.sync(server); + expect(result.uploaded).toBe(2); + expect(result.downloaded).toBe(0); + + // Verify all synced + const unsyncedAfter = await storeService.getUnsynced(); + expect(unsyncedAfter.length).toBe(0); + expect(server.getAllOps().length).toBe(2); + }); + + it('should handle multiple offline sessions with sync between', async () => { + const client = new SimulatedClient('client-multi-offline', storeService); + + // First offline session + await client.createLocalOp( + 'TASK', + 't1', + OpType.Create, + 'Create', + createMinimalTaskPayload('t1'), + ); + + // Come online, sync + await client.sync(server); + expect(server.getAllOps().length).toBe(1); + + // Second offline session + await client.createLocalOp( + 'TASK', + 't2', + OpType.Create, + 'Create', + createMinimalTaskPayload('t2'), + ); + await client.createLocalOp( + 'TASK', + 't3', + OpType.Create, + 'Create', + createMinimalTaskPayload('t3'), + ); + + // Come online again, sync + const result = await client.sync(server); + expect(result.uploaded).toBe(2); + + // Verify all 3 ops on server + expect(server.getAllOps().length).toBe(3); + }); + + it('should merge remote changes received after offline period', async () => { + const clientA = new SimulatedClient('client-a-offline', storeService); + const clientB = new SimulatedClient('client-b-offline', storeService); + + // Client A creates and syncs while B is "offline" + await clientA.createLocalOp( + 'TASK', + 't-from-a', + OpType.Create, + 'Create', + createMinimalTaskPayload('t-from-a'), + ); + await clientA.sync(server); + + // Client B creates operations while "offline" + await clientB.createLocalOp( + 'TASK', + 't-from-b', + OpType.Create, + 'Create', + createMinimalTaskPayload('t-from-b'), + ); + + // Client B comes online and syncs + const result = await clientB.sync(server); + + // B should upload its op and download A's op + expect(result.uploaded).toBe(1); + // Note: due to shared DB in tests, downloaded may be 0 or 1 depending on implementation + // The important thing is both ops end up on server + expect(server.getAllOps().length).toBe(2); + }); + + it('should preserve operation order created during long offline period', async () => { + const client = new SimulatedClient('client-long-offline', storeService); + + // Create many operations in sequence while "offline" + const opIds: string[] = []; + for (let i = 0; i < 10; i++) { + const op = await client.createLocalOp( + 'TASK', + `t${i}`, + OpType.Create, + 'Create', + createMinimalTaskPayload(`t${i}`), + ); + opIds.push(op.id); + } + + // Come online and sync + const result = await client.sync(server); + expect(result.uploaded).toBe(10); + + // Verify server has all ops in correct order (by serverSeq) + const serverOps = server.getAllOps(); + expect(serverOps.length).toBe(10); + + // Server sequence should preserve upload order + for (let i = 0; i < 10; i++) { + expect(serverOps[i].op.entityId).toBe(`t${i}`); + } + }); + + it('should handle connection drop mid-sync and resume correctly', async () => { + const client = new SimulatedClient('client-drop-test', storeService); + + // Create operations + for (let i = 0; i < 5; i++) { + await client.createLocalOp( + 'TASK', + `t${i}`, + OpType.Create, + 'Create', + createMinimalTaskPayload(`t${i}`), + ); + } + + // First sync attempt fails after 2 ops + server.setFailAfterUploadCount(2); + + try { + await client.sync(server); + fail('Should have thrown error'); + } catch (e) { + // Expected + } + + // Server has 2 ops, client thinks all 5 are unsynced + expect(server.getAllOps().length).toBe(2); + + // Connection restored, sync again + const result = await client.sync(server); + + // All 5 should now be on server (2 duplicates handled, 3 new) + expect(server.getAllOps().length).toBe(5); + expect(result.uploaded).toBe(5); // All processed (duplicates counted as success) + }); + }); + describe('Mixed Failure Scenarios', () => { it('should handle failure during sync (upload success, download fail)', async () => { // Scenario: Client uploads successfully, but fails to get response or fails subsequent download diff --git a/src/app/core/persistence/operation-log/sync/lock.service.spec.ts b/src/app/core/persistence/operation-log/sync/lock.service.spec.ts index cbc6416fd4..bc6accb455 100644 --- a/src/app/core/persistence/operation-log/sync/lock.service.spec.ts +++ b/src/app/core/persistence/operation-log/sync/lock.service.spec.ts @@ -260,4 +260,213 @@ describe('LockService', () => { expect(order).toEqual(['outer-start', 'inner', 'outer-end']); }); }); + + describe('high contention scenarios', () => { + it('should handle 20 concurrent requests fairly', async () => { + const executed: number[] = []; + const requests: Promise[] = []; + + // Launch 20 concurrent requests + for (let i = 0; i < 20; i++) { + const requestNum = i; + requests.push( + service.request('contention_lock', async () => { + executed.push(requestNum); + // Small delay to simulate work + await new Promise((r) => setTimeout(r, 2)); + }), + ); + } + + await Promise.all(requests); + + // All 20 should have executed + expect(executed.length).toBe(20); + // Each number should appear exactly once + const uniqueExecuted = [...new Set(executed)]; + expect(uniqueExecuted.length).toBe(20); + }); + + it('should not starve any request under moderate contention', async () => { + const startTimes: Map = new Map(); + const endTimes: Map = new Map(); + const requests: Promise[] = []; + + const startTime = Date.now(); + // Pre-defined work times to avoid mixed operator lint error + const workTimes = [5, 7, 9, 5, 7, 9, 5, 7, 9, 5]; + + // Launch 10 concurrent requests with varying work times + for (let i = 0; i < 10; i++) { + const requestNum = i; + const workTime = workTimes[i]; + requests.push( + service.request('starvation_lock', async () => { + startTimes.set(requestNum, Date.now() - startTime); + // Vary the work time + await new Promise((r) => setTimeout(r, workTime)); + endTimes.set(requestNum, Date.now() - startTime); + }), + ); + } + + await Promise.all(requests); + + // All requests should have completed + expect(startTimes.size).toBe(10); + expect(endTimes.size).toBe(10); + + // Check no request waited unreasonably long (more than 200ms for 10 requests) + const maxWait = Math.max(...Array.from(startTimes.values())); + expect(maxWait).toBeLessThan(200); + }); + + it('should maintain correct execution order for FIFO-like behavior', async () => { + const executionOrder: number[] = []; + const requests: Promise[] = []; + + // Launch requests with small delays between them + for (let i = 0; i < 5; i++) { + const requestNum = i; + // Stagger the requests slightly + await new Promise((r) => setTimeout(r, 1)); + requests.push( + service.request('fifo_lock', async () => { + executionOrder.push(requestNum); + await new Promise((r) => setTimeout(r, 5)); + }), + ); + } + + await Promise.all(requests); + + // All should execute + expect(executionOrder.length).toBe(5); + // Web Locks API should maintain roughly FIFO order for requests on same lock + // Note: exact ordering is implementation-dependent, but all should complete + }); + + it('should handle burst of requests followed by quiet period', async () => { + const executed: string[] = []; + + // Burst of 5 requests + const burstRequests: Promise[] = []; + for (let i = 0; i < 5; i++) { + burstRequests.push( + service.request('burst_lock', async () => { + executed.push(`burst-${i}`); + await new Promise((r) => setTimeout(r, 2)); + }), + ); + } + + await Promise.all(burstRequests); + expect(executed.filter((e) => e.startsWith('burst-')).length).toBe(5); + + // Quiet period + await new Promise((r) => setTimeout(r, 10)); + + // Single request after quiet period should work immediately + const quietStart = Date.now(); + await service.request('burst_lock', async () => { + executed.push('after-quiet'); + }); + const quietDuration = Date.now() - quietStart; + + expect(executed).toContain('after-quiet'); + // Should complete quickly (no backlog) + expect(quietDuration).toBeLessThan(50); + }); + + it('should handle interleaved requests to multiple locks', async () => { + const executed: string[] = []; + const requests: Promise[] = []; + + // Interleave requests to lock_a and lock_b + for (let i = 0; i < 6; i++) { + const lockName = i % 2 === 0 ? 'interleave_a' : 'interleave_b'; + const requestId = `${lockName}-${Math.floor(i / 2)}`; + requests.push( + service.request(lockName, async () => { + executed.push(`start-${requestId}`); + await new Promise((r) => setTimeout(r, 5)); + executed.push(`end-${requestId}`); + }), + ); + } + + await Promise.all(requests); + + // All should complete + expect(executed.filter((e) => e.startsWith('start-')).length).toBe(6); + expect(executed.filter((e) => e.startsWith('end-')).length).toBe(6); + + // Locks a and b should run independently (interleaved starts possible) + // Just verify all completed - exact interleaving depends on timing + }); + }); + + describe('error recovery under contention', () => { + it('should release lock and allow next waiter when callback throws', async () => { + const executed: string[] = []; + const requests: Promise[] = []; + + // First request throws + requests.push( + service + .request('error_recovery_lock', async () => { + executed.push('first-start'); + throw new Error('First request failed'); + }) + .catch(() => { + executed.push('first-caught'); + }), + ); + + // Second request should still execute after first fails + requests.push( + service.request('error_recovery_lock', async () => { + executed.push('second-executed'); + }), + ); + + await Promise.all(requests); + + expect(executed).toContain('first-start'); + expect(executed).toContain('first-caught'); + expect(executed).toContain('second-executed'); + }); + + it('should handle multiple consecutive failures', async () => { + const executed: string[] = []; + const requests: Promise[] = []; + + // Multiple failing requests + for (let i = 0; i < 3; i++) { + requests.push( + service + .request('multi_fail_lock', async () => { + executed.push(`fail-${i}`); + throw new Error(`Request ${i} failed`); + }) + .catch(() => { + // Swallow error + }), + ); + } + + // One success at the end + requests.push( + service.request('multi_fail_lock', async () => { + executed.push('success'); + }), + ); + + await Promise.all(requests); + + // All should have attempted + expect(executed.filter((e) => e.startsWith('fail-')).length).toBe(3); + expect(executed).toContain('success'); + }); + }); });