diff --git a/docs/sync-and-op-log/operation-log-architecture.md b/docs/sync-and-op-log/operation-log-architecture.md index fd663c96bb..57592e02b3 100644 --- a/docs/sync-and-op-log/operation-log-architecture.md +++ b/docs/sync-and-op-log/operation-log-architecture.md @@ -2324,7 +2324,7 @@ When adding new entities or relationships: > - **Archive validation**: archiveOld tasks now validated for project/tag references, null-safety added > - **Lock service robustness**: Handle NaN timestamps and invalid lock formats in fallback lock > - **Array payload rejection**: Explicit check to reject arrays (which bypass `typeof === 'object'`) -> - **Pending operation expiry**: Operations pending >24h are quarantined as failed; reducer replay is skipped but archive recovery and the sync gate remain active (`PENDING_OPERATION_EXPIRY_MS`). +> - **Pending operation expiry**: Operations pending >24h are quarantined as failed; hydration still restores their reducer history, while archive recovery and the sync gate remain active (`PENDING_OPERATION_EXPIRY_MS`). --- diff --git a/docs/sync-and-op-log/operation-rules.md b/docs/sync-and-op-log/operation-rules.md index ea837e8dce..455cf1d3c2 100644 --- a/docs/sync-and-op-log/operation-rules.md +++ b/docs/sync-and-op-log/operation-rules.md @@ -193,15 +193,15 @@ export class MyEffects { See `operation-log.const.ts` for all configurable values: -| Constant | Value | Description | -| ----------------------------------- | -------- | ----------------------------------------- | -| `COMPACTION_TRIGGER` | 500 ops | Operations before automatic compaction | -| `COMPACTION_RETENTION_MS` | 7 days | Synced ops older than this may be deleted | -| `EMERGENCY_COMPACTION_RETENTION_MS` | 1 day | Shorter retention for quota exceeded | -| `MAX_COMPACTION_FAILURES` | 3 | Failures before user notification | -| `MAX_DOWNLOAD_OPS_IN_MEMORY` | 50,000 | Bounds memory during API download | -| `REMOTE_OP_FILE_RETENTION_MS` | 14 days | Server-side operation file retention | -| `PENDING_OPERATION_EXPIRY_MS` | 24 hours | Pending ops older than this are rejected | +| Constant | Value | Description | +| ----------------------------------- | -------- | -------------------------------------------------------------------------- | +| `COMPACTION_TRIGGER` | 500 ops | Operations before automatic compaction | +| `COMPACTION_RETENTION_MS` | 7 days | Synced ops older than this may be deleted | +| `EMERGENCY_COMPACTION_RETENTION_MS` | 1 day | Shorter retention for quota exceeded | +| `MAX_COMPACTION_FAILURES` | 3 | Failures before user notification | +| `MAX_DOWNLOAD_OPS_IN_MEMORY` | 50,000 | Bounds memory during API download | +| `REMOTE_OP_FILE_RETENTION_MS` | 14 days | Server-side operation file retention | +| `PENDING_OPERATION_EXPIRY_MS` | 24 hours | Pending ops older than this are quarantined as failed for archive recovery | ## 7. Quick Reference Checklist diff --git a/docs/sync-and-op-log/supersync-scenarios.md b/docs/sync-and-op-log/supersync-scenarios.md index 3d3a315a40..0b70f61e78 100644 --- a/docs/sync-and-op-log/supersync-scenarios.md +++ b/docs/sync-and-op-log/supersync-scenarios.md @@ -165,7 +165,7 @@ Comprehensive spec of all scenarios that can occur during SuperSync synchronizat **Expected:** 1. Download detects remote snapshot (file-based sync path) -2. Treat every pending op as meaningful except onboarding example-task creates and, before the first completed sync only, the GLOBAL_CONFIG write that enables the `sync` section. This protects non-task entities, recovered MIGRATION/RECOVERY genesis batches, and user configuration. +2. Treat every pending op as meaningful except onboarding example-task creates. GLOBAL_CONFIG writes remain protected even before the first completed sync because the sync-section payload can also contain user preferences. This also protects non-task entities and recovered MIGRATION/RECOVERY genesis batches. 3. If meaningful → throw `LocalDataConflictError` → full conflict dialog 4. If only the explicitly discardable startup ops remain → proceed without dialog diff --git a/src/app/op-log/core/operation-log.const.ts b/src/app/op-log/core/operation-log.const.ts index f19d0c3fa1..5355851a0a 100644 --- a/src/app/op-log/core/operation-log.const.ts +++ b/src/app/op-log/core/operation-log.const.ts @@ -154,7 +154,8 @@ export const MAX_BATCH_OPERATIONS_SIZE = 50; /** * Maximum age for pending operations before they are quarantined as failed (milliseconds). * If an operation has been pending for longer than this (e.g., due to data corruption - * or repeated crashes), reducer replay is skipped and archive recovery remains required. + * or repeated crashes), hydration still restores its reducer history and archive recovery + * remains required. * Default: 24 hours - enough time for legitimate recovery scenarios */ export const PENDING_OPERATION_EXPIRY_MS = 24 * 60 * 60 * 1000; diff --git a/src/app/op-log/persistence/db-keys.const.ts b/src/app/op-log/persistence/db-keys.const.ts index 7d59f6319f..dcce2bd4f1 100644 --- a/src/app/op-log/persistence/db-keys.const.ts +++ b/src/app/op-log/persistence/db-keys.const.ts @@ -54,6 +54,13 @@ export const FULL_STATE_OPS_META_KEY = 'full_state_ops' as const; */ export const RAW_REBUILD_INCOMPLETE_META_KEY = 'raw_rebuild_incomplete' as const; +/** Versioned marker for the one-time legacy terminal remote failure repair. */ +export const LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY = + 'legacy_terminal_remote_failures_migration' as const; + +/** Increment when the legacy terminal remote failure repair needs to run again. */ +export const LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION = 1; + /** Index names for ops object store */ export const OPS_INDEXES = { BY_ID: 'byId' as const, diff --git a/src/app/op-log/persistence/operation-log-store.service.spec.ts b/src/app/op-log/persistence/operation-log-store.service.spec.ts index 40e099fb63..c0ef012b90 100644 --- a/src/app/op-log/persistence/operation-log-store.service.spec.ts +++ b/src/app/op-log/persistence/operation-log-store.service.spec.ts @@ -1608,6 +1608,27 @@ describe('OperationLogStoreService', () => { expect(recovered.applicationStatus).toBe('failed'); }); + it('should run the legacy terminal remote failure repair only once', async () => { + const firstLegacyOp = createTestOperation({ entityId: 'first-legacy' }); + await service.append(firstLegacyOp, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([firstLegacyOp.id]); + } + await service.markRejected([firstLegacyOp.id]); + + expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(1); + + const laterLegacyOp = createTestOperation({ entityId: 'later-legacy' }); + await service.append(laterLegacyOp, 'remote', { pendingApply: true }); + for (let retry = 0; retry < 4; retry++) { + await service.markFailed([laterLegacyOp.id]); + } + await service.markRejected([laterLegacyOp.id]); + + expect(await service.recoverLegacyTerminalRemoteFailures()).toBe(0); + expect((await service.getOpById(laterLegacyOp.id))?.rejectedAt).toBeDefined(); + }); + it('should handle empty array', async () => { await service.markFailed([]); // Should not throw diff --git a/src/app/op-log/persistence/operation-log-store.service.ts b/src/app/op-log/persistence/operation-log-store.service.ts index af7ce7ef65..cc02a797a8 100644 --- a/src/app/op-log/persistence/operation-log-store.service.ts +++ b/src/app/op-log/persistence/operation-log-store.service.ts @@ -18,6 +18,8 @@ import { SINGLETON_KEY, BACKUP_KEY, FULL_STATE_OPS_META_KEY, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION, RAW_REBUILD_INCOMPLETE_META_KEY, OPS_INDEXES, ArchiveStoreEntry, @@ -86,6 +88,15 @@ export interface RawRebuildIncompleteEntry { preservedLocalOps: Operation[]; } +interface LegacyTerminalRemoteFailuresMigrationEntry { + version: number; +} + +type OpLogMetaEntry = + | FullStateOpsMetaEntry + | RawRebuildIncompleteEntry + | LegacyTerminalRemoteFailuresMigrationEntry; + /** * Stored operation log entry that can hold either compact or full operation format. * Used internally for backwards compatibility with existing data. @@ -211,7 +222,7 @@ interface OpLogDB extends DBSchema { */ [STORE_NAMES.META]: { key: string; - value: FullStateOpsMetaEntry; + value: OpLogMetaEntry; }; } @@ -1122,21 +1133,43 @@ export class OperationLogStoreService implements RemoteOperationApplyStorePort { await this._ensureInit(); let recoveredCount = 0; - await this._adapter.transaction([STORE_NAMES.OPS], 'readwrite', async (tx) => { - const entries = await tx.getAll(STORE_NAMES.OPS); - for (const entry of entries) { + await this._adapter.transaction( + [STORE_NAMES.OPS, STORE_NAMES.META], + 'readwrite', + async (tx) => { + const migration = await tx.get( + STORE_NAMES.META, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, + ); if ( - entry.source === 'remote' && - entry.rejectedAt !== undefined && - (entry.retryCount ?? 0) >= 4 + (migration?.version ?? 0) >= LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION ) { - entry.rejectedAt = undefined; - entry.applicationStatus = 'failed'; - await tx.put(STORE_NAMES.OPS, entry); - recoveredCount++; + return; } - } - }); + + const entries = await tx.getAll(STORE_NAMES.OPS); + for (const entry of entries) { + if ( + entry.source === 'remote' && + entry.rejectedAt !== undefined && + (entry.retryCount ?? 0) >= 4 + ) { + entry.rejectedAt = undefined; + entry.applicationStatus = 'failed'; + await tx.put(STORE_NAMES.OPS, entry); + recoveredCount++; + } + } + + await tx.put( + STORE_NAMES.META, + { + version: LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_VERSION, + } satisfies LegacyTerminalRemoteFailuresMigrationEntry, + LEGACY_TERMINAL_REMOTE_FAILURES_MIGRATION_META_KEY, + ); + }, + ); return recoveredCount; }