mirror of
https://github.com/johannesjo/super-productivity.git
synced 2026-01-23 02:36:05 +00:00
docs: update operation log architecture paths
This commit is contained in:
Johannes Millan
parent
fce75317e2
commit
4e018a8064
8 changed files with 3819 additions and 86 deletions
|
|
@ -605,7 +605,8 @@ Remote Storage Layout (v2):
|
|||
Code Files:
|
||||
src/app/core/persistence/operation-log/
|
||||
├── operation.types.ts # Add HybridManifest types
|
||||
├── operation-log-sync.service.ts # Buffer/overflow logic
|
||||
├── sync/
|
||||
│ └── operation-log-sync.service.ts # Buffer/overflow logic
|
||||
├── hybrid-snapshot.service.ts # NEW: Snapshot generation/loading
|
||||
└── manifest-recovery.service.ts # NEW: Corruption recovery
|
||||
```
|
||||
|
|
|
|||
|
|
@ -21,13 +21,13 @@ graph TD
|
|||
Filter -- No --> Ignore[Ignore / UI Only]
|
||||
Filter -- Yes --> Transform["Transform to Operation<br/>UUIDv7, Timestamp, VectorClock<br/><sub>operation-converter.util.ts</sub>"]
|
||||
|
||||
Transform -->|2. Validate| PayloadValid{"Payload<br/>Valid?<br/><sub>validate-operation-payload.ts</sub>"}
|
||||
Transform -->|2. Validate| PayloadValid{"Payload<br/>Valid?<br/><sub>processing/validate-operation-payload.ts</sub>"}
|
||||
PayloadValid -- No --> ErrorSnack[Show Error Snackbar]
|
||||
PayloadValid -- Yes --> DBWrite
|
||||
end
|
||||
|
||||
subgraph "Persistence Layer (IndexedDB)"
|
||||
DBWrite["Write to SUP_OPS<br/><sub>operation-log-store.service.ts</sub>"]:::storage
|
||||
DBWrite["Write to SUP_OPS<br/><sub>store/operation-log-store.service.ts</sub>"]:::storage
|
||||
|
||||
DBWrite -->|Append| OpsTable["Table: ops<br/>The Event Log<br/><sub>IndexedDB</sub>"]:::storage
|
||||
DBWrite -->|Update| StateCache["Table: state_cache<br/>Snapshots<br/><sub>IndexedDB</sub>"]:::storage
|
||||
|
|
@ -41,18 +41,18 @@ graph TD
|
|||
|
||||
subgraph "Compaction System"
|
||||
OpsTable -->|Count > 500| CompactionTrig{"Compaction<br/>Trigger<br/><sub>operation-log.effects.ts</sub>"}:::trigger
|
||||
CompactionTrig -->|Yes| Compactor["CompactionService<br/><sub>operation-log-compaction.service.ts</sub>"]:::process
|
||||
CompactionTrig -->|Yes| Compactor["CompactionService<br/><sub>store/operation-log-compaction.service.ts</sub>"]:::process
|
||||
Compactor -->|Read State| NgRx
|
||||
Compactor -->|Save Snapshot| StateCache
|
||||
Compactor -->|Delete Old Ops| OpsTable
|
||||
end
|
||||
|
||||
subgraph "Read Path (Hydration)"
|
||||
Startup((App Startup)) --> Hydrator["OperationLogHydrator<br/><sub>operation-log-hydrator.service.ts</sub>"]:::process
|
||||
Startup((App Startup)) --> Hydrator["OperationLogHydrator<br/><sub>store/operation-log-hydrator.service.ts</sub>"]:::process
|
||||
Hydrator -->|1. Load| StateCache
|
||||
|
||||
StateCache -->|Check| Schema{"Schema<br/>Version?<br/><sub>schema-migration.service.ts</sub>"}
|
||||
Schema -- Old --> Migrator["SchemaMigrationService<br/><sub>schema-migration.service.ts</sub>"]:::process
|
||||
StateCache -->|Check| Schema{"Schema<br/>Version?<br/><sub>store/schema-migration.service.ts</sub>"}
|
||||
Schema -- Old --> Migrator["SchemaMigrationService<br/><sub>store/schema-migration.service.ts</sub>"]:::process
|
||||
Migrator -->|Transform State| MigratedState
|
||||
Schema -- Current --> CurrentState
|
||||
|
||||
|
|
@ -60,12 +60,12 @@ graph TD
|
|||
MigratedState -->|Load State| StoreInit
|
||||
|
||||
Hydrator -->|2. Load Tail| OpsTable
|
||||
OpsTable -->|Replay Ops| Replayer["OperationApplier<br/><sub>operation-applier.service.ts</sub>"]:::process
|
||||
OpsTable -->|Replay Ops| Replayer["OperationApplier<br/><sub>processing/operation-applier.service.ts</sub>"]:::process
|
||||
Replayer -->|Dispatch| NgRx
|
||||
end
|
||||
|
||||
subgraph "Multi-Tab"
|
||||
DBWrite -->|4. Broadcast| BC["BroadcastChannel<br/><sub>multi-tab-coordinator.service.ts</sub>"]
|
||||
DBWrite -->|4. Broadcast| BC["BroadcastChannel<br/><sub>sync/multi-tab-coordinator.service.ts</sub>"]
|
||||
BC -->|Notify| OtherTabs((Other Tabs))
|
||||
end
|
||||
|
||||
|
|
@ -91,9 +91,9 @@ graph TD
|
|||
end
|
||||
|
||||
subgraph "Client: Sync Loop"
|
||||
Scheduler((Scheduler)) -->|Interval| SyncService["OperationLogSyncService<br/><sub>operation-log-sync.service.ts</sub>"]
|
||||
Scheduler((Scheduler)) -->|Interval| SyncService["OperationLogSyncService<br/><sub>sync/operation-log-sync.service.ts</sub>"]
|
||||
|
||||
SyncService -->|1. Get Last Seq| LocalMeta["Sync Metadata<br/><sub>operation-log-store.service.ts</sub>"]
|
||||
SyncService -->|1. Get Last Seq| LocalMeta["Sync Metadata<br/><sub>store/operation-log-store.service.ts</sub>"]
|
||||
|
||||
%% Download Flow
|
||||
SyncService -->|2. Download Ops| API
|
||||
|
|
@ -105,7 +105,7 @@ graph TD
|
|||
end
|
||||
|
||||
subgraph "Client: Conflict Management"
|
||||
ConflictDet{"Conflict<br/>Detection<br/><sub>conflict-resolution.service.ts</sub>"}:::conflict
|
||||
ConflictDet{"Conflict<br/>Detection<br/><sub>sync/conflict-resolution.service.ts</sub>"}:::conflict
|
||||
|
||||
ConflictDet -->|Check Vector Clocks| VCCheck[Entity-Level Check]
|
||||
|
||||
|
|
@ -125,10 +125,10 @@ graph TD
|
|||
subgraph "Client: Application & Validation"
|
||||
ApplyRemote -->|Apply to Store| Store[NgRx Store]
|
||||
|
||||
Store -->|Post-Apply| Validator{"Validate<br/>State?<br/><sub>validate-state.service.ts</sub>"}:::repair
|
||||
Store -->|Post-Apply| Validator{"Validate<br/>State?<br/><sub>processing/validate-state.service.ts</sub>"}:::repair
|
||||
|
||||
Validator -- Valid --> Done((Sync Done))
|
||||
Validator -- Invalid --> Repair["Auto-Repair Service<br/><sub>repair-operation.service.ts</sub>"]:::repair
|
||||
Validator -- Invalid --> Repair["Auto-Repair Service<br/><sub>processing/repair-operation.service.ts</sub>"]:::repair
|
||||
|
||||
Repair -->|Fix Data| RepairedState
|
||||
Repair -->|Create Op| RepairOp[Create REPAIR Op]:::repair
|
||||
|
|
|
|||
|
|
@ -1763,23 +1763,28 @@ What if data exists in both `pf` AND `SUP_OPS` databases?
|
|||
```
|
||||
src/app/core/persistence/operation-log/
|
||||
├── operation.types.ts # Type definitions (Operation, OpType, EntityType)
|
||||
├── operation-log-store.service.ts # SUP_OPS IndexedDB wrapper
|
||||
├── operation-log.const.ts # Constants
|
||||
├── operation-log.effects.ts # Action capture + META_MODEL bridge
|
||||
├── operation-log-hydrator.service.ts # Startup hydration
|
||||
├── operation-log-compaction.service.ts # Snapshot + cleanup
|
||||
├── operation-log-migration.service.ts # Genesis migration from legacy
|
||||
├── operation-log-sync.service.ts # Upload/download operations (Part C)
|
||||
├── operation-applier.service.ts # Apply ops to store with dependency handling
|
||||
├── operation-converter.util.ts # Op ↔ Action conversion
|
||||
├── persistent-action.interface.ts # PersistentAction type + isPersistentAction guard
|
||||
├── lock.service.ts # Cross-tab locking (Web Locks + fallback)
|
||||
├── multi-tab-coordinator.service.ts # BroadcastChannel coordination
|
||||
├── schema-migration.service.ts # State schema migrations
|
||||
├── dependency-resolver.service.ts # Extract/check operation dependencies
|
||||
├── conflict-resolution.service.ts # Conflict UI presentation
|
||||
├── validate-state.service.ts # Typia + cross-model validation wrapper
|
||||
├── validate-operation-payload.ts # Checkpoint A - payload validation
|
||||
└── repair-operation.service.ts # REPAIR operation creation + notification
|
||||
├── entity-key.util.ts # Entity key generation utilities
|
||||
├── store/
|
||||
│ ├── operation-log-store.service.ts # SUP_OPS IndexedDB wrapper
|
||||
│ ├── operation-log-hydrator.service.ts # Startup hydration
|
||||
│ ├── operation-log-compaction.service.ts # Snapshot + cleanup
|
||||
│ ├── operation-log-migration.service.ts # Genesis migration from legacy
|
||||
│ └── schema-migration.service.ts # State schema migrations
|
||||
├── sync/
|
||||
│ ├── operation-log-sync.service.ts # Upload/download operations (Part C)
|
||||
│ ├── lock.service.ts # Cross-tab locking (Web Locks + fallback)
|
||||
│ ├── multi-tab-coordinator.service.ts # BroadcastChannel coordination
|
||||
│ ├── dependency-resolver.service.ts # Extract/check operation dependencies
|
||||
│ └── conflict-resolution.service.ts # Conflict UI presentation
|
||||
└── processing/
|
||||
├── operation-applier.service.ts # Apply ops to store with dependency handling
|
||||
├── validate-state.service.ts # Typia + cross-model validation wrapper
|
||||
├── validate-operation-payload.ts # Checkpoint A - payload validation
|
||||
└── repair-operation.service.ts # REPAIR operation creation + notification
|
||||
|
||||
src/app/pfapi/
|
||||
├── pfapi-store-delegate.service.ts # Reads NgRx for sync (Part B)
|
||||
|
|
|
|||
|
|
@ -1,58 +0,0 @@
|
|||
# Operation Log: Remaining Tasks & Future Enhancements
|
||||
|
||||
**Status:** Core Implementation Complete (Parts A, B, C, D)
|
||||
**Branch:** `feat/operation-logs`
|
||||
**Last Updated:** December 3, 2025
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The core Operation Log architecture (Local Persistence, Legacy Bridge, Server Sync, Validation & Repair) is fully implemented and operational. This document tracks future enhancements and optimizations.
|
||||
|
||||
---
|
||||
|
||||
## 1. Performance & Storage Optimizations
|
||||
|
||||
| Enhancement | Description | Priority | Effort |
|
||||
| ---------------------------- | ------------------------------------------------------------------------- | -------- | ------ |
|
||||
| **IndexedDB index** | Add index on `syncedAt` for O(1) `getUnsynced()` queries | Low | Low |
|
||||
| **Persistent compaction** | Track `opsSinceCompaction` counter in DB to persist across restarts | Low | Low |
|
||||
| **Optimize getAppliedOpIds** | Consider Merkle trees or Bloom filters if log grows very large (>10k ops) | Low | Medium |
|
||||
| **Diff-based storage** | Store diffs (e.g., diff-match-patch) for large text fields (Notes) | Defer | High |
|
||||
|
||||
## 2. Observability & Tooling
|
||||
|
||||
| Enhancement | Description | Priority | Effort |
|
||||
| ----------------- | --------------------------------------------------------------------------- | -------- | ------ |
|
||||
| **Op Log Viewer** | Hidden debug panel (in Settings → About) to view/inspect raw operation logs | Medium | Medium |
|
||||
|
||||
**Implementation Idea:**
|
||||
|
||||
- Tab showing total ops, pending ops, last sync time, vector clock.
|
||||
- List of recent operations (seq, id, type, timestamp) with JSON expansion.
|
||||
|
||||
## 3. Feature Enhancements
|
||||
|
||||
| Enhancement | Description | Priority | Effort |
|
||||
| -------------- | ----------------------------------------------------------------------- | -------- | ------ |
|
||||
| **Auto-merge** | Automatically merge non-conflicting field changes on the same entity | Low | High |
|
||||
| **Undo/Redo** | Leverage the operation log history to implement robust global Undo/Redo | Low | High |
|
||||
|
||||
## 4. Migration System Improvements
|
||||
|
||||
Refinements for the Schema Migration system (Part A.7).
|
||||
|
||||
| Enhancement | Description | Priority | Effort |
|
||||
| ---------------------------- | --------------------------------------------- | -------- | ------ |
|
||||
| **Operation migration** | Transform old ops to new schema during replay | Low | High |
|
||||
| **Conflict-aware migration** | Special handling for version conflicts | Medium | High |
|
||||
| **Migration rollback** | Undo migration if it fails partway | Low | Medium |
|
||||
| **Progressive migration** | Migrate in background over multiple sessions | Low | High |
|
||||
|
||||
---
|
||||
|
||||
# References
|
||||
|
||||
- [Architecture](./operation-log-architecture.md) - Complete System Design
|
||||
- [PFAPI Architecture](./pfapi-sync-persistence-architecture.md) - Legacy Sync System
|
||||
|
|
@ -0,0 +1,612 @@
|
|||
# Hybrid Manifest & Snapshot Architecture for File-Based Sync
|
||||
|
||||
**Status:** Proposal / Planned
|
||||
**Context:** Optimizing WebDAV/Dropbox sync for the Operation Log architecture.
|
||||
**Related:** [Operation Log Architecture](./operation-log-architecture.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. The Problem
|
||||
|
||||
The current `OperationLogSyncService` fallback for file-based providers (WebDAV, Dropbox) is inefficient for frequent, small updates.
|
||||
|
||||
**Current Workflow (Naive Fallback):**
|
||||
|
||||
1. **Write Operation File:** Upload `ops/ops_CLIENT_TIMESTAMP.json`.
|
||||
2. **Read Manifest:** Download `ops/manifest.json` to get current list.
|
||||
3. **Update Manifest:** Upload new `ops/manifest.json` with the new filename added.
|
||||
|
||||
**Issues:**
|
||||
|
||||
- **High Request Count:** Minimum 3 HTTP requests per sync cycle.
|
||||
- **File Proliferation:** Rapidly creates thousands of small files, degrading WebDAV directory listing performance.
|
||||
- **Latency:** On slow connections (standard WebDAV), this makes sync feel sluggish.
|
||||
|
||||
---
|
||||
|
||||
## 2. Proposed Solution: Hybrid Manifest
|
||||
|
||||
Instead of treating the manifest solely as an _index_ of files, we treat it as a **buffer** for recent operations.
|
||||
|
||||
### 2.1. Concept
|
||||
|
||||
- **Embedded Operations:** Small batches of operations are stored directly inside `manifest.json`.
|
||||
- **Lazy Flush:** New operation files (`ops_*.json`) are only created when the manifest buffer fills up.
|
||||
- **Snapshots:** A "base state" file allows us to delete old operation files and clear the manifest history.
|
||||
|
||||
### 2.2. Data Structures
|
||||
|
||||
**Updated Manifest:**
|
||||
|
||||
```typescript
|
||||
interface HybridManifest {
|
||||
version: 2;
|
||||
|
||||
// The baseline state (snapshot). If present, clients load this first.
|
||||
lastSnapshot?: SnapshotReference;
|
||||
|
||||
// Ops stored directly in the manifest (The Buffer)
|
||||
// Limit: ~50 ops or 100KB payload size
|
||||
embeddedOperations: EmbeddedOperation[];
|
||||
|
||||
// References to external operation files (The Overflow)
|
||||
// Older ops that were flushed out of the buffer
|
||||
operationFiles: OperationFileReference[];
|
||||
|
||||
// Merged vector clock from all embedded operations
|
||||
// Used for quick conflict detection without parsing all ops
|
||||
frontierClock: VectorClock;
|
||||
|
||||
// Last modification timestamp (for ETag-like cache invalidation)
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
interface SnapshotReference {
|
||||
fileName: string; // e.g. "snapshots/snap_1701234567890.json"
|
||||
schemaVersion: number; // Schema version of the snapshot
|
||||
vectorClock: VectorClock; // Clock state at snapshot time
|
||||
timestamp: number; // When snapshot was created
|
||||
}
|
||||
|
||||
interface OperationFileReference {
|
||||
fileName: string; // e.g. "ops/overflow_1701234567890.json"
|
||||
opCount: number; // Number of operations in file (for progress estimation)
|
||||
minSeq: number; // First operation's logical sequence in this file
|
||||
maxSeq: number; // Last operation's logical sequence
|
||||
}
|
||||
|
||||
// Embedded operations are lightweight - full Operation minus redundant fields
|
||||
interface EmbeddedOperation {
|
||||
id: string;
|
||||
actionType: string;
|
||||
opType: OpType;
|
||||
entityType: EntityType;
|
||||
entityId?: string;
|
||||
entityIds?: string[];
|
||||
payload: unknown;
|
||||
clientId: string;
|
||||
vectorClock: VectorClock;
|
||||
timestamp: number;
|
||||
schemaVersion: number;
|
||||
}
|
||||
```
|
||||
|
||||
**Snapshot File Format:**
|
||||
|
||||
```typescript
|
||||
interface SnapshotFile {
|
||||
version: 1;
|
||||
schemaVersion: number; // App schema version
|
||||
vectorClock: VectorClock; // Merged clock at snapshot time
|
||||
timestamp: number;
|
||||
data: AppDataComplete; // Full application state
|
||||
checksum?: string; // Optional SHA-256 for integrity verification
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Workflows
|
||||
|
||||
### 3.1. Upload (Write Path)
|
||||
|
||||
When a client has local pending operations to sync:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Upload Flow │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 1. Download manifest.json │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 2. Detect remote changes │
|
||||
│ (compare frontierClock) │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
▼ ▼
|
||||
Remote has new ops? No remote changes
|
||||
│ │
|
||||
▼ │
|
||||
Download & apply first ◄───────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 3. Check buffer capacity │
|
||||
│ embedded.length + pending │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
▼ ▼
|
||||
< BUFFER_LIMIT (50) >= BUFFER_LIMIT
|
||||
│ │
|
||||
▼ ▼
|
||||
Append to embedded Flush embedded to file
|
||||
│ + add pending to empty buffer
|
||||
│ │
|
||||
└───────────────┬───────────────┘
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 4. Check snapshot trigger │
|
||||
│ (operationFiles > 50 OR │
|
||||
│ total ops > 5000) │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
▼ ▼
|
||||
Trigger snapshot No snapshot needed
|
||||
│ │
|
||||
└───────────────┬───────────────┘
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 5. Upload manifest.json │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
**Detailed Steps:**
|
||||
|
||||
1. **Download Manifest:** Fetch `manifest.json` (or create empty v2 manifest if not found).
|
||||
2. **Detect Remote Changes:**
|
||||
- Compare `manifest.frontierClock` with local `lastSyncedClock`.
|
||||
- If remote has unseen changes → download and apply before uploading (prevents lost updates).
|
||||
3. **Evaluate Buffer:**
|
||||
- `BUFFER_LIMIT = 50` operations (configurable)
|
||||
- `BUFFER_SIZE_LIMIT = 100KB` payload size (prevents manifest bloat)
|
||||
4. **Strategy Selection:**
|
||||
- **Scenario A (Append):** If `embedded.length + pending.length < BUFFER_LIMIT`:
|
||||
- Append `pendingOps` to `manifest.embeddedOperations`.
|
||||
- Update `manifest.frontierClock` with merged clocks.
|
||||
- **Result:** 1 Write (manifest). Fast path.
|
||||
- **Scenario B (Overflow):** If buffer would exceed limit:
|
||||
- Upload `manifest.embeddedOperations` to new file `ops/overflow_TIMESTAMP.json`.
|
||||
- Add file reference to `manifest.operationFiles`.
|
||||
- Place `pendingOps` into now-empty `manifest.embeddedOperations`.
|
||||
- **Result:** 1 Upload (overflow file) + 1 Write (manifest).
|
||||
5. **Upload Manifest:** Write updated `manifest.json`.
|
||||
|
||||
### 3.2. Download (Read Path)
|
||||
|
||||
When a client checks for updates:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Download Flow │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 1. Download manifest.json │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 2. Quick-check: any changes? │
|
||||
│ Compare frontierClock │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
▼ ▼
|
||||
No changes (clocks equal) Changes detected
|
||||
│ │
|
||||
▼ ▼
|
||||
Done ┌────────────────────────┐
|
||||
│ 3. Need snapshot? │
|
||||
│ (local behind snapshot)│
|
||||
└────────────────────────┘
|
||||
│
|
||||
┌───────────────┴───────────────┐
|
||||
▼ ▼
|
||||
Download snapshot Skip to ops
|
||||
+ apply as base │
|
||||
│ │
|
||||
└───────────────┬───────────────┘
|
||||
▼
|
||||
┌────────────────────────┐
|
||||
│ 4. Download new op │
|
||||
│ files (filter seen) │
|
||||
└────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌────────────────────────┐
|
||||
│ 5. Apply embedded ops │
|
||||
│ (filter by op.id) │
|
||||
└────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌────────────────────────┐
|
||||
│ 6. Update local │
|
||||
│ lastSyncedClock │
|
||||
└────────────────────────┘
|
||||
```
|
||||
|
||||
**Detailed Steps:**
|
||||
|
||||
1. **Download Manifest:** Fetch `manifest.json`.
|
||||
2. **Quick-Check Changes:**
|
||||
- Compare `manifest.frontierClock` against local `lastSyncedClock`.
|
||||
- If clocks are equal → no changes, done.
|
||||
3. **Check Snapshot Needed:**
|
||||
- If local state is older than `manifest.lastSnapshot.vectorClock` → download snapshot first.
|
||||
- Apply snapshot as base state (replaces local state).
|
||||
4. **Download Operation Files:**
|
||||
- Filter `manifest.operationFiles` to only files with `maxSeq > localLastAppliedSeq`.
|
||||
- Download and parse each file.
|
||||
- Collect all operations.
|
||||
5. **Apply Embedded Operations:**
|
||||
- Filter `manifest.embeddedOperations` by `op.id` (skip already-applied).
|
||||
- Add to collected operations.
|
||||
6. **Apply All Operations:**
|
||||
- Sort by `vectorClock` (causal order).
|
||||
- Detect conflicts using existing `detectConflicts()` logic.
|
||||
- Apply non-conflicting ops; present conflicts to user.
|
||||
7. **Update Tracking:**
|
||||
- Set `localLastSyncedClock = manifest.frontierClock`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Snapshotting (Compaction)
|
||||
|
||||
To prevent unbounded growth of operation files, any client can trigger a snapshot.
|
||||
|
||||
### 4.1. Triggers
|
||||
|
||||
| Condition | Threshold | Rationale |
|
||||
| ------------------------------- | --------- | -------------------------------------- |
|
||||
| External `operationFiles` count | > 50 | Prevent WebDAV directory bloat |
|
||||
| Total operations since snapshot | > 5000 | Bound replay time for fresh installs |
|
||||
| Time since last snapshot | > 7 days | Ensure periodic cleanup |
|
||||
| Manifest size | > 500KB | Prevent manifest from becoming too big |
|
||||
|
||||
### 4.2. Process
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Snapshot Flow │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 1. Ensure full sync complete │
|
||||
│ (no pending local/remote) │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 2. Read current state from │
|
||||
│ NgRx (authoritative) │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 3. Generate snapshot file │
|
||||
│ + compute checksum │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 4. Upload snapshot file │
|
||||
│ (atomic, verify success) │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 5. Update manifest │
|
||||
│ - Set lastSnapshot │
|
||||
│ - Clear operationFiles │
|
||||
│ - Clear embeddedOperations │
|
||||
│ - Reset frontierClock │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 6. Upload manifest │
|
||||
└───────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────────────────┐
|
||||
│ 7. Cleanup (async, best- │
|
||||
│ effort): delete old files │
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
### 4.3. Snapshot Atomicity
|
||||
|
||||
**Problem:** If the client crashes between uploading snapshot and updating manifest, other clients won't see the new snapshot.
|
||||
|
||||
**Solution:** Snapshot files are immutable and safe to leave orphaned. The manifest is the source of truth. Cleanup is best-effort.
|
||||
|
||||
**Invariant:** Never delete the current `lastSnapshot` file until a new snapshot is confirmed.
|
||||
|
||||
---
|
||||
|
||||
## 5. Conflict Handling
|
||||
|
||||
The hybrid manifest doesn't change conflict detection - it still uses vector clocks. However, the `frontierClock` in the manifest enables **early conflict detection**.
|
||||
|
||||
### 5.1. Early Conflict Detection
|
||||
|
||||
Before downloading all operations, compare clocks:
|
||||
|
||||
```typescript
|
||||
const comparison = compareVectorClocks(localFrontierClock, manifest.frontierClock);
|
||||
|
||||
switch (comparison) {
|
||||
case VectorClockComparison.LESS_THAN:
|
||||
// Remote is ahead - safe to download
|
||||
break;
|
||||
case VectorClockComparison.GREATER_THAN:
|
||||
// Local is ahead - upload our changes
|
||||
break;
|
||||
case VectorClockComparison.CONCURRENT:
|
||||
// Potential conflicts - download ops for detailed analysis
|
||||
break;
|
||||
case VectorClockComparison.EQUAL:
|
||||
// No changes - skip download
|
||||
break;
|
||||
}
|
||||
```
|
||||
|
||||
### 5.2. Conflict Resolution
|
||||
|
||||
When conflicts are detected at the operation level, the existing `ConflictResolutionService` handles them. The hybrid manifest doesn't change this flow.
|
||||
|
||||
---
|
||||
|
||||
## 6. Edge Cases & Failure Modes
|
||||
|
||||
### 6.1. Concurrent Uploads (Race Condition)
|
||||
|
||||
**Scenario:** Two clients download the manifest simultaneously, both append ops, both upload.
|
||||
|
||||
**Problem:** Second upload overwrites first client's operations.
|
||||
|
||||
**Solution:** Use provider-specific mechanisms:
|
||||
|
||||
| Provider | Mechanism |
|
||||
| ----------- | ------------------------------------------- |
|
||||
| **Dropbox** | Use `update` mode with `rev` parameter |
|
||||
| **WebDAV** | Use `If-Match` header with ETag |
|
||||
| **Local** | File locking (already implemented in PFAPI) |
|
||||
|
||||
**Implementation:**
|
||||
|
||||
```typescript
|
||||
interface HybridManifest {
|
||||
// ... existing fields
|
||||
|
||||
// Optimistic concurrency control
|
||||
etag?: string; // Server-assigned revision (Dropbox rev, WebDAV ETag)
|
||||
}
|
||||
|
||||
async uploadManifest(manifest: HybridManifest, expectedEtag?: string): Promise<void> {
|
||||
// If expectedEtag provided, use conditional upload
|
||||
// On conflict (412 Precondition Failed), re-download and retry
|
||||
}
|
||||
```
|
||||
|
||||
### 6.2. Manifest Corruption
|
||||
|
||||
**Scenario:** Manifest JSON is invalid (partial write, encoding issue).
|
||||
|
||||
**Recovery Strategy:**
|
||||
|
||||
1. Attempt to parse manifest.
|
||||
2. On parse failure, check for backup manifest (`manifest.json.bak`).
|
||||
3. If no backup, reconstruct from operation files using `listFiles()`.
|
||||
4. If reconstruction fails, fall back to snapshot-only state.
|
||||
|
||||
```typescript
|
||||
async loadManifestWithRecovery(): Promise<HybridManifest> {
|
||||
try {
|
||||
return await this._loadRemoteManifest();
|
||||
} catch (parseError) {
|
||||
PFLog.warn('Manifest corrupted, attempting recovery...');
|
||||
|
||||
// Try backup
|
||||
try {
|
||||
return await this._loadBackupManifest();
|
||||
} catch {
|
||||
// Reconstruct from files
|
||||
return await this._reconstructManifestFromFiles();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 6.3. Snapshot File Missing
|
||||
|
||||
**Scenario:** Manifest references a snapshot that doesn't exist on the server.
|
||||
|
||||
**Recovery Strategy:**
|
||||
|
||||
1. Log error and notify user.
|
||||
2. Fall back to replaying all available operation files.
|
||||
3. If operation files also reference missing ops, show data loss warning.
|
||||
|
||||
### 6.4. Schema Version Mismatch
|
||||
|
||||
**Scenario:** Snapshot was created with schema version 3, but local app is version 2.
|
||||
|
||||
**Handling:**
|
||||
|
||||
- If `snapshot.schemaVersion > CURRENT_SCHEMA_VERSION + MAX_VERSION_SKIP`:
|
||||
- Reject snapshot, prompt user to update app.
|
||||
- If `snapshot.schemaVersion > CURRENT_SCHEMA_VERSION`:
|
||||
- Load with warning (some fields may be stripped by Typia).
|
||||
- If `snapshot.schemaVersion < CURRENT_SCHEMA_VERSION`:
|
||||
- Run migrations on loaded state.
|
||||
|
||||
### 6.5. Large Pending Operations
|
||||
|
||||
**Scenario:** User was offline for a week, has 500 pending operations.
|
||||
|
||||
**Handling:**
|
||||
|
||||
- Don't try to embed all 500 in manifest.
|
||||
- Batch into multiple overflow files (100 ops each).
|
||||
- Upload files first, then update manifest once.
|
||||
|
||||
```typescript
|
||||
const BATCH_SIZE = 100;
|
||||
const chunks = chunkArray(pendingOps, BATCH_SIZE);
|
||||
|
||||
for (const chunk of chunks) {
|
||||
await this._uploadOverflowFile(chunk);
|
||||
}
|
||||
|
||||
// Single manifest update at the end
|
||||
await this._uploadManifest(manifest);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Advantages Summary
|
||||
|
||||
| Metric | Current (v1) | Hybrid Manifest (v2) |
|
||||
| :---------------------- | :----------------------------------- | :---------------------------------------------------- |
|
||||
| **Requests per Sync** | 3 (Upload Op + Read Man + Write Man) | **1-2** (Read Man, optional Write) |
|
||||
| **Files on Server** | Unbounded growth | **Bounded** (1 Manifest + 0-50 Op Files + 1 Snapshot) |
|
||||
| **Fresh Install Speed** | O(n) - replay all ops | **O(1)** - load snapshot + small delta |
|
||||
| **Conflict Detection** | Must parse all ops | **Quick check** via frontierClock |
|
||||
| **Bandwidth per Sync** | ~2KB (op file) + manifest overhead | **~1KB** (manifest only for small changes) |
|
||||
| **Offline Resilience** | Good | **Same** (operations buffered locally) |
|
||||
|
||||
---
|
||||
|
||||
## 8. Implementation Plan
|
||||
|
||||
### Phase 1: Core Infrastructure
|
||||
|
||||
1. **Update Types** (`operation.types.ts`):
|
||||
|
||||
- Add `HybridManifest`, `SnapshotReference`, `OperationFileReference` interfaces.
|
||||
- Keep backward compatibility with existing `OperationLogManifest`.
|
||||
|
||||
2. **Manifest Handling** (`operation-log-sync.service.ts`):
|
||||
|
||||
- Update `_loadRemoteManifest()` to detect version and parse accordingly.
|
||||
- Add `_migrateV1ToV2Manifest()` for automatic upgrade.
|
||||
- Implement buffer/overflow logic in `_uploadPendingOpsViaFiles()`.
|
||||
|
||||
3. **Add FrontierClock Tracking**:
|
||||
- Merge vector clocks when adding embedded operations.
|
||||
- Store `lastSyncedFrontierClock` locally for quick-check.
|
||||
|
||||
### Phase 2: Snapshot Support
|
||||
|
||||
4. **Create `HybridSnapshotService`**:
|
||||
|
||||
- `generateSnapshot()`: Serialize current state + compute checksum.
|
||||
- `uploadSnapshot()`: Upload with retry logic.
|
||||
- `loadSnapshot()`: Download + validate + apply.
|
||||
|
||||
5. **Integrate Snapshot Triggers**:
|
||||
- Check conditions after each upload.
|
||||
- Add manual "Force Snapshot" option in settings for debugging.
|
||||
|
||||
### Phase 3: Robustness
|
||||
|
||||
6. **Optimistic Concurrency**:
|
||||
|
||||
- Implement ETag/rev-based conditional uploads.
|
||||
- Add retry-on-conflict logic.
|
||||
|
||||
7. **Recovery Logic**:
|
||||
- Manifest corruption recovery.
|
||||
- Missing file handling.
|
||||
- Schema migration for snapshots.
|
||||
|
||||
### Phase 4: Testing & Migration
|
||||
|
||||
8. **Add Tests**:
|
||||
|
||||
- Unit tests for buffer overflow logic.
|
||||
- Integration tests for multi-client scenarios.
|
||||
- Stress tests for large operation counts.
|
||||
|
||||
9. **Migration Path**:
|
||||
- v1 clients continue to work (read v2 manifest, ignore new fields).
|
||||
- v2 clients auto-upgrade v1 manifests on first write.
|
||||
|
||||
---
|
||||
|
||||
## 9. Configuration Constants
|
||||
|
||||
```typescript
|
||||
// Buffer limits
|
||||
const EMBEDDED_OP_LIMIT = 50; // Max operations in manifest buffer
|
||||
const EMBEDDED_SIZE_LIMIT_KB = 100; // Max payload size in KB
|
||||
|
||||
// Snapshot triggers
|
||||
const SNAPSHOT_FILE_THRESHOLD = 50; // Trigger when operationFiles exceeds this
|
||||
const SNAPSHOT_OP_THRESHOLD = 5000; // Trigger when total ops exceed this
|
||||
const SNAPSHOT_AGE_DAYS = 7; // Trigger if no snapshot in N days
|
||||
|
||||
// Batching
|
||||
const UPLOAD_BATCH_SIZE = 100; // Ops per overflow file
|
||||
|
||||
// Retry
|
||||
const MAX_UPLOAD_RETRIES = 3;
|
||||
const RETRY_DELAY_MS = 1000;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Open Questions
|
||||
|
||||
1. **Encryption:** Should snapshots be encrypted differently than operation files? (Same encryption is simpler)
|
||||
|
||||
2. **Compression:** Should we gzip the snapshot file? (Trade-off: smaller size vs. no partial reads)
|
||||
|
||||
3. **Checksum Verification:** Is SHA-256 overkill for snapshot integrity? (Consider CRC32 for speed)
|
||||
|
||||
4. **Clock Drift:** How to handle clients with significantly wrong system clocks? (Vector clocks help, but timestamps in snapshot could confuse users)
|
||||
|
||||
---
|
||||
|
||||
## 11. File Reference
|
||||
|
||||
```
|
||||
Remote Storage Layout (v2):
|
||||
├── manifest.json # HybridManifest (buffer + references)
|
||||
├── ops/
|
||||
│ ├── overflow_170123.json # Flushed operations (batches of 100)
|
||||
│ └── overflow_170456.json
|
||||
└── snapshots/
|
||||
└── snap_170789.json # Full state snapshot
|
||||
```
|
||||
|
||||
```
|
||||
Code Files:
|
||||
src/app/core/persistence/operation-log/
|
||||
├── operation.types.ts # Add HybridManifest types
|
||||
├── sync/
|
||||
│ └── operation-log-sync.service.ts # Buffer/overflow logic
|
||||
├── hybrid-snapshot.service.ts # NEW: Snapshot generation/loading
|
||||
└── manifest-recovery.service.ts # NEW: Corruption recovery
|
||||
```
|
||||
|
|
@ -0,0 +1,513 @@
|
|||
# Operation Log: Architecture Diagrams
|
||||
|
||||
## 1. Operation Log Architecture (Local Persistence & Legacy Bridge)
|
||||
|
||||
This diagram illustrates how user actions flow through the system, how they are persisted to IndexedDB (`SUP_OPS`), how the system hydrates on startup, and how it bridges to the legacy PFAPI system.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
%% Styles
|
||||
classDef storage fill:#f9f,stroke:#333,stroke-width:2px,color:black;
|
||||
classDef process fill:#e1f5fe,stroke:#0277bd,stroke-width:2px,color:black;
|
||||
classDef legacy fill:#fff3e0,stroke:#ef6c00,stroke-width:2px,stroke-dasharray: 5 5,color:black;
|
||||
classDef trigger fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px,color:black;
|
||||
|
||||
User((User / UI)) -->|Dispatch Action| NgRx["NgRx Store <br/> Runtime Source of Truth<br/><sub>*.effects.ts / *.reducer.ts</sub>"]
|
||||
|
||||
subgraph "Write Path (Runtime)"
|
||||
NgRx -->|Action Stream| OpEffects["OperationLogEffects<br/><sub>operation-log.effects.ts</sub>"]
|
||||
|
||||
OpEffects -->|1. Check isPersistent| Filter{"Is Persistent?<br/><sub>persistent-action.interface.ts</sub>"}
|
||||
Filter -- No --> Ignore[Ignore / UI Only]
|
||||
Filter -- Yes --> Transform["Transform to Operation<br/>UUIDv7, Timestamp, VectorClock<br/><sub>operation-converter.util.ts</sub>"]
|
||||
|
||||
Transform -->|2. Validate| PayloadValid{"Payload<br/>Valid?<br/><sub>processing/validate-operation-payload.ts</sub>"}
|
||||
PayloadValid -- No --> ErrorSnack[Show Error Snackbar]
|
||||
PayloadValid -- Yes --> DBWrite
|
||||
end
|
||||
|
||||
subgraph "Persistence Layer (IndexedDB)"
|
||||
DBWrite["Write to SUP_OPS<br/><sub>store/operation-log-store.service.ts</sub>"]:::storage
|
||||
|
||||
DBWrite -->|Append| OpsTable["Table: ops<br/>The Event Log<br/><sub>IndexedDB</sub>"]:::storage
|
||||
DBWrite -->|Update| StateCache["Table: state_cache<br/>Snapshots<br/><sub>IndexedDB</sub>"]:::storage
|
||||
end
|
||||
|
||||
subgraph "Legacy Bridge (PFAPI)"
|
||||
DBWrite -.->|3. Bridge| LegacyMeta["META_MODEL<br/>Vector Clock<br/><sub>pfapi.service.ts</sub>"]:::legacy
|
||||
LegacyMeta -.->|Update| LegacySync["Legacy Sync Adapters<br/>WebDAV / Dropbox / Local<br/><sub>pfapi.service.ts</sub>"]:::legacy
|
||||
noteLegacy[Updates Vector Clock so<br/>Legacy Sync detects changes]:::legacy
|
||||
end
|
||||
|
||||
subgraph "Compaction System"
|
||||
OpsTable -->|Count > 500| CompactionTrig{"Compaction<br/>Trigger<br/><sub>operation-log.effects.ts</sub>"}:::trigger
|
||||
CompactionTrig -->|Yes| Compactor["CompactionService<br/><sub>store/operation-log-compaction.service.ts</sub>"]:::process
|
||||
Compactor -->|Read State| NgRx
|
||||
Compactor -->|Save Snapshot| StateCache
|
||||
Compactor -->|Delete Old Ops| OpsTable
|
||||
end
|
||||
|
||||
subgraph "Read Path (Hydration)"
|
||||
Startup((App Startup)) --> Hydrator["OperationLogHydrator<br/><sub>store/operation-log-hydrator.service.ts</sub>"]:::process
|
||||
Hydrator -->|1. Load| StateCache
|
||||
|
||||
StateCache -->|Check| Schema{"Schema<br/>Version?<br/><sub>store/schema-migration.service.ts</sub>"}
|
||||
Schema -- Old --> Migrator["SchemaMigrationService<br/><sub>store/schema-migration.service.ts</sub>"]:::process
|
||||
Migrator -->|Transform State| MigratedState
|
||||
Schema -- Current --> CurrentState
|
||||
|
||||
CurrentState -->|Load State| StoreInit[Init NgRx State]
|
||||
MigratedState -->|Load State| StoreInit
|
||||
|
||||
Hydrator -->|2. Load Tail| OpsTable
|
||||
OpsTable -->|Replay Ops| Replayer["OperationApplier<br/><sub>processing/operation-applier.service.ts</sub>"]:::process
|
||||
Replayer -->|Dispatch| NgRx
|
||||
end
|
||||
|
||||
subgraph "Multi-Tab"
|
||||
DBWrite -->|4. Broadcast| BC["BroadcastChannel<br/><sub>sync/multi-tab-coordinator.service.ts</sub>"]
|
||||
BC -->|Notify| OtherTabs((Other Tabs))
|
||||
end
|
||||
|
||||
class OpsTable,StateCache storage;
|
||||
class LegacyMeta,LegacySync,noteLegacy legacy;
|
||||
```
|
||||
|
||||
## 2. Operation Log Sync Architecture (Server Sync)
|
||||
|
||||
This diagram details the flow for syncing individual operations with a server (`Part C`), including conflict detection, resolution strategies, and the validation loop (`Part D`).
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
%% Styles
|
||||
classDef remote fill:#e3f2fd,stroke:#1565c0,stroke-width:2px,color:black;
|
||||
classDef local fill:#fff,stroke:#333,stroke-width:2px,color:black;
|
||||
classDef conflict fill:#ffebee,stroke:#c62828,stroke-width:2px,color:black;
|
||||
classDef repair fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px,color:black;
|
||||
|
||||
subgraph "Remote Server"
|
||||
ServerDB[(Server Database)]:::remote
|
||||
API[Sync API Endpoint]:::remote
|
||||
end
|
||||
|
||||
subgraph "Client: Sync Loop"
|
||||
Scheduler((Scheduler)) -->|Interval| SyncService["OperationLogSyncService<br/><sub>sync/operation-log-sync.service.ts</sub>"]
|
||||
|
||||
SyncService -->|1. Get Last Seq| LocalMeta["Sync Metadata<br/><sub>store/operation-log-store.service.ts</sub>"]
|
||||
|
||||
%% Download Flow
|
||||
SyncService -->|2. Download Ops| API
|
||||
API -->|Return Ops > Seq| DownOps[Downloaded Operations]
|
||||
|
||||
DownOps --> FilterApplied{Already<br/>Applied?}
|
||||
FilterApplied -- Yes --> Discard[Discard]
|
||||
FilterApplied -- No --> ConflictDet
|
||||
end
|
||||
|
||||
subgraph "Client: Conflict Management"
|
||||
ConflictDet{"Conflict<br/>Detection<br/><sub>sync/conflict-resolution.service.ts</sub>"}:::conflict
|
||||
|
||||
ConflictDet -->|Check Vector Clocks| VCCheck[Entity-Level Check]
|
||||
|
||||
VCCheck -- Concurrent --> ConflictFound[Conflict Found!]:::conflict
|
||||
VCCheck -- Sequential --> NoConflict[No Conflict]
|
||||
|
||||
ConflictFound --> UserDialog["User Resolution Dialog<br/><sub>dialog-conflict-resolution.component.ts</sub>"]:::conflict
|
||||
|
||||
UserDialog -- "Keep Remote" --> MarkRejected[Mark Local Ops<br/>as Rejected]:::conflict
|
||||
MarkRejected --> ApplyRemote[Apply Remote Ops]
|
||||
|
||||
UserDialog -- "Keep Local" --> IgnoreRemote[Ignore Remote Ops]
|
||||
|
||||
NoConflict --> ApplyRemote
|
||||
end
|
||||
|
||||
subgraph "Client: Application & Validation"
|
||||
ApplyRemote -->|Apply to Store| Store[NgRx Store]
|
||||
|
||||
Store -->|Post-Apply| Validator{"Validate<br/>State?<br/><sub>processing/validate-state.service.ts</sub>"}:::repair
|
||||
|
||||
Validator -- Valid --> Done((Sync Done))
|
||||
Validator -- Invalid --> Repair["Auto-Repair Service<br/><sub>processing/repair-operation.service.ts</sub>"]:::repair
|
||||
|
||||
Repair -->|Fix Data| RepairedState
|
||||
Repair -->|Create Op| RepairOp[Create REPAIR Op]:::repair
|
||||
RepairOp -->|Append| OpLog[(SUP_OPS)]
|
||||
RepairedState -->|Update| Store
|
||||
end
|
||||
|
||||
subgraph "Client: Upload Flow"
|
||||
OpLog -->|Get Unsynced| PendingOps[Pending Ops]
|
||||
PendingOps -->|Filter| FilterRejected{Is<br/>Rejected?}
|
||||
|
||||
FilterRejected -- Yes --> Skip[Skip Upload]
|
||||
FilterRejected -- No --> UploadBatch[Batch for Upload]
|
||||
|
||||
UploadBatch -->|3. Upload| API
|
||||
API -->|Ack| ServerAck[Server Acknowledgement]
|
||||
ServerAck -->|Update| MarkSynced[Mark Ops Synced]
|
||||
MarkSynced --> OpLog
|
||||
end
|
||||
|
||||
API <--> ServerDB
|
||||
```
|
||||
|
||||
## 3. Conflict-Aware Migration Strategy (The Migration Shield)
|
||||
|
||||
> **Note:** Sections 3, 4.1, and 4.2 describe **planned architecture** that is not yet implemented. Currently, only state cache snapshots are migrated via `SchemaMigrationService.migrateIfNeeded()`. Individual operation migration (`migrateOperation()`) is not implemented—tail ops are replayed directly without per-operation migration.
|
||||
|
||||
This diagram visualizes the "Receiver-Side Migration" strategy. The Migration Layer acts as a shield, ensuring that _only_ operations matching the current schema version ever reach the core conflict detection and application logic.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
%% Nodes
|
||||
subgraph "Sources of Operations (Mixed Versions)"
|
||||
Remote[Remote Client Sync]:::src
|
||||
Disk[Local Disk Tail Ops]:::src
|
||||
end
|
||||
|
||||
subgraph "Migration Layer (The Shield)"
|
||||
Check{"Is Op Old?<br/>(vOp < vCurrent)"}:::logic
|
||||
Migrate["Run migrateOperation()<br/>Pipeline"]:::action
|
||||
CheckDrop{"Result is<br/>Null?"}:::logic
|
||||
Pass["Pass Through"]:::pass
|
||||
end
|
||||
|
||||
subgraph "Core System (Current Version Only)"
|
||||
Conflict["Conflict Detection<br/>(Apples-to-Apples)"]:::core
|
||||
Apply["Apply to State"]:::core
|
||||
end
|
||||
|
||||
%% Flow
|
||||
Remote --> Check
|
||||
Disk --> Check
|
||||
|
||||
Check -- Yes --> Migrate
|
||||
Check -- No --> Pass
|
||||
|
||||
Migrate --> CheckDrop
|
||||
CheckDrop -- Yes --> Drop[("🗑️ Drop Op<br/>(Destructive Change)")]:::drop
|
||||
CheckDrop -- No --> Conflict
|
||||
|
||||
Pass --> Conflict
|
||||
Conflict --> Apply
|
||||
|
||||
%% Styles
|
||||
classDef src fill:#fff3e0,stroke:#ef6c00,stroke-width:2px;
|
||||
classDef logic fill:#fff,stroke:#333,stroke-width:2px;
|
||||
classDef action fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px;
|
||||
classDef pass fill:#e3f2fd,stroke:#1565c0,stroke-width:2px;
|
||||
classDef drop fill:#ffebee,stroke:#c62828,stroke-width:2px,stroke-dasharray: 5 5;
|
||||
classDef core fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px;
|
||||
```
|
||||
|
||||
## 4. Migration Scenarios
|
||||
|
||||
### 4.1 Tail Ops Migration (Local Startup Consistency)
|
||||
|
||||
Ensures that operations occurring after a snapshot ("Tail Ops") are migrated to the current version before being applied to the migrated state.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant IDB as IndexedDB (SUP_OPS)
|
||||
participant Hydrator as OpLogHydrator
|
||||
participant Migrator as SchemaMigrationService
|
||||
participant Applier as OperationApplier
|
||||
participant Store as NgRx Store
|
||||
|
||||
Note over IDB, Store: App Updated from V1 -> V2
|
||||
|
||||
Hydrator->>IDB: Load Snapshot (Version 1)
|
||||
IDB-->>Hydrator: Returns Snapshot V1
|
||||
|
||||
Hydrator->>Migrator: migrateIfNeeded(Snapshot V1)
|
||||
Migrator-->>Hydrator: Returns Migrated Snapshot (Version 2)
|
||||
|
||||
Hydrator->>Store: Load Initial State (V2)
|
||||
|
||||
Hydrator->>IDB: Load Tail Ops (Version 1)
|
||||
Note right of IDB: Ops created after snapshot<br/>but before update
|
||||
IDB-->>Hydrator: Returns Ops [OpA(v1), OpB(v1)]
|
||||
|
||||
loop For Each Op
|
||||
Hydrator->>Migrator: migrateOperation(Op V1)
|
||||
Migrator-->>Hydrator: Returns Op V2 (or null)
|
||||
|
||||
alt Op was Dropped (null)
|
||||
Hydrator->>Hydrator: Ignore
|
||||
else Op Migrated
|
||||
Hydrator->>Applier: Apply(Op V2)
|
||||
Applier->>Store: Dispatch Action (V2 Payload)
|
||||
end
|
||||
end
|
||||
|
||||
Note over Store: State matches V2 Schema<br/>Consistency Preserved
|
||||
```
|
||||
|
||||
### 4.2 Receiver-Side Sync Migration
|
||||
|
||||
Demonstrates how a client on V2 handles incoming data from a client still on V1.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Remote as Remote Client (V1)
|
||||
participant Server as Sync Server
|
||||
participant Local as Local Client (V2)
|
||||
participant Conflict as Conflict Detector
|
||||
|
||||
Remote->>Server: Upload Operation (Version 1)<br/>{ payload: { oldField: 'X' } }
|
||||
Server-->>Local: Download Operation (Version 1)
|
||||
|
||||
Note over Local: Client V2 receives V1 data
|
||||
|
||||
Local->>Local: Check Op Schema Version (v1 < v2)
|
||||
Local->>Local: Call SchemaMigrationService.migrateOperation()
|
||||
|
||||
Note over Local: Transforms payload:<br/>{ oldField: 'X' } -> { newField: 'X' }
|
||||
|
||||
Local->>Conflict: detectConflicts(Remote Op V2)
|
||||
|
||||
alt Conflict Detected
|
||||
Conflict->>Local: Show Dialog (V2 vs V2 comparison)
|
||||
else No Conflict
|
||||
Local->>Local: Apply Operation (V2)
|
||||
end
|
||||
```
|
||||
|
||||
## 5. Hybrid Manifest (File-Based Sync)
|
||||
|
||||
This diagram illustrates the "Hybrid Manifest" optimization (`hybrid-manifest-architecture.md`) which reduces HTTP request overhead for WebDAV/Dropbox sync by buffering small operations directly inside the manifest file.
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
%% Nodes
|
||||
subgraph "Hybrid Manifest File (JSON)"
|
||||
ManVer[Version: 2]:::file
|
||||
SnapRef[Last Snapshot: 'snap_123.json']:::file
|
||||
Buffer[Embedded Ops Buffer<br/>Op1, Op2, ...]:::buffer
|
||||
ExtFiles[External Files List<br/>ops_A.json, ...]:::file
|
||||
end
|
||||
|
||||
subgraph "Sync Logic (Upload Path)"
|
||||
Start((Start Sync)) --> ReadMan[Download Manifest]
|
||||
ReadMan --> CheckSize{Buffer Full?<br/>more than 50 ops}
|
||||
|
||||
CheckSize -- No --> AppendBuffer[Append to<br/>Embedded Ops]:::action
|
||||
AppendBuffer --> WriteMan[Upload Manifest]:::io
|
||||
|
||||
CheckSize -- Yes --> Flush[Flush Buffer]:::action
|
||||
Flush --> CreateFile[Create 'ops_NEW.json'<br/>with old buffer content]:::io
|
||||
CreateFile --> UpdateRef[Add 'ops_NEW.json'<br/>to External Files]:::action
|
||||
UpdateRef --> ClearBuffer[Clear Buffer &<br/>Add Pending Ops]:::action
|
||||
ClearBuffer --> WriteMan
|
||||
end
|
||||
|
||||
%% Styles
|
||||
classDef file fill:#fff3e0,stroke:#ef6c00,stroke-width:2px;
|
||||
classDef buffer fill:#e8f5e9,stroke:#2e7d32,stroke-width:2px;
|
||||
classDef action fill:#e3f2fd,stroke:#1565c0,stroke-width:2px;
|
||||
classDef io fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px;
|
||||
```
|
||||
|
||||
## 6. Hybrid Manifest Conceptual Overview
|
||||
|
||||
This diagram shows the Hybrid Manifest architecture: how operations flow from "hot" (recent, in manifest) to "cold" (archived files) to "frozen" (snapshot), and the decision logic for each transition.
|
||||
|
||||
### 6.1 Data Lifecycle: Hot → Cold → Frozen
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph "HOT: Manifest Buffer"
|
||||
direction TB
|
||||
Buffer["embeddedOperations[]<br/>━━━━━━━━━━━━━━━<br/>• Op 47<br/>• Op 48<br/>• Op 49<br/>━━━━━━━━━━━━━━━<br/>~50 ops max"]
|
||||
end
|
||||
|
||||
subgraph "COLD: Operation Files"
|
||||
direction TB
|
||||
Files["operationFiles[]<br/>━━━━━━━━━━━━━━━<br/>• overflow_001.json<br/>• overflow_002.json<br/>• overflow_003.json<br/>━━━━━━━━━━━━━━━<br/>~50 files max"]
|
||||
end
|
||||
|
||||
subgraph "FROZEN: Snapshot"
|
||||
direction TB
|
||||
Snap["lastSnapshot<br/>━━━━━━━━━━━━━━━<br/>snap_170789.json<br/>━━━━━━━━━━━━━━━<br/>Full app state"]
|
||||
end
|
||||
|
||||
NewOp((New Op)) -->|"Always"| Buffer
|
||||
Buffer -->|"When full<br/>(overflow)"| Files
|
||||
Files -->|"When too many<br/>(compaction)"| Snap
|
||||
|
||||
style Buffer fill:#c8e6c9,stroke:#2e7d32,stroke-width:2px
|
||||
style Files fill:#fff3e0,stroke:#ef6c00,stroke-width:2px
|
||||
style Snap fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
|
||||
style NewOp fill:#fff,stroke:#333,stroke-width:2px
|
||||
```
|
||||
|
||||
### 6.2 Manifest File Structure
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph Manifest["manifest.json"]
|
||||
direction TB
|
||||
|
||||
V["version: 2"]
|
||||
FC["frontierClock: { A: 5, B: 3 }"]
|
||||
|
||||
subgraph SnapRef["lastSnapshot (optional)"]
|
||||
SF["fileName: 'snap_170789.json'"]
|
||||
SV["vectorClock: { A: 2, B: 1 }"]
|
||||
end
|
||||
|
||||
subgraph EmbeddedOps["embeddedOperations[] — THE BUFFER"]
|
||||
E1["Op { id: 'abc', entityType: 'TASK', ... }"]
|
||||
E2["Op { id: 'def', entityType: 'PROJECT', ... }"]
|
||||
E3["...up to 50 ops"]
|
||||
end
|
||||
|
||||
subgraph OpFiles["operationFiles[] — OVERFLOW REFERENCES"]
|
||||
F1["{ fileName: 'overflow_001.json', opCount: 100 }"]
|
||||
F2["{ fileName: 'overflow_002.json', opCount: 100 }"]
|
||||
end
|
||||
end
|
||||
|
||||
style Manifest fill:#fff,stroke:#333,stroke-width:3px
|
||||
style EmbeddedOps fill:#c8e6c9,stroke:#2e7d32,stroke-width:2px
|
||||
style OpFiles fill:#fff3e0,stroke:#ef6c00,stroke-width:2px
|
||||
style SnapRef fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
|
||||
```
|
||||
|
||||
### 6.3 Write Path: Buffer vs Overflow Decision
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Start([Client has pending ops]) --> Download[Download manifest.json]
|
||||
Download --> CheckRemote{Remote has<br/>new ops?}
|
||||
|
||||
CheckRemote -->|Yes| ApplyFirst[Download & apply<br/>remote ops first]
|
||||
ApplyFirst --> CheckBuffer
|
||||
CheckRemote -->|No| CheckBuffer
|
||||
|
||||
CheckBuffer{Buffer + Pending<br/>< 50 ops?}
|
||||
|
||||
CheckBuffer -->|Yes| FastPath
|
||||
CheckBuffer -->|No| SlowPath
|
||||
|
||||
subgraph FastPath["⚡ FAST PATH (1 request)"]
|
||||
Append[Append pending to<br/>embeddedOperations]
|
||||
Append --> Upload1[Upload manifest.json]
|
||||
end
|
||||
|
||||
subgraph SlowPath["📦 OVERFLOW PATH (2 requests)"]
|
||||
Flush[Upload embeddedOperations<br/>as overflow_XXX.json]
|
||||
Flush --> AddRef[Add file to operationFiles]
|
||||
AddRef --> Clear[Put pending ops in<br/>now-empty buffer]
|
||||
Clear --> Upload2[Upload manifest.json]
|
||||
end
|
||||
|
||||
Upload1 --> CheckSnap
|
||||
Upload2 --> CheckSnap
|
||||
|
||||
CheckSnap{Files > 50 OR<br/>Ops > 5000?}
|
||||
CheckSnap -->|Yes| Compact[Trigger Compaction]
|
||||
CheckSnap -->|No| Done([Done])
|
||||
Compact --> Done
|
||||
|
||||
style FastPath fill:#c8e6c9,stroke:#2e7d32,stroke-width:2px
|
||||
style SlowPath fill:#fff3e0,stroke:#ef6c00,stroke-width:2px
|
||||
style Start fill:#fff,stroke:#333
|
||||
style Done fill:#fff,stroke:#333
|
||||
```
|
||||
|
||||
### 6.4 Read Path: Reconstructing State
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Start([Client checks for updates]) --> Download[Download manifest.json]
|
||||
|
||||
Download --> QuickCheck{frontierClock<br/>changed?}
|
||||
QuickCheck -->|No| Done([No changes - done])
|
||||
|
||||
QuickCheck -->|Yes| NeedSnap{Local behind<br/>snapshot?}
|
||||
|
||||
NeedSnap -->|Yes| LoadSnap
|
||||
NeedSnap -->|No| LoadFiles
|
||||
|
||||
subgraph LoadSnap["🧊 Load Snapshot (fresh install / behind)"]
|
||||
DownSnap[Download snapshot file]
|
||||
DownSnap --> ApplySnap[Apply as base state]
|
||||
end
|
||||
|
||||
ApplySnap --> LoadFiles
|
||||
|
||||
subgraph LoadFiles["📁 Load Operation Files"]
|
||||
FilterFiles[Filter to unseen files only]
|
||||
FilterFiles --> DownFiles[Download each file]
|
||||
DownFiles --> CollectOps[Collect all operations]
|
||||
end
|
||||
|
||||
CollectOps --> LoadEmbed
|
||||
|
||||
subgraph LoadEmbed["⚡ Load Embedded Ops"]
|
||||
FilterEmbed[Filter by op.id<br/>skip already-applied]
|
||||
FilterEmbed --> AddOps[Add to collected ops]
|
||||
end
|
||||
|
||||
AddOps --> Apply
|
||||
|
||||
subgraph Apply["✅ Apply All"]
|
||||
Sort[Sort by vectorClock]
|
||||
Sort --> Detect[Detect conflicts]
|
||||
Detect --> ApplyOps[Apply non-conflicting]
|
||||
end
|
||||
|
||||
ApplyOps --> UpdateClock[Update local<br/>lastSyncedClock]
|
||||
UpdateClock --> Done2([Done])
|
||||
|
||||
style LoadSnap fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
|
||||
style LoadFiles fill:#fff3e0,stroke:#ef6c00,stroke-width:2px
|
||||
style LoadEmbed fill:#c8e6c9,stroke:#2e7d32,stroke-width:2px
|
||||
style Apply fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px
|
||||
```
|
||||
|
||||
### 6.5 Compaction: Freezing State
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Trigger{{"Trigger Conditions"}}
|
||||
Trigger --> C1["operationFiles > 50"]
|
||||
Trigger --> C2["Total ops > 5000"]
|
||||
Trigger --> C3["7+ days since snapshot"]
|
||||
|
||||
C1 --> Start
|
||||
C2 --> Start
|
||||
C3 --> Start
|
||||
|
||||
Start([Begin Compaction]) --> Sync[Ensure full sync<br/>no pending ops]
|
||||
Sync --> Read[Read current state<br/>from NgRx]
|
||||
Read --> Generate[Generate snapshot file<br/>+ checksum]
|
||||
Generate --> UpSnap[Upload snapshot file]
|
||||
|
||||
UpSnap --> UpdateMan
|
||||
|
||||
subgraph UpdateMan["Update Manifest"]
|
||||
SetSnap[Set lastSnapshot →<br/>new file reference]
|
||||
SetSnap --> ClearFiles[Clear operationFiles]
|
||||
ClearFiles --> ClearBuffer[Clear embeddedOperations]
|
||||
ClearBuffer --> ResetClock[Set frontierClock →<br/>snapshot's clock]
|
||||
end
|
||||
|
||||
UpdateMan --> UpMan[Upload manifest.json]
|
||||
UpMan --> Cleanup[Async: Delete old files<br/>from server]
|
||||
Cleanup --> Done([Done])
|
||||
|
||||
style Trigger fill:#ffebee,stroke:#c62828,stroke-width:2px
|
||||
style UpdateMan fill:#e3f2fd,stroke:#1565c0,stroke-width:2px
|
||||
```
|
||||
|
||||
### 6.6 Request Count Comparison
|
||||
|
||||
| Scenario | Old (v1) | Hybrid (v2) | Savings |
|
||||
| -------------------- | ---------------------- | -------------------------------- | ------- |
|
||||
| Small sync (1-5 ops) | 3 requests | **1 request** | 67% |
|
||||
| Buffer overflow | 3 requests | **2 requests** | 33% |
|
||||
| Fresh install | N requests (all files) | **2 requests** (snap + manifest) | ~95% |
|
||||
| No changes | 1 request (manifest) | **1 request** (manifest) | Same |
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,860 @@
|
|||
# PFAPI Sync and Persistence Architecture
|
||||
|
||||
This document describes the architecture and implementation of the persistence and synchronization system (PFAPI) in Super Productivity.
|
||||
|
||||
## Overview
|
||||
|
||||
PFAPI (Persistence Framework API) is a comprehensive system for:
|
||||
|
||||
1. **Local Persistence**: Storing application data in IndexedDB
|
||||
2. **Cross-Device Synchronization**: Syncing data across devices via multiple cloud providers
|
||||
3. **Conflict Detection**: Using vector clocks for distributed conflict detection
|
||||
4. **Data Validation & Migration**: Ensuring data integrity across versions
|
||||
|
||||
## Architecture Layers
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Angular Application │
|
||||
│ (Components & Services) │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ PfapiService (Angular) │
|
||||
│ - Injectable wrapper around Pfapi │
|
||||
│ - Exposes RxJS Observables for UI integration │
|
||||
│ - Manages sync provider activation │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Pfapi (Core) │
|
||||
│ - Main orchestrator for all persistence operations │
|
||||
│ - Coordinates Database, Models, Sync, and Migration │
|
||||
└────────────────────────────┬────────────────────────────────────┘
|
||||
│
|
||||
┌────────────────────┼────────────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
|
||||
│ Database │ │ SyncService │ │ Migration │
|
||||
│ (IndexedDB) │ │ (Orchestrator)│ │ Service │
|
||||
└───────────────┘ └───────┬───────┘ └───────────────┘
|
||||
│
|
||||
┌────────────┼────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌──────────┐ ┌───────────┐ ┌───────────┐
|
||||
│ Meta │ │ Model │ │ Encrypt/ │
|
||||
│ Sync │ │ Sync │ │ Compress │
|
||||
└──────────┘ └───────────┘ └───────────┘
|
||||
│ │
|
||||
└────────────┼────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
┌───────────────────────────┐
|
||||
│ SyncProvider Interface │
|
||||
└───────────────┬───────────┘
|
||||
│
|
||||
┌───────────────────────────┼───────────────────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
|
||||
│ Dropbox │ │ WebDAV │ │ Local File │
|
||||
└───────────────┘ └───────────────┘ └───────────────┘
|
||||
```
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
src/app/pfapi/
|
||||
├── pfapi.service.ts # Angular service wrapper
|
||||
├── pfapi-config.ts # Model and provider configuration
|
||||
├── pfapi-helper.ts # RxJS integration helpers
|
||||
├── api/
|
||||
│ ├── pfapi.ts # Main API class
|
||||
│ ├── pfapi.model.ts # Type definitions
|
||||
│ ├── pfapi.const.ts # Enums and constants
|
||||
│ ├── db/ # Database abstraction
|
||||
│ │ ├── database.ts # Database wrapper with locking
|
||||
│ │ ├── database-adapter.model.ts
|
||||
│ │ └── indexed-db-adapter.ts # IndexedDB implementation
|
||||
│ ├── model-ctrl/ # Model controllers
|
||||
│ │ ├── model-ctrl.ts # Generic model controller
|
||||
│ │ └── meta-model-ctrl.ts # Metadata controller
|
||||
│ ├── sync/ # Sync orchestration
|
||||
│ │ ├── sync.service.ts # Main sync orchestrator
|
||||
│ │ ├── meta-sync.service.ts # Metadata sync
|
||||
│ │ ├── model-sync.service.ts # Model sync
|
||||
│ │ ├── sync-provider.interface.ts
|
||||
│ │ ├── encrypt-and-compress-handler.service.ts
|
||||
│ │ └── providers/ # Provider implementations
|
||||
│ ├── migration/ # Data migration
|
||||
│ ├── util/ # Utilities (vector-clock, etc.)
|
||||
│ └── errors/ # Custom error types
|
||||
├── migrate/ # Cross-model migrations
|
||||
├── repair/ # Data repair utilities
|
||||
└── validate/ # Validation functions
|
||||
```
|
||||
|
||||
## Core Components
|
||||
|
||||
### 1. Database Layer
|
||||
|
||||
#### Database Class (`api/db/database.ts`)
|
||||
|
||||
The `Database` class wraps the storage adapter and provides:
|
||||
|
||||
- **Locking mechanism**: Prevents concurrent writes during sync
|
||||
- **Error handling**: Centralized error management
|
||||
- **CRUD operations**: `load`, `save`, `remove`, `loadAll`, `clearDatabase`
|
||||
|
||||
```typescript
|
||||
class Database {
|
||||
lock(): void; // Prevents writes
|
||||
unlock(): void; // Re-enables writes
|
||||
load<T>(key: string): Promise<T>;
|
||||
save<T>(key: string, data: T, isIgnoreDBLock?: boolean): Promise<void>;
|
||||
remove(key: string): Promise<unknown>;
|
||||
}
|
||||
```
|
||||
|
||||
The database is locked during sync operations to prevent race conditions.
|
||||
|
||||
#### IndexedDB Adapter (`api/db/indexed-db-adapter.ts`)
|
||||
|
||||
Implements `DatabaseAdapter` interface using IndexedDB:
|
||||
|
||||
- Database name: `'pf'`
|
||||
- Main store: `'main'`
|
||||
- Uses the `idb` library for async IndexedDB operations
|
||||
|
||||
```typescript
|
||||
class IndexedDbAdapter implements DatabaseAdapter {
|
||||
async init(): Promise<IDBPDatabase>; // Opens/creates database
|
||||
async load<T>(key: string): Promise<T>; // db.get(store, key)
|
||||
async save<T>(key: string, data: T): Promise<void>; // db.put(store, data, key)
|
||||
async remove(key: string): Promise<unknown>; // db.delete(store, key)
|
||||
async loadAll<A>(): Promise<A>; // Returns all entries as object
|
||||
async clearDatabase(): Promise<void>; // db.clear(store)
|
||||
}
|
||||
```
|
||||
|
||||
## Local Storage Structure (IndexedDB)
|
||||
|
||||
All data is stored in a single IndexedDB database with one object store. Each entry is keyed by a string identifier.
|
||||
|
||||
### IndexedDB Keys
|
||||
|
||||
#### System Keys
|
||||
|
||||
| Key | Content | Description |
|
||||
| --------------------- | ------------------------- | ------------------------------------------------------- |
|
||||
| `__meta_` | `LocalMeta` | Sync metadata (vector clock, revMap, timestamps) |
|
||||
| `__client_id_` | `string` | Unique client identifier (e.g., `"BCL1234567890_12_5"`) |
|
||||
| `__sp_cred_Dropbox` | `DropboxPrivateCfg` | Dropbox credentials |
|
||||
| `__sp_cred_WebDAV` | `WebdavPrivateCfg` | WebDAV credentials |
|
||||
| `__sp_cred_LocalFile` | `LocalFileSyncPrivateCfg` | Local file sync config |
|
||||
| `__TMP_BACKUP` | `AllSyncModels` | Temporary backup during imports |
|
||||
|
||||
#### Model Keys (all defined in `pfapi-config.ts`)
|
||||
|
||||
| Key | Content | Main File | Description |
|
||||
| ---------------- | --------------------- | --------- | ----------------------------- |
|
||||
| `task` | `TaskState` | Yes | Tasks data (EntityState) |
|
||||
| `timeTracking` | `TimeTrackingState` | Yes | Time tracking records |
|
||||
| `project` | `ProjectState` | Yes | Projects (EntityState) |
|
||||
| `tag` | `TagState` | Yes | Tags (EntityState) |
|
||||
| `simpleCounter` | `SimpleCounterState` | Yes | Simple counters (EntityState) |
|
||||
| `note` | `NoteState` | Yes | Notes (EntityState) |
|
||||
| `taskRepeatCfg` | `TaskRepeatCfgState` | Yes | Recurring task configs |
|
||||
| `reminders` | `Reminder[]` | Yes | Reminder array |
|
||||
| `planner` | `PlannerState` | Yes | Planner state |
|
||||
| `boards` | `BoardsState` | Yes | Kanban boards |
|
||||
| `menuTree` | `MenuTreeState` | No | Menu structure |
|
||||
| `globalConfig` | `GlobalConfigState` | No | User settings |
|
||||
| `issueProvider` | `IssueProviderState` | No | Issue tracker configs |
|
||||
| `metric` | `MetricState` | No | Metrics (EntityState) |
|
||||
| `improvement` | `ImprovementState` | No | Improvements (EntityState) |
|
||||
| `obstruction` | `ObstructionState` | No | Obstructions (EntityState) |
|
||||
| `pluginUserData` | `PluginUserDataState` | No | Plugin user data |
|
||||
| `pluginMetadata` | `PluginMetaDataState` | No | Plugin metadata |
|
||||
| `archiveYoung` | `ArchiveModel` | No | Recent archived tasks |
|
||||
| `archiveOld` | `ArchiveModel` | No | Old archived tasks |
|
||||
|
||||
### Local Storage Diagram
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ IndexedDB: "pf" │
|
||||
│ Store: "main" │
|
||||
├──────────────────────┬───────────────────────────────────────────┤
|
||||
│ Key │ Value │
|
||||
├──────────────────────┼───────────────────────────────────────────┤
|
||||
│ __meta_ │ { lastUpdate, vectorClock, revMap, ... } │
|
||||
│ __client_id_ │ "BCLm1abc123_12_5" │
|
||||
│ __sp_cred_Dropbox │ { accessToken, refreshToken, encryptKey } │
|
||||
│ __sp_cred_WebDAV │ { url, username, password, encryptKey } │
|
||||
├──────────────────────┼───────────────────────────────────────────┤
|
||||
│ task │ { ids: [...], entities: {...} } │
|
||||
│ project │ { ids: [...], entities: {...} } │
|
||||
│ tag │ { ids: [...], entities: {...} } │
|
||||
│ note │ { ids: [...], entities: {...} } │
|
||||
│ globalConfig │ { misc: {...}, keyboard: {...}, ... } │
|
||||
│ timeTracking │ { ... } │
|
||||
│ planner │ { ... } │
|
||||
│ boards │ { ... } │
|
||||
│ archiveYoung │ { task: {...}, timeTracking: {...} } │
|
||||
│ archiveOld │ { task: {...}, timeTracking: {...} } │
|
||||
│ ... │ ... │
|
||||
└──────────────────────┴───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### How Models Are Saved Locally
|
||||
|
||||
When a model is saved via `ModelCtrl.save()`:
|
||||
|
||||
```typescript
|
||||
// 1. Data is validated
|
||||
if (modelCfg.validate) {
|
||||
const result = modelCfg.validate(data);
|
||||
if (!result.success && modelCfg.repair) {
|
||||
data = modelCfg.repair(data); // Auto-repair if possible
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Metadata is updated (if requested via isUpdateRevAndLastUpdate)
|
||||
// Always:
|
||||
vectorClock = incrementVectorClock(vectorClock, clientId);
|
||||
lastUpdate = Date.now();
|
||||
|
||||
// Only for NON-main-file models (isMainFileModel: false):
|
||||
if (!modelCfg.isMainFileModel) {
|
||||
revMap[modelId] = Date.now().toString();
|
||||
}
|
||||
// Main file models are tracked via mainModelData in the meta file, not revMap
|
||||
|
||||
// 3. Data is saved to IndexedDB
|
||||
await db.put('main', data, modelId); // e.g., key='task', value=TaskState
|
||||
```
|
||||
|
||||
**Important distinction:**
|
||||
|
||||
- **Main file models** (`isMainFileModel: true`): Vector clock is incremented, but `revMap` is NOT updated. These models are embedded in `mainModelData` within the meta file.
|
||||
- **Separate model files** (`isMainFileModel: false`): Both vector clock and `revMap` are updated. The `revMap` entry tracks the revision of the individual remote file.
|
||||
|
||||
### 2. Model Control Layer
|
||||
|
||||
#### ModelCtrl (`api/model-ctrl/model-ctrl.ts`)
|
||||
|
||||
Generic controller for each data model (tasks, projects, tags, etc.):
|
||||
|
||||
```typescript
|
||||
class ModelCtrl<MT extends ModelBase> {
|
||||
save(
|
||||
data: MT,
|
||||
options?: {
|
||||
isUpdateRevAndLastUpdate: boolean;
|
||||
isIgnoreDBLock?: boolean;
|
||||
},
|
||||
): Promise<unknown>;
|
||||
|
||||
load(): Promise<MT>;
|
||||
remove(): Promise<unknown>;
|
||||
}
|
||||
```
|
||||
|
||||
Key behaviors:
|
||||
|
||||
- **Validation on save**: Uses Typia for runtime type checking
|
||||
- **Auto-repair**: Attempts to repair invalid data if `repair` function is provided
|
||||
- **In-memory caching**: Keeps data in memory for fast reads
|
||||
- **Revision tracking**: Updates metadata on save when `isUpdateRevAndLastUpdate` is true
|
||||
|
||||
#### MetaModelCtrl (`api/model-ctrl/meta-model-ctrl.ts`)
|
||||
|
||||
Manages synchronization metadata:
|
||||
|
||||
```typescript
|
||||
interface LocalMeta {
|
||||
lastUpdate: number; // Timestamp of last local change
|
||||
lastSyncedUpdate: number | null; // Timestamp of last sync
|
||||
metaRev: string | null; // Remote metadata revision
|
||||
vectorClock: VectorClock; // Client-specific clock values
|
||||
lastSyncedVectorClock: VectorClock | null;
|
||||
revMap: RevMap; // Model ID -> revision mapping
|
||||
crossModelVersion: number; // Data schema version
|
||||
}
|
||||
```
|
||||
|
||||
Key responsibilities:
|
||||
|
||||
- **Client ID management**: Generates and stores unique client identifiers
|
||||
- **Vector clock updates**: Increments on local changes
|
||||
- **Revision map tracking**: Tracks which model versions are synced
|
||||
|
||||
### 3. Sync Service Layer
|
||||
|
||||
#### SyncService (`api/sync/sync.service.ts`)
|
||||
|
||||
Main sync orchestrator. The `sync()` method:
|
||||
|
||||
1. **Check readiness**: Verify sync provider is configured and authenticated
|
||||
2. **Operation log sync**: Upload/download operation logs (new feature)
|
||||
3. **Early return check**: If `lastSyncedUpdate === lastUpdate` and meta revision matches, return `InSync`
|
||||
4. **Download remote metadata**: Get current remote state
|
||||
5. **Determine sync direction**: Compare local and remote states using `getSyncStatusFromMetaFiles`
|
||||
6. **Execute sync**: Upload, download, or report conflict
|
||||
|
||||
```typescript
|
||||
async sync(): Promise<{ status: SyncStatus; conflictData?: ConflictData }>
|
||||
```
|
||||
|
||||
Possible sync statuses:
|
||||
|
||||
- `InSync` - No changes needed
|
||||
- `UpdateLocal` - Download needed (remote is newer)
|
||||
- `UpdateRemote` - Upload needed (local is newer)
|
||||
- `UpdateLocalAll` / `UpdateRemoteAll` - Full sync needed
|
||||
- `Conflict` - Concurrent changes detected
|
||||
- `NotConfigured` - No sync provider set
|
||||
|
||||
#### MetaSyncService (`api/sync/meta-sync.service.ts`)
|
||||
|
||||
Handles metadata file operations:
|
||||
|
||||
- `download()`: Gets remote metadata, checks for locks
|
||||
- `upload()`: Uploads metadata with encryption
|
||||
- `lock()`: Creates a lock file during multi-file upload
|
||||
- `getRev()`: Gets remote metadata revision
|
||||
|
||||
#### ModelSyncService (`api/sync/model-sync.service.ts`)
|
||||
|
||||
Handles individual model file operations:
|
||||
|
||||
- `upload()`: Uploads a model with encryption
|
||||
- `download()`: Downloads a model with revision verification
|
||||
- `remove()`: Deletes a remote model file
|
||||
- `getModelIdsToUpdateFromRevMaps()`: Determines which models need syncing
|
||||
|
||||
### 4. Vector Clock System
|
||||
|
||||
#### Purpose
|
||||
|
||||
Vector clocks provide **causality-based conflict detection** for distributed systems. Unlike simple timestamps:
|
||||
|
||||
- They detect **concurrent changes** (true conflicts)
|
||||
- They preserve **happened-before relationships**
|
||||
- They work without synchronized clocks
|
||||
|
||||
#### Implementation (`api/util/vector-clock.ts`)
|
||||
|
||||
```typescript
|
||||
interface VectorClock {
|
||||
[clientId: string]: number; // Maps client ID to update count
|
||||
}
|
||||
|
||||
enum VectorClockComparison {
|
||||
EQUAL, // Same state
|
||||
LESS_THAN, // A happened before B
|
||||
GREATER_THAN, // B happened before A
|
||||
CONCURRENT, // True conflict - both changed independently
|
||||
}
|
||||
```
|
||||
|
||||
Key operations:
|
||||
|
||||
- `incrementVectorClock(clock, clientId)` - Increment on local change
|
||||
- `mergeVectorClocks(a, b)` - Take max of each component
|
||||
- `compareVectorClocks(a, b)` - Determine relationship
|
||||
- `hasVectorClockChanges(current, reference)` - Check for local changes
|
||||
- `limitVectorClockSize(clock, clientId)` - Prune to max 50 clients
|
||||
|
||||
#### Sync Status Determination (`api/util/get-sync-status-from-meta-files.ts`)
|
||||
|
||||
```typescript
|
||||
function getSyncStatusFromMetaFiles(remote: RemoteMeta, local: LocalMeta) {
|
||||
// 1. Check for empty local/remote
|
||||
// 2. Compare vector clocks
|
||||
// 3. Return appropriate SyncStatus
|
||||
}
|
||||
```
|
||||
|
||||
The algorithm (simplified - actual implementation has more nuances):
|
||||
|
||||
1. **Empty data checks:**
|
||||
|
||||
- If remote has no data (`isRemoteDataEmpty`), return `UpdateRemoteAll`
|
||||
- If local has no data (`isLocalDataEmpty`), return `UpdateLocalAll`
|
||||
|
||||
2. **Vector clock validation:**
|
||||
|
||||
- If either local or remote lacks a vector clock, return `Conflict` with reason `NoLastSync`
|
||||
- Both `vectorClock` and `lastSyncedVectorClock` must be present
|
||||
|
||||
3. **Change detection using `hasVectorClockChanges`:**
|
||||
|
||||
- Local changes: Compare current `vectorClock` vs `lastSyncedVectorClock`
|
||||
- Remote changes: Compare remote `vectorClock` vs local `lastSyncedVectorClock`
|
||||
|
||||
4. **Sync status determination:**
|
||||
- No local changes + no remote changes -> `InSync`
|
||||
- Local changes only -> `UpdateRemote`
|
||||
- Remote changes only -> `UpdateLocal`
|
||||
- Both have changes -> `Conflict` with reason `BothNewerLastSync`
|
||||
|
||||
**Note:** The actual implementation also handles edge cases like minimal-update bootstrap scenarios and validates that clocks are properly initialized.
|
||||
|
||||
### 5. Sync Providers
|
||||
|
||||
#### Interface (`api/sync/sync-provider.interface.ts`)
|
||||
|
||||
```typescript
|
||||
interface SyncProviderServiceInterface<PID extends SyncProviderId> {
|
||||
id: PID;
|
||||
isUploadForcePossible?: boolean;
|
||||
isLimitedToSingleFileSync?: boolean;
|
||||
maxConcurrentRequests: number;
|
||||
|
||||
getFileRev(targetPath: string, localRev: string | null): Promise<FileRevResponse>;
|
||||
downloadFile(targetPath: string): Promise<FileDownloadResponse>;
|
||||
uploadFile(
|
||||
targetPath: string,
|
||||
dataStr: string,
|
||||
revToMatch: string | null,
|
||||
isForceOverwrite?: boolean,
|
||||
): Promise<FileRevResponse>;
|
||||
removeFile(targetPath: string): Promise<void>;
|
||||
listFiles?(targetPath: string): Promise<string[]>;
|
||||
isReady(): Promise<boolean>;
|
||||
setPrivateCfg(privateCfg): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
#### Available Providers
|
||||
|
||||
| Provider | Description | Force Upload | Max Concurrent |
|
||||
| ------------- | --------------------------- | ------------ | -------------- |
|
||||
| **Dropbox** | OAuth2 PKCE authentication | Yes | 4 |
|
||||
| **WebDAV** | Nextcloud, ownCloud, etc. | No | 10 |
|
||||
| **LocalFile** | Electron/Android filesystem | No | 10 |
|
||||
| **SuperSync** | WebDAV-based custom sync | No | 10 |
|
||||
|
||||
### 6. Data Encryption & Compression
|
||||
|
||||
#### EncryptAndCompressHandlerService
|
||||
|
||||
Handles data transformation before upload/after download:
|
||||
|
||||
- **Compression**: Uses compression algorithms to reduce data size
|
||||
- **Encryption**: AES encryption with user-provided key
|
||||
|
||||
Data format prefix: `pf_` indicates processed data.
|
||||
|
||||
### 7. Migration System
|
||||
|
||||
#### MigrationService (`api/migration/migration.service.ts`)
|
||||
|
||||
Handles data schema evolution:
|
||||
|
||||
- Checks version on app startup
|
||||
- Applies cross-model migrations sequentially in order
|
||||
- **Only supports forward (upgrade) migrations** - throws `CanNotMigrateMajorDownError` if data version is higher than code version (major version mismatch)
|
||||
|
||||
```typescript
|
||||
interface CrossModelMigrations {
|
||||
[version: number]: (fullData) => transformedData;
|
||||
}
|
||||
```
|
||||
|
||||
**Migration behavior:**
|
||||
|
||||
- If `dataVersion === codeVersion`: No migration needed
|
||||
- If `dataVersion < codeVersion`: Run all migrations from `dataVersion` to `codeVersion`
|
||||
- If `dataVersion > codeVersion` (major version differs): Throws error - downgrade not supported
|
||||
|
||||
Current version: `4.4` (from `pfapi-config.ts`)
|
||||
|
||||
### 8. Validation & Repair
|
||||
|
||||
#### Validation
|
||||
|
||||
Uses **Typia** for runtime type validation:
|
||||
|
||||
- Each model can define a `validate` function
|
||||
- Returns `IValidation<T>` with success flag and errors
|
||||
|
||||
#### Repair
|
||||
|
||||
Auto-repair system for corrupted data:
|
||||
|
||||
- Each model can define a `repair` function
|
||||
- Applied when validation fails
|
||||
- Falls back to error if repair fails
|
||||
|
||||
## Sync Flow Diagrams
|
||||
|
||||
### Normal Sync Flow
|
||||
|
||||
```
|
||||
┌─────────┐ ┌─────────┐ ┌─────────┐
|
||||
│ Device A│ │ Remote │ │ Device B│
|
||||
└────┬────┘ └────┬────┘ └────┬────┘
|
||||
│ │ │
|
||||
│ 1. sync() │ │
|
||||
├────────────────►│ │
|
||||
│ │ │
|
||||
│ 2. download │ │
|
||||
│ metadata │ │
|
||||
│◄────────────────┤ │
|
||||
│ │ │
|
||||
│ 3. compare │ │
|
||||
│ vector clocks │ │
|
||||
│ │ │
|
||||
│ 4. upload │ │
|
||||
│ changes │ │
|
||||
├────────────────►│ │
|
||||
│ │ │
|
||||
│ │ 5. sync() │
|
||||
│ │◄────────────────┤
|
||||
│ │ │
|
||||
│ │ 6. download │
|
||||
│ │ metadata │
|
||||
│ ├────────────────►│
|
||||
│ │ │
|
||||
│ │ 7. download │
|
||||
│ │ changed │
|
||||
│ │ models │
|
||||
│ ├────────────────►│
|
||||
```
|
||||
|
||||
### Conflict Detection Flow
|
||||
|
||||
```
|
||||
┌─────────┐ ┌─────────┐
|
||||
│ Device A│ │ Device B│
|
||||
│ VC: {A:5, B:3} │ VC: {A:4, B:5}
|
||||
└────┬────┘ └────┬────┘
|
||||
│ │
|
||||
│ Both made changes offline │
|
||||
│ │
|
||||
│ ┌─────────────────────────┼───────────────────────────┐
|
||||
│ │ Compare: CONCURRENT │ │
|
||||
│ │ A has A:5 (higher) │ B has B:5 (higher) │
|
||||
│ │ Neither dominates │ │
|
||||
│ └─────────────────────────┴───────────────────────────┘
|
||||
│ │
|
||||
│ Conflict! │
|
||||
│ User must choose which │
|
||||
│ version to keep │
|
||||
```
|
||||
|
||||
### Multi-File Upload with Locking
|
||||
|
||||
```
|
||||
┌─────────┐ ┌─────────┐
|
||||
│ Client │ │ Remote │
|
||||
└────┬────┘ └────┬────┘
|
||||
│ │
|
||||
│ 1. Create lock │
|
||||
│ (upload lock │
|
||||
│ content) │
|
||||
├────────────────►│
|
||||
│ │
|
||||
│ 2. Upload │
|
||||
│ model A │
|
||||
├────────────────►│
|
||||
│ │
|
||||
│ 3. Upload │
|
||||
│ model B │
|
||||
├────────────────►│
|
||||
│ │
|
||||
│ 4. Upload │
|
||||
│ metadata │
|
||||
│ (replaces lock)│
|
||||
├────────────────►│
|
||||
│ │
|
||||
│ Lock released │
|
||||
```
|
||||
|
||||
## Remote Storage Structure
|
||||
|
||||
The remote storage (Dropbox, WebDAV, local folder) contains multiple files. The structure is designed to optimize sync performance by separating frequently-changed small data from large archives.
|
||||
|
||||
### Remote Files Overview
|
||||
|
||||
```
|
||||
/ (or /DEV/ in development)
|
||||
├── __meta_ # Metadata file (REQUIRED - always synced first)
|
||||
├── globalConfig # User settings
|
||||
├── menuTree # Menu structure
|
||||
├── issueProvider # Issue tracker configurations
|
||||
├── metric # Metrics data
|
||||
├── improvement # Improvement entries
|
||||
├── obstruction # Obstruction entries
|
||||
├── pluginUserData # Plugin user data
|
||||
├── pluginMetadata # Plugin metadata
|
||||
├── archiveYoung # Recent archived tasks (can be large)
|
||||
└── archiveOld # Old archived tasks (can be very large)
|
||||
```
|
||||
|
||||
### The Meta File (`__meta_`)
|
||||
|
||||
The meta file is the **central coordination file** for sync. It contains:
|
||||
|
||||
1. **Sync metadata** (vector clock, timestamps, version)
|
||||
2. **Revision map** (`revMap`) - tracks which revision each model file has
|
||||
3. **Main file model data** - frequently-accessed data embedded directly
|
||||
|
||||
```typescript
|
||||
interface RemoteMeta {
|
||||
// Sync coordination
|
||||
lastUpdate: number; // When data was last changed
|
||||
crossModelVersion: number; // Schema version (e.g., 4.4)
|
||||
vectorClock: VectorClock; // For conflict detection
|
||||
revMap: RevMap; // Model ID -> revision string
|
||||
|
||||
// Embedded data (main file models)
|
||||
mainModelData: {
|
||||
task: TaskState;
|
||||
project: ProjectState;
|
||||
tag: TagState;
|
||||
note: NoteState;
|
||||
timeTracking: TimeTrackingState;
|
||||
simpleCounter: SimpleCounterState;
|
||||
taskRepeatCfg: TaskRepeatCfgState;
|
||||
reminders: Reminder[];
|
||||
planner: PlannerState;
|
||||
boards: BoardsState;
|
||||
};
|
||||
|
||||
// For single-file sync providers
|
||||
isFullData?: boolean; // If true, all data is in this file
|
||||
}
|
||||
```
|
||||
|
||||
### Main File Models vs Separate Model Files
|
||||
|
||||
Models are categorized into two types:
|
||||
|
||||
#### Main File Models (`isMainFileModel: true`)
|
||||
|
||||
These are embedded in the `__meta_` file's `mainModelData` field:
|
||||
|
||||
| Model | Reason |
|
||||
| --------------- | ------------------------------------- |
|
||||
| `task` | Frequently accessed, relatively small |
|
||||
| `project` | Core data, always needed |
|
||||
| `tag` | Small, frequently referenced |
|
||||
| `note` | Often viewed together with tasks |
|
||||
| `timeTracking` | Frequently updated |
|
||||
| `simpleCounter` | Small, frequently updated |
|
||||
| `taskRepeatCfg` | Needed for task creation |
|
||||
| `reminders` | Small array, time-critical |
|
||||
| `planner` | Viewed on app startup |
|
||||
| `boards` | Part of main UI |
|
||||
|
||||
**Benefits:**
|
||||
|
||||
- Single HTTP request to get all core data
|
||||
- Atomic update of related models
|
||||
- Faster initial sync
|
||||
|
||||
#### Separate Model Files (`isMainFileModel: false` or undefined)
|
||||
|
||||
These are stored as individual files:
|
||||
|
||||
| Model | Reason |
|
||||
| -------------------------------------- | ------------------------------------------- |
|
||||
| `globalConfig` | User-specific, rarely synced |
|
||||
| `menuTree` | UI state, not critical |
|
||||
| `issueProvider` | Contains credentials, separate for security |
|
||||
| `metric`, `improvement`, `obstruction` | Historical data, can grow large |
|
||||
| `archiveYoung` | Can be large, changes infrequently |
|
||||
| `archiveOld` | Very large, rarely accessed |
|
||||
| `pluginUserData`, `pluginMetadata` | Plugin-specific, isolated |
|
||||
|
||||
**Benefits:**
|
||||
|
||||
- Only download what changed (via `revMap` comparison)
|
||||
- Large files (archives) don't slow down regular sync
|
||||
- Can sync individual models independently
|
||||
|
||||
### RevMap: Tracking Model Versions
|
||||
|
||||
The `revMap` tracks which version of each separate model file is on the remote:
|
||||
|
||||
```typescript
|
||||
interface RevMap {
|
||||
[modelId: string]: string; // Model ID -> revision/timestamp
|
||||
}
|
||||
|
||||
// Example
|
||||
{
|
||||
"globalConfig": "1701234567890",
|
||||
"menuTree": "1701234567891",
|
||||
"archiveYoung": "1701234500000",
|
||||
"archiveOld": "1701200000000",
|
||||
// ... (main file models NOT included - they're in mainModelData)
|
||||
}
|
||||
```
|
||||
|
||||
When syncing:
|
||||
|
||||
1. Download `__meta_` file
|
||||
2. Compare remote `revMap` with local `revMap`
|
||||
3. Only download model files where revision differs
|
||||
|
||||
### Upload Flow
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ UPLOAD FLOW │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
1. Determine what changed (compare local/remote revMaps)
|
||||
local.revMap: { archiveYoung: "100", globalConfig: "200" }
|
||||
remote.revMap: { archiveYoung: "100", globalConfig: "150" }
|
||||
→ globalConfig needs upload
|
||||
|
||||
2. For multi-file upload, create lock:
|
||||
Upload to __meta_: "SYNC_IN_PROGRESS__BCLm1abc123_12_5"
|
||||
|
||||
3. Upload changed model files:
|
||||
Upload to globalConfig: { encrypted/compressed data }
|
||||
→ Get new revision: "250"
|
||||
|
||||
4. Upload metadata (replaces lock):
|
||||
Upload to __meta_: {
|
||||
lastUpdate: 1701234567890,
|
||||
vectorClock: { "BCLm1abc123_12_5": 42 },
|
||||
revMap: { archiveYoung: "100", globalConfig: "250" },
|
||||
mainModelData: { task: {...}, project: {...}, ... }
|
||||
}
|
||||
```
|
||||
|
||||
### Download Flow
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ DOWNLOAD FLOW │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
|
||||
1. Download __meta_ file
|
||||
→ Get mainModelData (task, project, tag, etc.)
|
||||
→ Get revMap for separate files
|
||||
|
||||
2. Compare revMaps:
|
||||
remote.revMap: { archiveYoung: "300", globalConfig: "250" }
|
||||
local.revMap: { archiveYoung: "100", globalConfig: "250" }
|
||||
→ archiveYoung needs download
|
||||
|
||||
3. Download changed model files (parallel with load balancing):
|
||||
Download archiveYoung → decrypt/decompress → save locally
|
||||
|
||||
4. Update local metadata:
|
||||
- Save all mainModelData to IndexedDB
|
||||
- Save downloaded models to IndexedDB
|
||||
- Update local revMap to match remote
|
||||
- Merge vector clocks
|
||||
- Set lastSyncedUpdate = lastUpdate
|
||||
```
|
||||
|
||||
### Single-File Sync Mode
|
||||
|
||||
Some providers (or configurations) use `isLimitedToSingleFileSync: true`. In this mode:
|
||||
|
||||
- **All data** is stored in the `__meta_` file
|
||||
- `mainModelData` contains ALL models, not just main file models
|
||||
- `isFullData: true` flag is set
|
||||
- No separate model files are created
|
||||
- Simpler but less efficient for large datasets
|
||||
|
||||
### File Content Format
|
||||
|
||||
All files are stored as JSON strings with optional encryption/compression:
|
||||
|
||||
```
|
||||
Raw: { "ids": [...], "entities": {...} }
|
||||
↓ (if compression enabled)
|
||||
Compressed: <binary compressed data>
|
||||
↓ (if encryption enabled)
|
||||
Encrypted: <AES encrypted data>
|
||||
↓
|
||||
Prefixed: "pf_" + <cross_model_version> + "__" + <base64 encoded data>
|
||||
```
|
||||
|
||||
The `pf_` prefix indicates the data has been processed and needs decryption/decompression.
|
||||
|
||||
## Data Model Configurations
|
||||
|
||||
From `pfapi-config.ts`:
|
||||
|
||||
| Model | Main File | Description |
|
||||
| ---------------- | --------- | ---------------------- |
|
||||
| `task` | Yes | Tasks data |
|
||||
| `timeTracking` | Yes | Time tracking records |
|
||||
| `project` | Yes | Projects |
|
||||
| `tag` | Yes | Tags |
|
||||
| `simpleCounter` | Yes | Simple Counters |
|
||||
| `note` | Yes | Notes |
|
||||
| `taskRepeatCfg` | Yes | Recurring task configs |
|
||||
| `reminders` | Yes | Reminders |
|
||||
| `planner` | Yes | Planner data |
|
||||
| `boards` | Yes | Kanban boards |
|
||||
| `menuTree` | No | Menu structure |
|
||||
| `globalConfig` | No | User settings |
|
||||
| `issueProvider` | No | Issue tracker configs |
|
||||
| `metric` | No | Metrics data |
|
||||
| `improvement` | No | Metric improvements |
|
||||
| `obstruction` | No | Metric obstructions |
|
||||
| `pluginUserData` | No | Plugin user data |
|
||||
| `pluginMetadata` | No | Plugin metadata |
|
||||
| `archiveYoung` | No | Recent archive |
|
||||
| `archiveOld` | No | Old archive |
|
||||
|
||||
**Main file models** are stored in the metadata file itself for faster sync of frequently-accessed data.
|
||||
|
||||
## Error Handling
|
||||
|
||||
Custom error types in `api/errors/errors.ts`:
|
||||
|
||||
- **API Errors**: `NoRevAPIError`, `RemoteFileNotFoundAPIError`, `AuthFailSPError`
|
||||
- **Sync Errors**: `LockPresentError`, `LockFromLocalClientPresentError`, `UnknownSyncStateError`
|
||||
- **Data Errors**: `DataValidationFailedError`, `ModelValidationError`, `DataRepairNotPossibleError`
|
||||
|
||||
## Event System
|
||||
|
||||
```typescript
|
||||
type PfapiEvents =
|
||||
| 'syncDone' // Sync completed
|
||||
| 'syncStart' // Sync starting
|
||||
| 'syncError' // Sync failed
|
||||
| 'syncStatusChange' // Status changed
|
||||
| 'metaModelChange' // Metadata updated
|
||||
| 'providerChange' // Provider switched
|
||||
| 'providerReady' // Provider authenticated
|
||||
| 'providerPrivateCfgChange' // Provider credentials updated
|
||||
| 'onBeforeUpdateLocal'; // About to download changes
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **Encryption**: Optional AES encryption with user-provided key
|
||||
2. **No tracking**: All data stays local unless explicitly synced
|
||||
3. **Credential storage**: Provider credentials stored in IndexedDB with prefix `__sp_cred_`
|
||||
4. **OAuth security**: Dropbox uses PKCE flow
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
1. **Vector clocks over timestamps**: More reliable conflict detection in distributed systems
|
||||
2. **Main file models**: Frequently accessed data bundled with metadata for faster sync
|
||||
3. **Database locking**: Prevents corruption during sync operations
|
||||
4. **Adapter pattern**: Easy to add new storage backends
|
||||
5. **Provider abstraction**: Consistent interface across Dropbox, WebDAV, local files
|
||||
6. **Typia validation**: Runtime type safety without heavy dependencies
|
||||
|
||||
## Future Considerations
|
||||
|
||||
The system has been extended with **Operation Log Sync** for more granular synchronization at the operation level rather than full model replacement. See `operation-log-architecture.md` for details.
|
||||
Loading…
Add table
Add a link
Reference in a new issue