From d56084e4bcbf2ae22e171a36607a74b1236112e9 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 21 Sep 2026 21:29:52 +0100 Subject: [PATCH] power loss --- README.md | 2 +- SYNC_ENGINE_HANDOVER.md | 647 +++++++ docs/architecture/sync-algorithm.md | 523 +---- docs/guide/limitations.md | 31 +- frontend/deterministic-tests/README.md | 224 ++- frontend/deterministic-tests/package.json | 2 +- frontend/deterministic-tests/src/cli.ts | 32 +- .../deterministic-tests/src/crash-worker.ts | 93 + .../src/deterministic-agent.ts | 713 +++---- .../src/managed-websocket.ts | 21 + .../deterministic-tests/src/parse-args.ts | 4 +- .../deterministic-tests/src/server-control.ts | 120 +- .../deterministic-tests/src/server-manager.ts | 5 +- .../src/test-definition.ts | 46 +- .../deterministic-tests/src/test-registry.ts | 2 + .../deterministic-tests/src/test-runner.ts | 144 +- .../src/utils/assertable-state.ts | 93 +- .../deterministic-tests/src/v4-regressions.ts | 253 +++ frontend/deterministic-tests/tsconfig.json | 1 + .../deterministic-tests/webpack.config.js | 5 +- .../src/node-filesystem.test.ts | 45 +- .../local-client-cli/src/node-filesystem.ts | 65 +- .../src/obsidian-file-system.ts | 32 +- frontend/package-lock.json | 1128 +++++++---- frontend/package.json | 3 + frontend/sync-client/README.md | 137 ++ frontend/sync-client/package.json | 1 - .../file-operations.recovery.test.ts | 428 ++++ .../src/file-operations/file-operations.ts | 1147 ++++++++--- .../file-operations/filesystem-operations.ts | 76 +- .../sync-client/src/persistence/database.ts | 521 ++--- .../sync-client/src/services/sync-service.ts | 64 +- .../services/types/DocumentUpdateResponse.ts | 3 +- .../src/services/types/EventBatch.ts | 6 +- .../src/services/websocket-manager.test.ts | 181 +- .../src/services/websocket-manager.ts | 119 +- frontend/sync-client/src/sync-client.ts | 879 ++++----- .../sync-operations/cursor-tracker.test.ts | 209 ++ .../src/sync-operations/cursor-tracker.ts | 203 +- .../src/sync-operations/file-manifest.ts | 69 + .../src/sync-operations/local-changes.ts | 82 + .../sync-operations/outgoing-cursor.test.ts | 59 + .../sync-client/src/sync-operations/scan.ts | 152 ++ .../src/sync-operations/syncer.test.ts | 351 ++++ .../sync-client/src/sync-operations/syncer.ts | 1724 +++++++++++++---- .../data-structures/event-listeners.test.ts | 24 + .../utils/data-structures/event-listeners.ts | 70 +- .../src/utils/portable-path.test.ts | 39 + .../sync-client/src/utils/portable-path.ts | 190 ++ frontend/test-client/README.md | 36 + frontend/test-client/src/agent/mock-agent.ts | 368 ---- frontend/test-client/src/agent/mock-client.ts | 197 -- frontend/test-client/src/cli.ts | 285 +-- .../src/utils/random-casing.test.ts | 2 +- .../test-client/src/utils/random-casing.ts | 4 +- frontend/test-client/src/workload.test.ts | 71 + frontend/test-client/src/workload.ts | 310 +++ frontend/test-client/tsconfig.json | 1 + frontend/test-client/webpack.config.js | 1 + frontend/test-support/README.md | 163 ++ frontend/test-support/canonical.ts | 157 ++ frontend/test-support/content-limits.test.ts | 103 + .../test-support/engine-regressions.test.ts | 910 +++++++++ frontend/test-support/event-delivery.test.ts | 157 ++ frontend/test-support/harness.test.ts | 316 +++ .../journal-interleavings.test.ts | 407 ++++ .../test-support/journal-protocol.test.ts | 612 ++++++ frontend/test-support/listener-crash-child.ts | 30 + frontend/test-support/native-fs.c | 27 + frontend/test-support/network.ts | 243 +++ frontend/test-support/observer.test.ts | 15 + .../test-support/oracle-regressions.test.ts | 351 ++++ frontend/test-support/oracles.ts | 173 ++ frontend/test-support/polling.test.ts | 317 +++ frontend/test-support/protocol-audit.test.ts | 392 ++++ frontend/test-support/protocol.test.ts | 420 ++++ frontend/test-support/random.ts | 25 + frontend/test-support/real-disk.test.ts | 137 ++ frontend/test-support/real-disk.ts | 215 ++ frontend/test-support/recovery.test.ts | 695 +++++++ frontend/test-support/server-restore.test.ts | 179 ++ frontend/test-support/storage.ts | 320 +++ frontend/test-support/sync-audit.test.ts | 900 +++++++++ frontend/test-support/sync-fixture.ts | 136 ++ frontend/test-support/sync-server-fixture.ts | 94 + frontend/test-support/tsconfig.json | 13 + .../harness-review/deterministic-results.json | 1121 +++++++++++ logs/harness-review/result-42.json | 4 + logs/harness-review/result-43.json | 4 + logs/harness-review/results.json | 111 ++ logs/harness-review/trace-42.json | 368 ++++ logs/harness-review/trace-43.json | 404 ++++ sync-server/README.md | 72 +- sync-server/src/app_state/cursors.rs | 95 +- sync-server/src/app_state/database.rs | 122 +- .../20260920000000_history_checkpoint.sql | 4 + sync-server/src/app_state/database/models.rs | 4 + .../src/app_state/database/mutations.rs | 15 +- sync-server/src/app_state/database/queries.rs | 70 +- sync-server/src/app_state/database/tests.rs | 269 +++ .../src/app_state/websocket/broadcasts.rs | 20 +- sync-server/src/app_state/websocket/utils.rs | 14 +- sync-server/src/config.rs | 116 +- sync-server/src/config/server_config.rs | 5 +- sync-server/src/config/user_config.rs | 2 +- sync-server/src/server.rs | 9 + sync-server/src/server/auth.rs | 42 +- .../fetch_latest_document_version.rs | 20 +- .../src/server/endpoints/put_file_content.rs | 149 +- .../src/server/endpoints/put_file_manifest.rs | 16 +- sync-server/src/server/endpoints/websocket.rs | 37 +- sync-server/src/server/responses.rs | 4 +- sync-server/src/utils/portable_path.rs | 28 +- 113 files changed, 18820 insertions(+), 4055 deletions(-) create mode 100644 SYNC_ENGINE_HANDOVER.md create mode 100644 frontend/deterministic-tests/src/crash-worker.ts create mode 100644 frontend/deterministic-tests/src/v4-regressions.ts create mode 100644 frontend/sync-client/README.md create mode 100644 frontend/sync-client/src/file-operations/file-operations.recovery.test.ts create mode 100644 frontend/sync-client/src/sync-operations/file-manifest.ts create mode 100644 frontend/sync-client/src/sync-operations/local-changes.ts create mode 100644 frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts create mode 100644 frontend/sync-client/src/sync-operations/scan.ts create mode 100644 frontend/sync-client/src/sync-operations/syncer.test.ts create mode 100644 frontend/sync-client/src/utils/portable-path.test.ts create mode 100644 frontend/sync-client/src/utils/portable-path.ts create mode 100644 frontend/test-client/README.md delete mode 100644 frontend/test-client/src/agent/mock-agent.ts delete mode 100644 frontend/test-client/src/agent/mock-client.ts create mode 100644 frontend/test-client/src/workload.test.ts create mode 100644 frontend/test-client/src/workload.ts create mode 100644 frontend/test-support/README.md create mode 100644 frontend/test-support/canonical.ts create mode 100644 frontend/test-support/content-limits.test.ts create mode 100644 frontend/test-support/engine-regressions.test.ts create mode 100644 frontend/test-support/event-delivery.test.ts create mode 100644 frontend/test-support/harness.test.ts create mode 100644 frontend/test-support/journal-interleavings.test.ts create mode 100644 frontend/test-support/journal-protocol.test.ts create mode 100644 frontend/test-support/listener-crash-child.ts create mode 100644 frontend/test-support/native-fs.c create mode 100644 frontend/test-support/network.ts create mode 100644 frontend/test-support/observer.test.ts create mode 100644 frontend/test-support/oracle-regressions.test.ts create mode 100644 frontend/test-support/oracles.ts create mode 100644 frontend/test-support/polling.test.ts create mode 100644 frontend/test-support/protocol-audit.test.ts create mode 100644 frontend/test-support/protocol.test.ts create mode 100644 frontend/test-support/random.ts create mode 100644 frontend/test-support/real-disk.test.ts create mode 100644 frontend/test-support/real-disk.ts create mode 100644 frontend/test-support/recovery.test.ts create mode 100644 frontend/test-support/server-restore.test.ts create mode 100644 frontend/test-support/storage.ts create mode 100644 frontend/test-support/sync-audit.test.ts create mode 100644 frontend/test-support/sync-fixture.ts create mode 100644 frontend/test-support/sync-server-fixture.ts create mode 100644 frontend/test-support/tsconfig.json create mode 100644 logs/harness-review/deterministic-results.json create mode 100644 logs/harness-review/result-42.json create mode 100644 logs/harness-review/result-43.json create mode 100644 logs/harness-review/results.json create mode 100644 logs/harness-review/trace-42.json create mode 100644 logs/harness-review/trace-43.json create mode 100644 sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql diff --git a/README.md b/README.md index 74c6ee97..f13c18a4 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ scripts/bump-version.sh patch #### Run E2E tests ```sh -scripts/e2e.sh 8 +E2E_WORKERS=8 scripts/e2e.sh ``` And to clean up the logs & database files, run `scripts/clean-up.sh` diff --git a/SYNC_ENGINE_HANDOVER.md b/SYNC_ENGINE_HANDOVER.md new file mode 100644 index 00000000..aebacadc --- /dev/null +++ b/SYNC_ENGINE_HANDOVER.md @@ -0,0 +1,647 @@ +# Sync engine v4 handover + +Prepared on 2026-09-19. This records the implementation delivered in the preceding +work session: its starting point, intended behavior, changes and their reasons, +validation, and integration work still required. + +Follow-up on 2026-09-19: file modification times have since been removed from the +server schema/protocol, client snapshots, and recovery records. Concurrent binary +or unmergeable edits now keep the server version, with displaced local bytes +retained in recovery; changes on only one side still keep that side. References +to `mtime` below describe the earlier implementation and are superseded by this +change. See the server and client READMEs for the current contracts. + +Further follow-up on 2026-09-19: request tracking is now part of `events`, using +unique `request_id` and `request_fingerprint` columns. The separate +`request_receipts` table and duplicated response JSON have been removed. Retries +check the fingerprint and reconstruct the original `Accepted` response from the +stored event before CAS. References to separate receipts below describe the +earlier implementation and are superseded by this change. + +File manifest storage follow-up on 2026-09-19: complete maps are now stored as rows in +`file_manifest_entries`, keyed by `(file_manifest_id, document_id)`, with exact path +uniqueness per version. `file_manifests` retains a header for every accepted version, +including empty maps. File manifest reads reconstruct the map from these rows, and +vault snapshots join them to the latest content versions. Event rows contain only +ordering and request-deduplication metadata; replay and retry acknowledgements +reconstruct events from normalized document/manifest rows. References below to JSON file +manifest storage describe the earlier implementation. + +The baseline is commit `1619c51` (`Migrate server to CAS only`). The implementation +changes are currently in the working tree/index, rather than a new implementation +commit. The file inventory below describes the rewrite relative to that baseline. + +The following changes must not be attributed to this implementation: + +- While this handover was being written, additional unstaged edits appeared in + `sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql`, + `sync-server/src/server/endpoints/get_file_manifest.rs`, + `sync-server/src/server/endpoints/put_file_manifest.rs`, + `sync-server/src/server/endpoints/vault_snapshot.rs`, + `sync-server/src/server/endpoints/put_file_content.rs`, and + `sync-server/src/server/endpoints/put_file_content/tests.rs`. They were left alone + and are outside this handover's implementation inventory. The earlier passing + validation applies to the rewrite as delivered, represented by the staged + implementation at the start of this handover, not subsequent edits. Review and + validate the final SQL before starting a new database. + +This handover task added only this document; it did not modify implementation +files or change staging decisions. + +## Initial state + +The repository was in the middle of a protocol migration. It was not a functioning +v4 implementation with a few missing routes. + +| Area | Starting behavior | Why it needed to change | +| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Server protocol | Advertised API v3. `PUT /documents/:id` already used a parent version, request UUID, immutable versions, SHA-256 request fingerprints, and durable acknowledgements. Concurrent text merging had already been removed from the server. | Preserve the existing content CAS design and extend its durability model to namespace changes. This rewrite did not originate server-side CAS or idempotency. | +| Content and names | A document version also contained `relative_path` and `is_deleted`; pushes carried a path and supported `Delete`. The server sanitized and allocated available paths. | Content history and namespace decisions were coupled. Whole rename sequences and cross-document conflicts could not be decided atomically as a file manifest. | +| Server persistence | `documents`, a latest-version view, and `push_acknowledgements`; new IDs were allocated from the maximum document update ID inside a write transaction. | There was no common durable event stream for content and file manifests, or file manifest history. | +| Broadcasts | A completed write committed and then broadcast its document metadata. The originating device was skipped. Reconnection fetched latest document heads rather than every event. | Post-commit task scheduling could change notification order; a crash after commit could lose a wakeup. Latest heads are not an ordered history of namespace transitions. | +| Client protocol | Advertised API v2. It still used the earlier create/text/binary/delete service calls and expected `FastForwardUpdate` / `MergingUpdate`. | The client had not caught up with the server's previous CAS migration, let alone the proposed file manifest protocol. | +| Client scheduling | `Syncer`, `UnrestrictedSyncer`, a configurable concurrent queue, document locks, mutable records, and in-memory promises coordinated operations. | Whole-file-manifest changes need a consistent view across documents. A single durable reconciliation loop is easier to reason about when performance is secondary. | +| Client metadata | Persisted document metadata and an update watermark, with mutable runtime records and gap tracking. | No persisted exact request/response workflow or filesystem application journal tied disk changes to metadata advancement. | +| Filesystem | Read/write/rename/delete operations, an atomic text updater, an in-memory safety wrapper, and native line-ending conversion. | The interface did not promise durable data and directory updates or crash-recoverable multi-file moves. | +| Content comparison | A 32-bit rolling checksum. | Collisions could incorrectly hide edits or identify unrelated files as a rename. | +| Reset/settings | Reset cleared local tracking; settings could cause a vault reset. Persistence had no explicit power-loss contract. | Clearing uncertain requests or application state could lose recovery information or apply it to the wrong vault. | + +The Obsidian plugin, CLI client, legacy test fixtures, and their adapters were +explicitly outside the implementation scope. + +## Expected result and selected policies + +The requested result is a simple server CAS store and a client that owns all +reconciliation. The content algorithm follows the fetch/merge/push model in +[Conflict-Free Three-Way Text Merging](https://schmelczer.dev/articles/reconcile-text-3-way-merge/), +including recording the sent snapshot after acceptance and retrying an uncertain +request with its original idempotency key. + +The implemented invariants are: + +1. Document UUIDs identify content; a file manifest maps UUIDs to paths and owns + membership. A content push cannot rename, delete, or restore membership. +2. File manifest acceptance replaces the entire map only when its parent is current. + Every referenced document must already have content. +3. An accepted content or file manifest mutation commits its state, event, and receipt + together. Each vault has one total order; separate vaults have separate orders. +4. Each WebSocket sends persistent events in that order, including the sender's + own events. Cursors remain ephemeral and do not consume event IDs. +5. The client persists its exact request before sending. An unknown outcome is + resolved before advancing to newer remote state. +6. The client advances a merge base to what was actually incorporated or accepted, + never to an assumption about the current live file. +7. Filesystem application has a durable journal. Every affected source is staged + before destinations are installed, so swaps and cycles do not overwrite files. +8. A received event watermark becomes durable with the corresponding state or + recoverable application obligation. Excluded downloads remain explicit work. + +The conflict policies implemented alongside these invariants are: + +| Situation | Decision | Reason | +| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Same UUID, local path unchanged | Use the server path, including absence. | Incorporate remote changes. | +| Same UUID, remote path unchanged | Use the local path, including absence. | Preserve local intent. | +| Same UUID, both paths changed | Use the server path, including absence. | The requested deterministic tie-breaker. | +| Different UUIDs claim conflicting paths | Keep both identities. A selected path matching the server wins priority; UUID order resolves other ties. Give displaced documents deterministic conflict names. | A per-ID merge alone cannot guarantee one-to-one paths. | +| Delete versus content edit | Membership follows the file manifest merge. Preserve locally displaced bytes in recovery; do not automatically recreate membership. | Avoid deletion/resurrection loops while retaining observed data. | +| First sync, existing local text at a remote path | Adopt the remote UUID and merge against an empty parent. | There is no known common ancestor; neither existing side should silently overwrite the other. | +| Concurrent binary/unmergeable changes | Larger originating mtime wins; server wins ties. | A defined policy is necessary when text reconciliation does not apply. Clock skew remains a limitation. | +| Rename notification | Preserve identity through the ordered logical move hints. | Retain identity through online rename sequences. | +| Unobserved/offline rename | Infer only from a unique, nonempty content hash match on both sides; otherwise treat as delete/create. | Identical files and rename-plus-edit sequences cannot be identified reliably from snapshots alone. | +| Deliberate notified delete/recreate | Allocate a new UUID, even at the same path with identical bytes. | A new document must not inherit the deleted document's identity accidentally. | +| Portable-name conflict | Repair names client-side; reject invalid canonical file manifests server-side. | Every client must receive one valid, unambiguous namespace. | +| Ignored, oversized, or unmaterialized remote file | Preserve membership and remember its remote head. | Skipping a download must not publish a deletion. | + +This is not a proof of a universally perfect sync engine. Power-loss recovery +depends on the adapter/storage contracts below. Snapshot-based observation cannot +recover writes that were never observed or unambiguously reconstruct offline +identity. Automatic text merging also cannot guarantee semantic correctness of +the resulting prose or code. + +## Server changes and rationale + +### Independent namespace and content CAS + +`PutFileContent` now contains `requestId`, nullable `parentVersionId`, `mtime`, and +`content`. `content` accepts `Snapshot` or `Diff`. Paths, delete payloads, and other +unknown fields are rejected. The server still reconstructs a transport diff against +the supplied parent; it does not reconcile competing edits. The existing guard +against the minimum signed integer in diff lengths was retained. + +`mtime` is the originating file time in milliseconds, stored as a finite floating +point value within the JavaScript date range. It is distinct from the server's +commit time. This lets clients apply the selected binary policy without treating +download time as a new edit. + +`PushFileManifest` contains `requestId`, `parentFileManifestId`, and `entries`, where entries +are the complete UUID-to-path map. The empty initial file manifest has ID 0. A custom +map deserializer rejects repeated UUID keys, including alternate spellings of the +same UUID; an ordinary JSON map decoder would silently keep one value. A sorted +map gives request fingerprinting a stable key order. + +Both mutation kinds return `Accepted` or `StaleBase`. A stale content response +contains the current bytes; a stale file manifest response contains the current map. +Only accepted mutations get durable receipts. Rejected requests have no state +effect and may receive a newer stale base if retried later. + +### Schema and transaction boundary + +The bootstrap schema was replaced with a fresh v4 schema: + +| Object | Purpose and reason | +| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `events` | AUTOINCREMENT IDs, request IDs, and fingerprints provide the common order and durable idempotency namespace. IDs are constrained to JavaScript's safe integer range. Event payloads are reconstructed from normalized rows. | +| `documents` | Immutable content versions reference their creating event; no path or deletion fields. UUIDs use the BLOB representation expected by SQLx's UUID binding. `mtime`, author, device, and server timestamp are retained. | +| `documents_by_id` and `latest_document_versions` | Find a UUID's current content without namespace lookups. | +| `file_manifests` and `file_manifest_entries` | Immutable manifest headers and normalized UUID/path rows keyed by their creating event ID. A file manifest version need not equal the latest event ID because content events can follow it. | + +The write transaction performs event/request lookup before CAS, then head validation, +event allocation, normalized state insertion, and commit. Request lookup must come +first: a retry of an accepted request is still successful after another +client advances the head. Reusing a request UUID with different content, parent, +document ID, or operation type is rejected. Session/device identity is excluded +from the fingerprint so a restarted client can recover its old outcome. + +Read operations were consolidated in `queries.rs`, and write operations in +`mutations.rs`. Runtime SQL queries replace the old schema-dependent SQLx macro +queries in this path. This keeps the new schema and transaction logic together +without requiring the old prepared-query cache to describe v4. + +SQLite now explicitly uses WAL, `synchronous=FULL`, `fullfsync`, and foreign keys. +On Unix, database creation also flushes the containing directory and ancestors. +The reason is durability of both transaction contents and newly created directory +entries. These settings still depend on the platform/filesystem honoring flushes. + +No migration or automatic deletion of old databases was added. Replacing the +bootstrap migration deliberately makes old migration checksums incompatible. +Use a fresh database directory and retain old data separately. + +### Endpoint and replay changes + +Paths below are relative to `/vaults/:vault_id`: + +| Endpoint | Change and rationale | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PUT /documents/:document_id` | Existing route becomes a thin content-only CAS handler. Database mutation code owns the complete atomic operation. | +| `GET /documents/:document_id` | Retained; returns current content using the new metadata shape. | +| `GET /documents/:document_id/versions/:version_id/content` | Retained; clients fetch immutable bases for merging. | +| `GET /file-manifest` | New; retrieves canonical membership and names. | +| `PUT /file-manifest` | New; validates and CAS-replaces the complete namespace. | +| `GET /vault-snapshot` | New; returns the file manifest, referenced content heads, and event watermark from one read transaction. Prevents a bootstrap gap between separately read state and history. | +| `GET /events-since?after=N` | New; returns every later event in increasing order with the transaction's watermark. Negative/ahead-of-history cursors are rejected rather than silently resetting a client. | +| `GET /ws` | Reworked to send the same ordered durable event history. | +| `GET /documents` | Removed; latest document rows no longer describe membership or preserve namespace transitions. | + +WebSocket messages use `vaultEvents` batches containing `events` and +`headEventId`. The handshake retains `lastSeenVaultUpdateId`. One sender per +connection drains the database in order; broadcasts are only wake hints. A +two-second timer also drains the log, covering lost wakes and commit-before-notify +crashes. Broadcast lag triggers another drain rather than loss of persistent +events. HTTP replay is available independently of the socket. + +Cursor broadcasts were adapted to the new notification wrapper but remain +ephemeral. Persistent events no longer filter out their originating device: +filtering them would create holes in that client's event sequence. + +### Portable file manifest validation + +Validation now checks the whole namespace, not just individual strings: + +- Relative `/`-separated paths and NFC normalization, without a universal + total-path byte limit. +- No empty, `.` or `..` components; no backslashes, controls, Windows-forbidden + characters, trailing dots/spaces, or reserved device names. The device-name + checks also cover extension forms, trailing spaces in the stem, and the + superscript COM/LPT digit variants. +- `.vault-link-sync` and case/normalization aliases are reserved. +- Comparison uses NFC → Unicode uppercase → NFC. +- No duplicate/alias paths, file-versus-ancestor conflicts, or inconsistent + directory spelling such as `Notes/a.md` alongside `notes/b.md`. +- Every UUID must refer to existing content before membership is accepted. + +The server rejects invalid input instead of allocating or sanitizing names. This +keeps the canonical update exactly equal to the file manifest whose CAS the client +requested. Client and server implement the same portable-name policy. + +## Client changes and rationale + +### Persistent state and the three file manifests + +The three conceptual file manifests are represented as follows: + +| Concept | Representation | Why | +| --------------------------- | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Incorporated server base, B | `state.fileManifest` with its version ID | A received map becomes the base only after its application is durable or journaled for recovery. | +| Last submitted file manifest, S | `state.pending.request.entries` while a file manifest request is unresolved | Freeze the exact request and parent across network failures and restarts. On acceptance, this exact S becomes B. | +| Current local view, L | `state.local`, refreshed by complete scans and logical notifications | Preserve changes made while an earlier S was in flight. It also tracks unmaterialized/excluded remote membership; it is more than a raw listing of existing disk files. | + +S is not kept as a redundant permanent third field after its outcome is resolved. +On acceptance it becomes the incorporated base. On rejection the client merges B, +the current L, and the returned remote map, then prepares a new request with a new +UUID. This preserves the three-way semantics without maintaining duplicate state. + +The new format also stores per-document incorporated content metadata and hash, +observed hash, materialization/bootstrap flags, `lastSeenUpdateId`, deferred +`remoteHeads`, the initial vault snapshot while bootstrap is incomplete, pending +requests/responses, and a filesystem application journal. + +`Database.commit` clones and saves the complete value before replacing in-memory +state. A save is marked uncertain before it is attempted. If it fails, the next +attempt reloads durable state before doing more work; a failed save may have +persisted either complete version. This avoids continuing from an assumed outcome. + +State is bound to the remote URI and vault name. Incompatible formats and live +vault rebinding are rejected instead of silently clearing recovery information. +A truly empty, uninitialized state can be rebound for initial configuration. + +### One reconciliation loop + +The configurable queue and `UnrestrictedSyncer` were replaced by one serialized +loop per vault. This intentionally trades throughput for a smaller set of states +that can overlap. Its normal progression is: + +1. Reload metadata after any uncertain save and recover any filesystem journal. + Recovery runs before a network ping, so an offline server does not block it. +2. Initialize server configuration; if necessary, persist and apply a consistent + bootstrap vault snapshot. +3. Resolve the exact pending request and persist its response before processing + newer remote state. +4. Scan local files, applying ordered logical create/delete/move hints. +5. Replay HTTP events from the durable cursor. Require contiguous IDs and the + advertised watermark; do not silently skip a missing event. +6. Apply file-manifest/content changes, retaining remote heads for deferred downloads. +7. Rescan, retry newly eligible deferred content, and prepare one next content or + file manifest submission. +8. Repeat while there is work; otherwise wake on notifications or a five-second + timer. Transient failures retry at the configured network interval. + +WebSocket batches wake this loop. The client uses HTTP replay as its authoritative +event input, avoiding separate merge paths for HTTP and socket delivery. A dirty +flag and completion handling preserve wakes arriving while an attempt finishes. +`waitUntilFinished()` follows successive active attempts and reports failures. + +The exact content snapshot, mtime, hash, request ID, parent, and optional cursors +are persisted before sending. After acceptance, the base hash comes from those +sent bytes, not a later disk read. A lost response therefore cannot cause an edit +to be merged into itself. Stale responses cause a client-side merge, and rejected +submission bytes are retained in recovery before the request is cleared. + +New UUIDs are saved before network use. Initial content is accepted before its +UUID is published in the file manifest. A crash between those operations can leave +unreferenced content, which is retained and harmless to the visible namespace. +It does not leave a file manifest pointing to absent content. + +### Scanning, identities, and conflicts + +Scans must complete or fail. A disappearing file during a scan is retried rather +than interpreted from a partial listing. Ignored paths, files above the configured +size limit, and files not yet materialized are handled explicitly so an unavailable +local copy does not masquerade as deletion. + +Known online moves are applied in notification order, including directory-prefix +moves. Logical delete/create notifications distinguish replacement from editing. +Offline inference requires exactly one missing identity and one unknown path with +the same nonempty SHA-256 hash. Initial same-path adoption uses portable path +comparison. On normalization-insensitive filesystems, an accessible known NFC +spelling is retained when a scan reports its decomposed alias. + +A local generation counter invalidates a prepared merge or push if a notification +arrives during scanning, remote fetches, or planning. The loop rescans rather than +applying the stale plan. This addresses, for example, moving a file and creating a +different file at its old path while remote content is being fetched. + +`file-manifest.ts` first performs the requested per-UUID three-way merge, then allocates +valid paths. Server-selected occupants get priority; remaining ties use UUID +ordering. Conflict suffixes contain the UUID, including file/ancestor conflicts. +Invalid components are repaired, but long paths are not silently flattened into +UUID filenames. Comparing a candidate with each +occupied path individually avoids an allocation loop when the temporary occupied +set itself contains overlapping desired and actual paths. + +### Content reconciliation + +`content.ts` centralizes snapshot serialization and merging. It compares exact +bytes for unchanged-side fast paths, fetches immutable bases when needed, and +calls `reconcile` for valid UTF-8 content with a mergeable extension. Unknown +first-contact text uses an empty parent. Binary or otherwise unmergeable concurrent +content follows originating mtime, with the server winning ties. + +Optional local cursor positions are transformed through text reconciliation and +passed to the adapter with the output. Text merge output uses the larger input +mtime. UTF-8 BOM is decoded as an actual character during merging, and native +line-ending conversion was removed. Existing bytes and CRLF are not rewritten +merely because another platform downloaded the file. + +The old rolling checksum was replaced by asynchronous Web Crypto SHA-256. Cursor +tracking was updated to await hashes, find remote cursors by document UUID, and +resolve their display path through the local file manifest. Missing/unreadable local +files mark ephemeral cursor data as prior instead of breaking synchronization. + +### Filesystem transaction and crash recovery + +`FileOperations` now bridges the metadata store and the filesystem with a persisted +application journal. It does not assume these two stores share a transaction. + +Each application gets a unique transaction directory under +`.vault-link-sync/transactions`. Each affected document has a source staging path, +an output path, expected/replacement snapshots, source/destination names, and phase: +`planned`, `staged`, `prepared`, `installing`, or `installed`. + +The application sequence and reasons are: + +| Action | Why it is needed | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Persist the complete plan before visible mutations | Restart must know what it was trying to do. The plan includes mergeable extensions so recovery is independent of server availability. | +| Move all affected sources into staging before installing any destination | Rename swaps/cycles and paths becoming ancestor directories cannot overwrite another source. Content replacement and deletion use the same retained-source machinery. | +| Detect an existing staging slot on recovery | It proves that the source move happened even if the subsequent metadata save failed. | +| Reconcile a staged source that changed after planning | Preserve observed external edits made between the expected snapshot and staging. | +| Persist the prepared result before creating its output file | Restart must not merge an already merged result a second time. | +| Persist `installing` before the atomic rename | If its origin is subsequently absent, recovery can recognize that installation completed even if its acknowledgement/save was lost. | +| Require exclusive creation and no-replace renames | A concurrent destination must cause a retry/conflict, not be silently overwritten. | +| Stage an unexpected file occupant as another UUID with a conflict path | Preserve an unrelated file that appeared at the destination or blocked an ancestor. | +| Preserve a nonempty blocking directory and rename the incoming document | Unmanaged descendants must not be recursively deleted to make space. | +| Prune only empty implicit directories | Directories are not synchronized objects; files must not be erased by cleanup. | +| Remove staged/output artifacts, then commit the planned next state | Completed applications do not become permanent byte/state archives; interrupted cleanup is retried while the active journal still exists. | + +An external deletion observed before staging prevents automatic resurrection. +Recovery artifact creation is idempotent: an existing artifact is checked for the +expected bytes instead of overwritten. + +Rejected request snapshots are stored separately under +`.vault-link-sync/requests`, with request IDs and identifying JSON. No recovery +artifact or orphan-content garbage collector was added. Ordinary transaction +artifacts are removed on completion; rejected-request snapshots remain explicit +recovery material. + +### Lifecycle, settings, transport, and compatibility + +`SyncClient.create` requires strong adapters and loads persisted state. Destruction +stops work and flushes local changes. Settings and engine saves serialize complete +replacements through the same persistence store so one cannot overwrite newer +data from the other. The client does not coordinate separate processes accessing +the same local vault. + +`reset()` restarts transports/tracking; it no longer clears pending requests or +application journals. Changing a live vault requires a separate state store and +client. Settings saves are assumed to succeed; settings are saved before becoming +visible to listeners, and getters copy the settings and ignore-pattern array to +prevent accidental in-place mutation. + +Public notification methods enqueue work instead of waiting for convergence. +Call `waitUntilFinished()` to await the active attempt. Sync status now checks the +requested path's manifest, content hash, exclusions, pending request, and active +application state. Remaining-operation reporting is a busy indicator rather than +the old concurrent queue length. Successful local submissions and remote filesystem +applications create history entries; the history UI is not a durable audit log. + +The service layer now exposes typed vault-snapshot/event/file-manifest/content operations. +Durable retry ownership moved out of its old `retryForever` request builders and +into the loop. Requests have a 30-second abort timeout; authentication errors and +permanent 4xx responses are distinguished from transient failures. The proper +`Device-Id` header is used. A rejected initial ping is no longer cached forever, +so startup while offline can recover on retry. + +The existing optional telemetry wiring was retained. Its build-injected version +constant now has a development fallback so importing source outside webpack does +not throw. No new telemetry service was introduced. + +The unused `syncConcurrency`, `diffCacheSizeMB`, and `minimumSaveIntervalMs` +settings and the old content cache/queue scaffolding were removed. +`nativeLineEndings` remains accepted by the creation type but no longer transforms +content. These compatibility surfaces should be reviewed when adapters/UI are +updated, rather than assumed to retain their former effects. + +## Required adapter contract and remaining integration + +No production Obsidian or CLI adapter was upgraded in this work. Their old +interfaces are incompatible and will require implementation changes before they +can use the v4 client. + +| Adapter responsibility | Required guarantee | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Listing and reading | Complete enumeration and coherent bytes/mtime snapshots; absence must be distinguishable from failure. Read races must reject instead of returning partial snapshots. | +| Path access | Stay inside the vault; reject symlinks, ancestor symlink traversal, hardlinks, and special files. Staging and visible files must share a filesystem. | +| `write` | Atomically create a complete file exclusively. A crash leaves the path absent or complete, never partially written. Preserve supplied mtime and supported cursor metadata. | +| `rename` | Atomic same-filesystem move without replacing an existing destination. Flush source data and both affected directories before resolving. | +| Directory operations | Durable recursive creation, including ancestors. Pruning uses only directory removal and never unlinks descendant files. | +| Metadata persistence | Atomically replace the complete state and flush data/directory entries before resolving. A failed save may leave either complete value, which must be reloadable. | +| Notifications | Report logical external operations; suppress self-generated create/delete/move notifications. Treat an editor's atomic save as a content update rather than a deliberate document delete/recreate. | +| State placement | Keep metadata outside the user-file namespace or in the reserved internal directory. | + +Known boundaries and deferred work: + +- No automatic migration of existing server databases or v2/v3 client stores. +- No guarantee for writes that occur and disappear between observations, or + perfect identity reconstruction while notifications are unavailable. Logical + notification hints themselves are not an external filesystem operation log. +- No multi-file atomic visibility to arbitrary external programs: staging is + recoverable, but an external reader can see a transaction partway through. +- No hardware power-cut testing or production-adapter durability certification. +- No bounded history/recovery storage, event pagination, large-vault performance + optimization, or guarantee against starvation under continuous competing writes. +- No protection against an administrator independently deleting server history, + pending requests, or recovery state. An ahead-of-history cursor fails rather + than silently accepting a replaced/truncated server database. +- No syncing of empty directories or special filesystem objects. +- No new migration of the history viewer, plugin, CLI, repository-wide architecture + docs, or legacy testing setup. They may still describe the earlier protocol. + +The next integration work is to implement and verify real adapters, update the +excluded callers to v4 and the new notification semantics, and port the behavior +checks into maintained tests. Fresh-state rollout must preserve old data and use +compatible server/client versions together. + +## Validation performed + +These results were obtained at the end of the implementation task, before the +subsequent edits noted above. They are not a claim that every current +working-tree change or every legacy test passes. + +| Check | Result and scope | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `cargo build --manifest-path sync-server/Cargo.toml --quiet` | Passed for the implemented server. | +| `cargo test --manifest-path sync-server/Cargo.toml export_bindings --quiet` | Passed 15 binding-export tests; used to generate the Rust wire types. This is not a behavioral server-suite result. | +| Strict TypeScript check of the production client entrypoint | Passed. Command below. | +| `npm run build -w sync-client` from `frontend` | Passed for both browser and Node production bundles. | +| `git diff --check` | Passed for the implementation diff. | +| Live server CAS/order checks | Passed with an isolated local server and fresh temporary databases. | +| Client reconciliation and interruption checks | Passed against that server using a memory filesystem/persistence harness with injected failures. | + +The TypeScript command was: + +```sh +node frontend/node_modules/typescript/bin/tsc --noEmit --strict \ + --target ESNext --module ESNext --moduleResolution bundler \ + --allowSyntheticDefaultImports --skipLibCheck --types node \ + --typeRoots frontend/node_modules/@types frontend/sync-client/src/index.ts +``` + +Behavior exercised: + +| Scenario | Observed result | +| --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 24 concurrent content creations, then 12 competing file manifest CAS submissions | Exactly one file manifest winner; committed event IDs remained contiguous. | +| WebSocket origin delivery and reconnect replay | Events arrived in increasing order, including events from the connecting device. | +| Duplicate JSON UUID keys and invalid portable file manifests | Rejected without creating events. Cases included duplicate/alias paths, ancestor conflicts, traversal, reserved names, and directory-spelling conflicts. | +| Content and file manifest receipt retry after a server restart and later heads | Returned the original acknowledgement without an additional event. | +| Creation and download; online rename sequence and remote swap | Identities and bytes converged correctly. | +| Concurrent text changes; first-sync same-path text | Both sides' text changes were retained in the checked examples. | +| Concurrent same-path creations | Both UUIDs survived with distinct paths. | +| Lost content acknowledgement followed by client restart | The original request was recovered, with only one accepted content event. | +| Lost file manifest acknowledgement followed by client restart | The original file manifest outcome was recovered without a duplicate mutation. | +| 22 injected filesystem interruption positions during staged swaps | Recovery produced the expected files without an unfinished application journal. | +| 18 injected metadata-save interruption positions during content replacement | Recovery installed the result without merging it a second time. | +| Unexpected destination file and blocking ancestor file | Incoming and unexpected content were both preserved. | +| Binary conflict and remote delete versus unsent local edit | Larger originating mtime won; downloaded mtime was retained; deleted edited bytes remained in recovery. | +| Ignored remote file, then re-enabled syncing | Membership was preserved and the file downloaded once eligible. | +| Save persisted and then reported failure | Reloading durable state allowed retry without restarting the process or duplicating a content mutation. | +| Initial ping offline, then network recovery | The next attempt progressed rather than reusing a permanently rejected ping promise. | +| Move and new file at the old path during a remote fetch | The moved document received the merge and the independent new file remained separate. | +| Notified delete/recreate with identical bytes at the same path | A new UUID was allocated. | +| UTF-8 BOM and CRLF during text merging | Both were preserved in the checked merge. | + +The temporary harnesses were `/tmp/vault-link-engine-check.ts` and +`/tmp/vault-link-server-check.mjs`. They were deliberately not added to the +repository's testing setup. They used a temporary local server on port 18749; +that server was stopped after validation. Their absolute imports and temporary +configuration are development artifacts, not a portable test runner. Additional +server-restart receipt checks were performed during the session. + +The 40 interruption positions are simulated failure points in adapters with the +required atomic semantics. They do not establish durability of real filesystem +adapters, all operating systems, all external-write schedules, or physical storage. + +In the delivered rewrite, the old `put_file_content/tests.rs` file was not ported, +and the rewritten handler no longer included its old test module. Subsequent +edits to those files during this handover are separate work. Removed old helper +modules also contained +legacy unit tests. Client fixtures still target the earlier interfaces. The +webpack loader now checks files reachable from production entrypoints so those +unmigrated fixtures do not block building the library. Full legacy-suite and +repository-wide lint success were not claimed. + +## Complete file inventory + +Every implementation file added, changed, or removed relative to the baseline is +listed below, including regenerated type files and incidental cleanup. Deletions +are listed separately from additions even where Git's similarity detection calls +them renames. Generated binding formatting changes have no additional runtime +meaning unless stated. + +This inventory accounts for **85 implementation paths**. It excludes the preexisting `CLAUDE.md` deletion and includes the old/new sides of Git-detected renames separately. + +### Server + +| File | Change | What changed and why | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [sync-server/Cargo.toml](sync-server/Cargo.toml) | Changed | Declare `sha2` and `unicode-normalization`, remove `sanitize-filename`, and enable ts-rs `no-serde-warnings`. The new mutation code needs request fingerprints and portable-name normalization; server-side filename repair was removed. The preexisting push code already referred to SHA-256; the direct dependency is now declared. | +| [sync-server/Cargo.lock](sync-server/Cargo.lock) | Changed | Record the matching dependency graph: direct SHA-256/normalization dependencies and removal of the filename-sanitizer package. Keeps builds reproducible for the changed file manifest. | +| [sync-server/README.md](sync-server/README.md) | Changed | Replace v3 push/reset instructions with v4 requests, routes, portable-path rules, ordering, recovery, and fresh-state requirements. Documents the actual breaking protocol without suggesting destructive reset commands. | +| [sync-server/src/consts.rs](sync-server/src/consts.rs) | Changed | Raise the supported API from 3 to 4 so incompatible clients fail the version check. | +| [sync-server/src/app_state/database.rs](sync-server/src/app_state/database.rs) | Changed | Wire the consolidated read/write modules, use SQLx `begin_with("BEGIN IMMEDIATE")`, and add explicit SQLite and directory-flush durability settings. Keep commit ordering and persistence in the database boundary. | +| [sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql](sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql) | Changed | Replace the bootstrap schema with events, content-only versions, file manifests, and complete request receipts. Separate membership from bytes and atomically order both. Requires fresh databases; later local edits to this file are outside the earlier validation. | +| [sync-server/src/app_state/database/models.rs](sync-server/src/app_state/database/models.rs) | Changed | Remove path/deletion from content models; add originating mtime, SQLx row decoding, file manifest, event, batch, and vault snapshot types. Flatten wire metadata and use JS-number-compatible exported IDs to express the v4 protocol. | +| [sync-server/src/app_state/database/mutations.rs](sync-server/src/app_state/database/mutations.rs) | Added | Add both CAS transactions, request fingerprint/receipt handling, validation, event allocation, immutable insertions, and post-commit wakeups. Ensures state, order, and retry outcomes commit together. | +| [sync-server/src/app_state/database/queries.rs](sync-server/src/app_state/database/queries.rs) | Added | Add/consolidate current and immutable content reads, current file manifest, consistent bootstrap vault snapshot, and full ordered event replay. Readers no longer infer membership or event history from latest document rows. | +| `sync-server/src/app_state/database/get_document_version.rs` | Removed | Remove the separate old-schema query; immutable content reads are retained in `queries.rs` with the new row shape. | +| `sync-server/src/app_state/database/get_latest_document.rs` | Removed | Remove the separate old-schema query; current-head reads are retained in `queries.rs` and can share the CAS transaction. | +| `sync-server/src/app_state/database/get_latest_document_by_path.rs` | Removed | Remove path-based document lookup because content versions no longer own names. | +| `sync-server/src/app_state/database/get_latest_documents.rs` | Removed | Remove the latest-document listing helper; file manifest membership and a consistent `/vault-snapshot` now define the visible vault. | +| `sync-server/src/app_state/database/get_latest_documents_since.rs` | Removed | Remove head-only catch-up; replay every durable event so intervening namespace changes are not omitted. | +| `sync-server/src/app_state/database/get_max_update_id_in_vault.rs` | Removed | Remove document-table maximum-ID allocation; the common event table allocates IDs for both mutation kinds. | +| `sync-server/src/app_state/database/get_push_acknowledgement.rs` | Removed | Remove the content-specific acknowledgement lookup; shared request receipts now return the original response for either CAS kind. | +| `sync-server/src/app_state/database/insert_document_version.rs` | Removed | Remove the old coupled content/path insertion and direct broadcast. `mutations.rs` now commits state/event/receipt and publishes only a wake hint. | +| [sync-server/src/app_state/websocket/broadcasts.rs](sync-server/src/app_state/websocket/broadcasts.rs) | Changed | Replace wire-event broadcasts with `VaultUpdate` and ephemeral cursor notifications; make channel capacity at least one. Database replay carries correctness when receivers lag or no subscriber exists. | +| [sync-server/src/app_state/websocket/models.rs](sync-server/src/app_state/websocket/models.rs) | Changed | Replace `VaultUpdate` with `VaultEvents(EventBatch)` and remove the origin-filtering wrapper. Durable event batches can represent both mutations without holes for the originating device. | +| [sync-server/src/app_state/websocket/utils.rs](sync-server/src/app_state/websocket/utils.rs) | Changed | Remove `get_unseen_documents`; retain handshake authentication and message sending. Catch-up belongs to the durable event query instead of latest-head selection. | +| [sync-server/src/app_state/cursors.rs](sync-server/src/app_state/cursors.rs) | Changed | Send cursor expiry/update broadcasts through the new ephemeral notification type. Preserve presence behavior independently of persistent event sequencing. | +| [sync-server/src/server.rs](sync-server/src/server.rs) | Changed | Expose request/response modules within the crate for database mutations, add file-manifest/vault-snapshot/events routes, and remove the collection document-list route. Route ownership now reflects the two independent CAS resources. | +| [sync-server/src/server/endpoints.rs](sync-server/src/server/endpoints.rs) | Changed | Register the new file manifest endpoint module and remove the old latest-document-list module so the compiled route set matches v4. | +| `sync-server/src/server/endpoints/get_file_manifest.rs`, `put_file_manifest.rs`, `vault_snapshot.rs`, and `events.rs` | Added | Implement file manifest read/CAS, a consistent vault snapshot, and event replay with cursor checks. Supplies the namespace and recovery endpoints. | +| [sync-server/src/server/endpoints/put_file_content.rs](sync-server/src/server/endpoints/put_file_content.rs) | Changed | Reduce the handler to extraction/auth-context forwarding and content CAS delegation. Remove path/delete decisions and the old v3 test-module inclusion; tests need porting separately. | +| `sync-server/src/server/endpoints/fetch_latest_documents.rs` | Removed | Remove the old collection endpoint; it cannot express file manifest membership plus a consistent event watermark. | +| [sync-server/src/server/endpoints/websocket.rs](sync-server/src/server/endpoints/websocket.rs) | Changed | Use one ordered sender draining database history, replay from the handshake cursor, poll every two seconds, tolerate lag, and process ephemeral cursors in the same connection loop. Notification timing no longer determines event order. | +| [sync-server/src/server/requests.rs](sync-server/src/server/requests.rs) | Changed | Make pushes content-only, add `PushFileManifest`, deny unknown request fields, and reject duplicate parsed UUID keys. Prevent silent interpretation of legacy payloads or ambiguous maps. | +| [sync-server/src/server/responses.rs](sync-server/src/server/responses.rs) | Changed | Add file manifest `Accepted`/`StaleBase` responses and deserialize support for stored acknowledgements. Enables exact receipt replay for both mutation kinds. | +| [sync-server/src/utils/portable_path.rs](sync-server/src/utils/portable_path.rs) | Added | Add namespace-wide portable-path validation and shared comparison conventions. Reject aliases, reserved names, duplicate destinations, and file/ancestor conflicts before canonical acceptance. | +| [sync-server/src/utils.rs](sync-server/src/utils.rs) | Changed | Register portable validation and remove old sanitizer/path-allocation module exports. Keep naming policy on the client. | +| `sync-server/src/utils/dedup_paths.rs` | Removed | Remove the old path-deduplication helper and its embedded legacy tests. Canonical conflicts now require a valid client-submitted file manifest, not a server-generated alternate view. | +| `sync-server/src/utils/find_first_available_path.rs` | Removed | Remove server-side available-name allocation so accepted state equals the submitted file manifest. | +| `sync-server/src/utils/sanitize_path.rs` | Removed | Remove server-side path rewriting and its embedded tests. Invalid paths are rejected; client repair is explicit before CAS. | +| [sync-server/src/errors.rs](sync-server/src/errors.rs) | Changed | Remove an unused `log::error` import. Compile cleanup only; error semantics were not changed here. | + +### Client core and build + +| File | Change | What changed and why | +| -------------------------------------------------------------------------------------------------------------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [frontend/sync-client/README.md](frontend/sync-client/README.md) | Added | Add state/merge policy, notification semantics, recovery layout, adapter requirements, and compatibility boundaries. Gives integrators the contract needed to make the core guarantees meaningful. | +| [frontend/sync-client/src/consts.ts](frontend/sync-client/src/consts.ts) | Changed | Raise the supported API from 2 to 4 so the rewritten client agrees with the server and rejects incompatible versions. | +| [frontend/sync-client/src/index.ts](frontend/sync-client/src/index.ts) | Changed | Export `FileSnapshot` and new file-manifest/event/request types alongside the retained client surface. Consumers can implement the new adapters and use the v4 protocol types. | +| [frontend/sync-client/src/persistence/database.ts](frontend/sync-client/src/persistence/database.ts) | Changed | Replace mutable per-file scheduling records with v4 engine state, pending snapshots/responses, deferred remote heads, and application journals. Save before publishing memory state, reload uncertain outcomes, bind state to a vault, and retain cursor-compatible read-only views. | +| [frontend/sync-client/src/persistence/persistence.ts](frontend/sync-client/src/persistence/persistence.ts) | Changed | Require an atomic durable complete-state save contract. Recovery needs to know that a successful save survives interruption. | +| [frontend/sync-client/src/persistence/settings.ts](frontend/sync-client/src/persistence/settings.ts) | Changed | Persist settings before updating memory/listeners and return copies including the ignore-pattern array. Settings saves are assumed to succeed; caller mutation must not silently change the live configuration. | +| [frontend/sync-client/src/file-operations/filesystem-operations.ts](frontend/sync-client/src/file-operations/filesystem-operations.ts) | Changed | Replace the weaker read/overwrite/text-update interface with coherent snapshots, explicit stat, exclusive durable writes, no-replace durable moves, and directory-only pruning. State the no-link and flush requirements needed for recovery. | +| [frontend/sync-client/src/file-operations/file-operations.ts](frontend/sync-client/src/file-operations/file-operations.ts) | Changed | Replace direct operations and line-ending conversion with journaled staged application, recovery phases, persisted prepared output, race reconciliation, unexpected-occupant preservation, completed-artifact cleanup, and empty-directory pruning. Makes multi-file namespace changes recoverable without permanent per-application archives. | +| `frontend/sync-client/src/file-operations/safe-filesystem-operations.ts` | Removed | Remove the in-memory filesystem safety/locking wrapper. Its guarantees cannot substitute for durable journaling and the stronger adapter contract. | +| [frontend/sync-client/src/services/protocol.ts](frontend/sync-client/src/services/protocol.ts) | Added | Add a single import surface over Rust-generated v4 types plus `FileManifestEntries`. Avoid maintaining a second handwritten protocol definition. | +| [frontend/sync-client/src/services/server-config.ts](frontend/sync-client/src/services/server-config.ts) | Changed | Clear a rejected cached ping promise. An initial offline attempt must be able to fetch configuration again. | +| [frontend/sync-client/src/services/sync-service.ts](frontend/sync-client/src/services/sync-service.ts) | Changed | Replace legacy multipart/create/text/binary/delete methods and retry builders with typed v4 CAS, vault-snapshot, and replay requests; use `Device-Id`, request timeouts, and permanent/auth/transient error distinctions. The persisted actor owns retries. | +| [frontend/sync-client/src/services/websocket-manager.ts](frontend/sync-client/src/services/websocket-manager.ts) | Changed | Parse `vaultEvents` as `EventBatch` and notify the actor. Keep existing transport/cursor support while switching the payload shape. | +| [frontend/sync-client/src/sync-client.ts](frontend/sync-client/src/sync-client.ts) | Changed | Rewire the public client around one durable actor; enforce adapter contracts, serialize persistence, preserve state on reset, restrict vault changes, and keep cursor/history/telemetry integration. Notifications enqueue work and explicit waiting reports completion/failure. | +| [frontend/sync-client/src/sync-operations/content.ts](frontend/sync-client/src/sync-operations/content.ts) | Added | Add stored snapshot conversion and client-only three-way content merging, originating-mtime binary decisions, cursor transformation, BOM handling, and exact-byte/unchanged-side fast paths. Centralizes the content policy used by normal sync and recovery. | +| [frontend/sync-client/src/sync-operations/file-manifest.ts](frontend/sync-client/src/sync-operations/file-manifest.ts) | Added | Add per-UUID three-way merge, absence/deletion handling, portable validation, and deterministic cross-ID conflict allocation, including ancestor and oversized-name cases. Produces a valid namespace before CAS and local application. | +| [frontend/sync-client/src/sync-operations/syncer.ts](frontend/sync-client/src/sync-operations/syncer.ts) | Changed | Replace concurrent per-document scheduling with the recover/bootstrap/retry/scan/replay/merge/push loop, durable requests, complete scans, identity hints, generation guards, deferred downloads, periodic wakeups, and accepted-snapshot base tracking. Makes namespace and content decisions share one consistent state machine. | +| `frontend/sync-client/src/sync-operations/unrestricted-syncer.ts` | Removed | Remove the old second scheduling/operation layer, server-merge response assumptions, and cache-driven update flow. Keeping it beside the new actor would preserve conflicting ownership and v2 behavior. | +| [frontend/sync-client/src/sync-operations/cursor-tracker.ts](frontend/sync-client/src/sync-operations/cursor-tracker.ts) | Changed | Await SHA-256 comparisons, match remote cursors by UUID, display current local paths, and treat missing reads as prior cursor data. Renames must not attach cursors to another document at the old path. | +| `frontend/sync-client/src/utils/find-matching-file.ts` | Removed | Remove the old standalone matching helper. Conservative unique-hash identity inference now lives in the actor with complete scan context. | +| [frontend/sync-client/src/utils/hash.ts](frontend/sync-client/src/utils/hash.ts) | Changed | Replace the 32-bit rolling checksum with async Web Crypto SHA-256 and its known empty digest. Reduce false unchanged/rename decisions from checksum collisions. | +| [frontend/sync-client/src/utils/set-up-telemetry.ts](frontend/sync-client/src/utils/set-up-telemetry.ts) | Changed | Declare the webpack-injected version constant and guard it with a development fallback. Direct source imports used by consumers/validation must not throw before synchronization starts. | +| [frontend/sync-client/webpack.config.js](frontend/sync-client/webpack.config.js) | Changed | Set ts-loader `onlyCompileBundledFiles: true`. Production browser/Node builds typecheck their dependency graph without pulling in unmigrated legacy fixtures; this is not a claim those fixtures pass. | + +### Client generated bindings + +| File | Change | What changed and why | +| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [frontend/sync-client/src/services/types/ClientCursors.ts](frontend/sync-client/src/services/types/ClientCursors.ts) | Changed | Regenerate the existing cursor payload as a type alias with generated array syntax. Keeps generated output current; no intended wire change. | +| [frontend/sync-client/src/services/types/CursorPositionFromClient.ts](frontend/sync-client/src/services/types/CursorPositionFromClient.ts) | Changed | Regenerate the existing cursor payload. Type-alias/array formatting only; the ephemeral wire shape remains the same. | +| [frontend/sync-client/src/services/types/CursorPositionFromServer.ts](frontend/sync-client/src/services/types/CursorPositionFromServer.ts) | Changed | Regenerate the existing cursor response. Formatting/type-alias change only. | +| [frontend/sync-client/src/services/types/CursorSpan.ts](frontend/sync-client/src/services/types/CursorSpan.ts) | Changed | Regenerate the start/end span definition. Formatting/type-alias change only. | +| [frontend/sync-client/src/services/types/DocumentWithCursors.ts](frontend/sync-client/src/services/types/DocumentWithCursors.ts) | Changed | Regenerate the existing document/cursor descriptor from Rust. Preserve its existing wire field names while aligning generated representation. | +| [frontend/sync-client/src/services/types/WebSocketHandshake.ts](frontend/sync-client/src/services/types/WebSocketHandshake.ts) | Changed | Regenerate the token/device/cursor handshake. The replay cursor field is retained; representation changes from interface to type alias. | +| [frontend/sync-client/src/services/types/PingResponse.ts](frontend/sync-client/src/services/types/PingResponse.ts) | Changed | Regenerate current server configuration metadata, including supported API version. No additional ping endpoint behavior is introduced by the generated-file change. | +| [frontend/sync-client/src/services/types/SerializedError.ts](frontend/sync-client/src/services/types/SerializedError.ts) | Changed | Regenerate the existing error shape. Formatting/type-alias/array syntax changes only. | +| [frontend/sync-client/src/services/types/DocumentUpdateResponse.ts](frontend/sync-client/src/services/types/DocumentUpdateResponse.ts) | Changed | Replace obsolete client `FastForwardUpdate`/`MergingUpdate` variants with `Accepted`/`StaleBase`. Align the client with CAS outcomes already introduced server-side and retained in v4. | +| [frontend/sync-client/src/services/types/DocumentVersion.ts](frontend/sync-client/src/services/types/DocumentVersion.ts) | Changed | Regenerate content bytes plus flattened v4 metadata: originating mtime and content size, without path/deletion. Matches the new content-only read/stale response. | +| [frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts](frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts) | Changed | Regenerate v4 content metadata with mtime and without namespace fields. Used by events, acknowledgements, and bootstrap heads. | +| [frontend/sync-client/src/services/types/PushContent.ts](frontend/sync-client/src/services/types/PushContent.ts) | Added | Add generated `Snapshot`/`Diff` transport payloads. No delete operation is allowed in the content resource. | +| [frontend/sync-client/src/services/types/PutFileContent.ts](frontend/sync-client/src/services/types/PutFileContent.ts) | Added | Add generated request ID, nullable content parent, mtime, and content payload. Makes exact content retry data typed. | +| [frontend/sync-client/src/services/types/PushFileManifest.ts](frontend/sync-client/src/services/types/PushFileManifest.ts) | Added | Add generated request ID, file manifest parent, and UUID-to-path map. Makes exact namespace retry data typed. | +| [frontend/sync-client/src/services/types/FileManifest.ts](frontend/sync-client/src/services/types/FileManifest.ts) | Added | Add generated file manifest ID and entries map. Namespace versions are independent of document versions. | +| [frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts](frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts) | Added | Add generated file manifest acceptance/stale variants. The client needs the canonical map when rebasing. | +| [frontend/sync-client/src/services/types/VaultEvent.ts](frontend/sync-client/src/services/types/VaultEvent.ts) | Added | Add generated content-or-file-manifest event union. Both mutation kinds participate in one sequence. | +| [frontend/sync-client/src/services/types/EventRecord.ts](frontend/sync-client/src/services/types/EventRecord.ts) | Added | Add generated event/request IDs and flattened event payload. Supports ordered replay and tracing of accepted operations. | +| [frontend/sync-client/src/services/types/EventBatch.ts](frontend/sync-client/src/services/types/EventBatch.ts) | Added | Add generated events and head watermark. Lets the client check complete, contiguous replay. | +| [frontend/sync-client/src/services/types/VaultSnapshot.ts](frontend/sync-client/src/services/types/VaultSnapshot.ts) | Added | Add the generated consistent file-manifest/content-head/watermark vault snapshot. Makes initial bootstrap independent of lossy notifications. | +| [frontend/sync-client/src/services/types/WebSocketServerMessage.ts](frontend/sync-client/src/services/types/WebSocketServerMessage.ts) | Changed | Replace `vaultUpdate` with `vaultEvents` while retaining cursor positions. Matches the ordered server stream. | +| `frontend/sync-client/src/services/types/CreateDocumentVersion.ts` | Removed | Remove the old optional-server-ID multipart/create request type. V4 creation is client-UUID content CAS followed by file manifest CAS. | +| `frontend/sync-client/src/services/types/DeleteDocumentVersion.ts` | Removed | Remove the old content deletion payload type. Deletion is absence from the submitted file manifest. | +| `frontend/sync-client/src/services/types/UpdateDocumentVersion.ts` | Removed | Remove the legacy content/path update type. V4 uses `PutFileContent` and independent namespace CAS. | +| `frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts` | Removed | Remove the old text-specific path/update request. Text merges locally and uses the same content CAS transport. | +| `frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts` | Removed | Remove the old document-list response type. Membership/bootstrap/replay use file manifests, snapshots, and event batches. | +| `frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts` | Removed | Remove the latest-documents/initial-sync update type. Ordered batches and a separate bootstrap vault snapshot replace its two roles. | + + +## Maintainer reading order + +For the protocol, read [the server README](sync-server/README.md), then server +`requests.rs`, `models.rs`, `mutations.rs`, `queries.rs`, and the WebSocket handler. +For the client, read [the client README](frontend/sync-client/README.md), then +`persistence/database.ts`, `sync-operations/file-manifest.ts`, `sync-operations/syncer.ts`, +and `file-operations/file-operations.ts`. The adapter interface is part of the +correctness argument and should be read before implementing an integration. + +Keep the distinction between a current file manifest version and the event watermark, +and between an accepted sent snapshot and today's disk contents, explicit in any +future change. Those distinctions are central to avoiding skipped namespace +changes and duplicated merges after a lost acknowledgement. diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index 35e63d50..cfdaa5bc 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -1,438 +1,85 @@ -# Sync Algorithm - -VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients. - -## Operational Transformation - -Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent. - -### Why OT? - -Traditional conflict resolution approaches: - -- **Last write wins**: Loses data, frustrating for users -- **Manual merging**: Interrupts workflow, requires user intervention -- **Version branching**: Complex, not suitable for real-time sync - -Operational transformation: - -- **Automatic**: No user intervention required -- **Preserves all edits**: No data loss -- **Real-time**: Changes appear immediately -- **Intuitive**: Behaviour matches user expectations - -## The reconcile-text Library - -VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents. - -### Why reconcile-text over CRDTs? - -VaultLink faces a **differential synchronisation** challenge: users edit Obsidian vaults with various editors (Obsidian desktop, Obsidian mobile, Vim, VS Code, or any text editor), often while offline. This means we only observe the **final state** of each document after editing, not the individual keystrokes or operations that produced it. - -**The fundamental problem**: - -- **CRDTs and traditional OT** require capturing every individual operation (each character insertion, deletion, cursor movement) -- **VaultLink's reality**: Users edit files with arbitrary tools, sync happens after the fact -- **What we know**: Parent version and two modified versions -- **What we don't know**: The sequence of operations that created those modifications - -**Why reconcile-text wins for this use case**: - -1. **Works with end states only**: reconcile-text performs conflict-free 3-way merging given just parent, left, and right versions—no operation history needed - -2. **Editor-agnostic**: Users can edit with any tool without requiring VaultLink-specific plugins or operation tracking - -3. **Offline-first**: Edits made while disconnected are merged cleanly when sync resumes, because we're diffing final states rather than replaying operations - -4. **No conflict markers**: Unlike Git merge, produces clean merged output without `<<<<<<<` markers that interrupt note-taking flow - -5. **Human text forgiveness**: For knowledge bases and documentation, a slightly imperfect merge (e.g., minor word order issues) is vastly preferable to manual conflict resolution - -6. **Simpler infrastructure**: No need for complex operation capture, transformation logs, or tombstone management that CRDTs require - -**The trade-off**: - -CRDTs excel when you control the entire editing infrastructure and can capture every operation. reconcile-text excels when you're synchronising independently-edited files—exactly VaultLink's scenario. The merge quality depends on Myers' diff algorithm rather than operation history, which is the correct trade-off for differential sync. - -For note-taking workflows where users value editor freedom and offline editing, this approach provides superior user experience compared to either CRDTs (which would require operation tracking) or Git-style merging (which requires manual conflict resolution). - -[Learn more about reconcile-text →](https://schmelczer.dev/reconcile) - -### How It Works - -Given three versions (parent, left, right), reconcile-text produces a merged result. - -**How reconcile-text works**: - -1. **Tokenisation**: Split text into words (using `BuiltinTokenizer::Word`) -2. **Three-way diff**: Compare parent→left and parent→right changes -3. **Merge**: Combine non-conflicting changes, prefer content preservation for conflicts -4. **Result**: Merged text with both edits applied - -**Example**: - -``` -Parent: "The quick brown fox" -User A: "The quick red fox" (changes "brown" → "red") -User B: "The very quick brown fox" (inserts "very ") - -Merged: "The very quick red fox" (both changes applied) -``` - -**Merge conditions**: Only `.md` and `.txt` files with valid UTF-8 get merged. Binary files or other extensions use last-write-wins. - -### Operation Types - -The algorithm handles these operations: - -- **Insert**: Add text at position -- **Delete**: Remove text from position -- **Retain**: Keep existing text unchanged - -### Transformation Process - -1. **Client A** makes edit and sends to server -2. **Client B** makes concurrent edit and sends to server -3. **Server** receives both edits -4. **Server** transforms operations to account for concurrent changes -5. **Server** applies merged result to database -6. **Server** sends transformed operations to both clients -7. **Clients** apply transformed operations locally - -## Sync State Management - -VaultLink maintains sync state to track which changes have been applied. - -### Version Vectors - -Each document has a version tracked by: - -- **Server version**: Incremented on each change -- **Client cursors**: Track which version each client has seen - -This enables: - -- Efficient syncing (only send changes since last sync) -- Conflict detection (concurrent edits to same version) -- Ordering of operations - -### Cursor Management - -Clients maintain a cursor position: - -```rust -struct Cursor { - vault_id: String, - client_id: String, - last_version: u64, - last_updated: DateTime, -} -``` - -On sync: - -1. Client sends cursor (last seen version) -2. Server returns all changes since that version -3. Client applies changes and updates cursor - -## Conflict Resolution Flow - -### Scenario: Concurrent Edits - -Two users edit the same paragraph simultaneously. - -**Initial state**: - -``` -Version 10: "The quick brown fox jumps over the lazy dog." -``` - -**User A's edit** (version 11): - -``` -"The quick brown fox jumps over the very lazy dog." -``` - -_Inserts "very " at position 40_ - -**User B's edit** (also from version 10): - -``` -"The quick red fox jumps over the lazy dog." -``` - -_Replaces "brown" with "red" at position 10_ - -### Server Processing - -1. **Receive User A's operation**: - - Base: version 10 - - Operation: Insert("very ", position=40) - - Apply to database → version 11 - -2. **Receive User B's operation**: - - Base: version 10 - - Operation: Replace("brown"→"red", position=10) - - **Conflict detected**: Base is version 10, but current is version 11 - -3. **Transform User B's operation**: - - Transform against User A's operation - - Adjust positions/content as needed - - Apply transformed operation → version 12 - -4. **Broadcast updates**: - - Send User A's operation to User B - - Send transformed User B's operation to User A - -### Final Result - -``` -Version 12: "The quick red fox jumps over the very lazy dog." -``` - -Both edits are preserved in the final document. - -## Edge Cases - -### 1. Delete vs Insert Conflict - -**Scenario**: User A deletes a paragraph while User B edits it. - -**Resolution**: - -- OT algorithm prioritizes preservation of content -- Insert operation is transformed to account for deletion -- Typically results in inserted content appearing nearby - -**Example**: - -``` -Base: "Line 1\nLine 2\nLine 3" - -User A: Delete Line 2 → "Line 1\nLine 3" -User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3" - -Result: "Line 1\nLine 2 modified\nLine 3" -``` - -(Insert takes precedence, preserving user content) - -### 2. Overlapping Edits - -**Scenario**: Two users edit overlapping regions. - -**Resolution**: - -- OT splits operations into non-overlapping segments -- Applies each segment independently -- Merges results - -### 3. Delete vs Delete - -**Scenario**: Two users delete overlapping text. - -**Resolution**: - -- Deletes are merged -- Final result has the union of deleted ranges removed - -### 4. Network Partitions - -**Scenario**: Client loses connection, makes edits offline, reconnects. - -**Resolution**: - -1. Client queues edits locally -2. On reconnect, sends all queued operations -3. Server applies OT against all operations that happened during partition -4. Client receives transformed operations and applies - -## Performance Characteristics - -### Time Complexity - -- **Single operation**: O(1) for most operations -- **Transformation**: O(n) where n is operation size -- **Conflict resolution**: O(m × n) where m is number of concurrent operations - -### Space Complexity - -- **Version history**: Grows with number of changes -- **Cursors**: O(clients × vaults) -- **Active operations**: Minimal (processed in real-time) - -### Optimisation - -VaultLink optimises for: - -- Small, frequent edits (typical typing patterns) -- Text documents (not binary files) -- Real-time processing (no batching delay) - -## Limitations - -### Binary and Non-Mergeable Files - -Only **`.md`** and **`.txt`** files get automatic merging. Everything else uses last-write-wins. - -**Binary detection**: - -- Files with NUL bytes (`0x00`) -- Files failing UTF-8 validation - -Even `.md` files are treated as binary if they fail UTF-8 checks. - -**Last-write-wins behaviour**: - -``` -User A uploads image.png → Server version 1 -User B uploads image.png → Server version 2 (A's upload lost) -``` - -**Workaround**: Avoid concurrent edits to non-text files. [See all limitations →](/guide/limitations) - -### Large Documents - -Very large documents (> 1MB) may have: - -- Higher transformation costs -- Slower sync times -- Increased memory usage - -**Workaround**: Split large documents or increase timeout settings. - -### Complex Formatting - -Markdown with complex structures may occasionally produce unexpected results: - -- Nested lists -- Tables -- Code blocks - -**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures. - -## Consistency Guarantees - -### Strong Consistency - -VaultLink provides **strong eventual consistency**: - -- All clients eventually converge to the same state -- Operations applied in causal order -- No data loss under normal operation - -### Ordering Guarantees - -- Operations from the same client are applied in order -- Concurrent operations may be applied in any order -- Final result is independent of operation order (commutative) - -### Durability - -- Operations are written to SQLite before acknowledgment -- SQLite ACID guarantees protect against data loss -- Clients retry failed uploads - -## Comparison with Other Approaches - -### Git-style Merging - -| Aspect | Git Merge | VaultLink OT | -| -------------------------- | ------------ | ----------------------- | -| Real-time | No | Yes | -| Manual conflict resolution | Yes | No | -| Branching | Yes | No | -| Automatic merge | Limited | Always | -| Use case | Code changes | Collaborative documents | - -### CRDTs (Conflict-free Replicated Data Types) - -| Aspect | CRDTs | VaultLink (reconcile-text) | -| ----------------------------- | ------------------------------------ | ------------------------------------------------- | -| **Operation tracking** | Required (every keystroke) | Not required (end states only) | -| **Editor freedom** | Limited (must use CRDT-aware editor) | Unlimited (any text editor works) | -| **Offline editing** | Requires operation log | Works with file comparison | -| **Server required** | No | Yes | -| **Memory overhead** | Higher (tombstones, metadata) | Lower (versions only) | -| **Infrastructure complexity** | Higher | Lower | -| **Best for** | Controlled editing environments | Independent file editing (Obsidian, Vim, VS Code) | - -**Key insight**: CRDTs are superior when you can capture every operation. reconcile-text is superior when users edit files independently with arbitrary tools—exactly VaultLink's scenario. - -### Last Write Wins - -| Aspect | LWW | VaultLink OT | -| --------------- | ---- | ------------ | -| Data loss | Yes | No | -| Simplicity | High | Medium | -| User experience | Poor | Excellent | -| Performance | Best | Good | - -## Algorithm Details - -### Transformation Rules - -When transforming operation `A` against operation `B`: - -1. **Insert vs Insert**: - - If positions equal: Order by client ID - - If different positions: Adjust positions - -2. **Insert vs Delete**: - - If insert in deleted range: Shift insert position - - If insert after delete: Adjust position by deleted length - -3. **Delete vs Delete**: - - If ranges overlap: Merge delete ranges - - If ranges disjoint: Adjust positions - -4. **Retain vs Any**: - - Retain operations don't conflict - - Simply adjust positions - -### Transformation Example - -```rust -// Pseudo-code for transformation -fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) { - match (op_a, op_b) { - (Insert(pos_a, text_a), Insert(pos_b, text_b)) => { - if pos_a < pos_b { - (op_a, Insert(pos_b + text_a.len(), text_b)) - } else if pos_a > pos_b { - (Insert(pos_a + text_b.len(), text_a), op_b) - } else { - // Same position, use client ID to break tie - if client_id_a < client_id_b { - (op_a, Insert(pos_b + text_a.len(), text_b)) - } else { - (Insert(pos_a + text_b.len(), text_a), op_b) - } - } - } - // ... other cases - } -} -``` - -## Best Practices - -### For Smooth Collaboration - -1. **Small edits**: Make small, focused changes for easier merging -2. **Coordinate major changes**: Discuss large refactors with team -3. **Monitor sync status**: Ensure changes are uploaded before signing off -4. **Test conflict resolution**: Verify behaviour matches expectations - -### For Developers - -1. **Text files preferred**: OT works best on text -2. **Limit file sizes**: Keep documents reasonably sized -3. **Binary files**: Use versioning or avoid concurrent edits -4. **Testing**: Test concurrent edit scenarios thoroughly - -## Further Reading - -- [reconcile-text library](https://crates.io/crates/reconcile-text) -- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation) -- [Data flow architecture →](/architecture/data-flow) +# Sync algorithm + +The current API v5 server stores immutable content versions and a separately +versioned file manifest. Clients reconcile snapshots and apply them through a +durable filesystem journal. The server does not merge competing edits. + +## Identity and ordering + +Each document has a UUID independent of its path. A file manifest maps live UUIDs +to paths; removing a mapping deletes membership. Updating content never restores +membership. Directories are implicit. + +Every accepted content or manifest update appends an event in the same SQLite +transaction. Event IDs order changes and serve as version IDs. A push names its +parent version and a unique request UUID. The server either accepts that exact +snapshot or returns current metadata as `StaleBase`. Retrying an accepted request +returns its original acknowledgement; changing its payload under the same request +UUID is rejected. + +## Reconciliation + +A new client reads a coherent vault snapshot, applies it durably, then replays +subsequent events. Replayed pages are folded to their final document heads and +manifest before changing local files. This avoids applying intermediate remote +edits that were already reverted. + +For text, the client compares the common base, current local bytes and current +remote bytes using `reconcile-text`. Configured extensions and valid UTF-8 without +NUL bytes determine whether text merging is allowed. Diffs optimize transport; +they do not authorize the server to merge a stale base. Large diff operation sets +are sent as snapshots. + +If only one side changed, that side is retained. Concurrent unmergeable changes +keep the server version while the journal archives displaced local bytes for +recovery. First contact with existing local text has no shared ancestor and uses +an empty parent. Concurrent edits may still require human review. + +Paths use a separate three-way decision: keep a local change when the server path +is unchanged, otherwise prefer the server path if both changed. Portable-name +collisions receive deterministic conflict names. A manifest deletion is not +reversed merely because another client edited the document; displaced bytes are +retained locally. + +## Filesystem application + +Before moving files, the client persists a journal describing sources, +destinations and replacement snapshots. It stages sources before installing any +destination, so swaps and rename cycles do not overwrite each other. Prepared +content is persisted before installation so a crash does not repeat a merge. + +External notifications retain logical document identities. Notifications received +during asynchronous filesystem reads or staging are incorporated before accepting +a source. An editor save to a staged document's old path remains part of that +document. Ignored and oversized physical occupants are protected during conflict +allocation. Adapters report atomic editor saves as content updates and suppress +notifications generated by the engine itself. + +The filesystem and metadata store have separate durability boundaries. Recovery +replays the journal before scanning, including when network sync is disabled. +Adapters must provide complete scans, coherent snapshots, exclusive vault access, +no-replace renames and durable writes. Recovery artifacts are excluded from sync. + +## Server restores and failures + +Numeric version equality cannot establish identity after restoring a backup. +Successful HTTP responses include a checkpoint with an event-incarnation token. +The client persists and echoes it; the server rejects a checkpoint from a discarded +history before executing its request. Each new event gets a fresh token even if +its numeric ID or request UUID was used on the discarded branch. + +After a mismatch, the client durably records recovery intent, retains pending +content, clears old history-dependent state and bootstraps against the restored +server. Clean files reconcile against the restored snapshot. Unsent edits with an +old base are preserved as separate local documents, with conflict names if needed. +Interrupted recovery saves are replayable. Backups must be restored with the server +stopped. + +WebSocket messages wake HTTP replay; missed notifications are recovered by replay +and transport liveness checks. Authentication and permanent errors remain visible; +transient errors retry according to client settings. Observer callback failures +are isolated from the background sync loop. + +API v5 clients and servers must be deployed together. V4 databases migrate without +removing history. The old Obsidian and CLI adapters still require the filesystem +and persistence contract required by the core engine. diff --git a/docs/guide/limitations.md b/docs/guide/limitations.md index 1c514939..87824d11 100644 --- a/docs/guide/limitations.md +++ b/docs/guide/limitations.md @@ -8,14 +8,9 @@ VaultLink works well for most Obsidian vaults, but has some constraints you shou Only **`.md`** and **`.txt`** files get automatic conflict-free merging. -Other file types (images, PDFs, etc.) use last-write-wins: - -``` -User A updates diagram.png → Server stores version 1 -User B updates diagram.png → Server stores version 2 (overwrites A's changes) -``` - -**Workaround**: Avoid editing the same non-text file simultaneously. +For other file types, a one-sided change is retained. Concurrent unmergeable +changes keep the server's version and archive displaced local bytes for recovery. +Review recovery artifacts when both devices edit a binary file. ### Binary Detection @@ -24,7 +19,7 @@ Files are treated as binary if they: - Contain NUL bytes (`0x00`) - Fail UTF-8 validation -Binary files within `.md` or `.txt` extensions still get last-write-wins (no merge). +Binary files within `.md` or `.txt` extensions follow the same unmergeable-content policy. ## Performance Constraints @@ -61,11 +56,11 @@ Rough estimates (varies by vault size and activity): - All versions stored indefinitely (no automatic cleanup) - Each vault is a separate SQLite database -- Deleted files marked as deleted (not purged) +- Deletion removes file-manifest membership; content history remains readable **Growth**: Version history grows with every change. A 10 MB vault with frequent edits might grow to 100+ MB over months. -**Cleanup**: Manual only (see [Advanced Configuration](/config/advanced#version-history-cleanup)). +Active journals, immutable merge bases and request receipts must not be independently deleted. There is no automatic safe history-pruning protocol. ### Implications @@ -92,7 +87,7 @@ Result: "The very quick red fox" ← Both changes preserved - Simultaneous edits to the same sentence - Large structural changes (moving sections around) -**Result**: Merged file might need manual cleanup in ~1-5% of concurrent edits. +Merged files may require manual review; there is no measured universal conflict-success rate. ## Scalability @@ -140,6 +135,14 @@ Not currently supported. Running multiple servers requires manual vault partitio - Tokens configured in server config file - No runtime user management +## Restored backups + +Restore server databases while the server is stopped. API v5 clients detect a +changed history even when version numbers are reused. Clean files reconcile with +the restored server; unsent edits are preserved as separate local documents. +Clients and server must be upgraded together. This does not replace independent +backups or protect against arbitrary physical-media corruption. + ## Known Edge Cases ### Simultaneous Deletes and Edits @@ -147,10 +150,10 @@ Not currently supported. Running multiple servers requires manual vault partitio ``` User A deletes note.md User B edits note.md -Result: Edit wins (file recreated with B's content) +Result: Manifest deletion removes the file; unsent edited bytes are retained for recovery. ``` -Operational transformation prioritises content preservation. +Content updates do not restore deleted membership. The client retains displaced bytes in its local recovery directory. ### Large File Uploads diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md index 6fa2848c..a5afbe4e 100644 --- a/frontend/deterministic-tests/README.md +++ b/frontend/deterministic-tests/README.md @@ -1,118 +1,134 @@ -# Deterministic Tests +# Sync test harnesses (API v4) -Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case. - -Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios. - -## How it works - -Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one. - -Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process. - -The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit. - -## Step types - -Clients always start with syncing disabled. - -**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited): - -- `create`, `update`, `rename`, `delete` -- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path. - -**Sync control:** - -- `sync` — wait for a specific client or all clients to finish pending operations -- `barrier` — retry until all clients converge to identical file state (60s timeout) -- `enable-sync` / `disable-sync` — simulate going online/offline -- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable -- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync` - -**WebSocket control** (per-client): - -- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client - -**Server control:** - -- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process -- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire. - -**Fault injection** (per-client): - -- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit. -- `wait-for-dropped-create-response` — wait until the armed drop has fired. - -**Assertions:** - -- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback - -## Running +Run the complete harness check from **any working directory**: ```sh -# Build server first -cd sync-server && cargo build --release && cd - - -# Run all tests -cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests - -# Filter by name -npm run test -w deterministic-tests -- --filter=rename - -# Control parallelism (default: number of CPU cores) -npm run test -w deterministic-tests -- -j 4 +E2E_WORKERS=2 /path/to/vault-link/scripts/e2e.sh ``` -## Adding a test +The `E2E_WORKERS` variable is the number of fuzz workers / simultaneous scripted tests. +The runner builds the server and the three in-scope frontend packages, then runs +the harness self-tests, recovery regressions, protocol/real-filesystem tests, +scripted scenarios, and seeded fuzz workers. It does not build the Obsidian plugin +or CLI application. Install frontend dependencies with `npm ci` first. Node's +built-in WebSocket, Rust, and a C compiler (`cc`, for test-only exclusive rename +and OS-lock primitives on macOS/Linux) are required. -1. Create `src/tests/my-scenario.test.ts`: +CI installs those dependencies in `frontend` before invoking the runner. It sets +`E2E_ARTIFACTS=logs/e2e` and uploads that same directory even when a suite fails. +The runner owns all test servers; no separate server or legacy cleanup script is +needed in the workflow. -```typescript -import type { TestDefinition } from "../test-definition"; +Each server gets a fresh temporary database, working directory, and port. Only +owned processes are terminated. There is no `pkill`, fixed database deletion, +or shared server PID file. Failing tests and teardown errors give nonzero exits; +the remaining suites still run. Logs, concrete fuzz traces, results, and failure +snapshots are retained in the printed artifact directory. Snapshots are captured +before teardown so cleanup failures retain the original evidence. -export const myScenarioTest: TestDefinition = { - description: - "Client 0 creates A.md offline. After syncing, both clients should have the file.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "hello" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s) => { - s.assertFileCount(1).assertContent("A.md", "hello"); - } - } - ] -}; +Environment options: + +- `E2E_WORKERS=1`: number of fuzz workers / simultaneous scripted tests (max 32). +- `E2E_SEED=1`: first uint32 seed; workers use successive seeds. +- `E2E_ITERATIONS=25`: generated conflict scenarios per worker. +- `E2E_TIMEOUT_SECONDS=900`: timeout for each build/suite/worker command. +- `E2E_ARTIFACTS=/absolute/path`: optional output directory; defaults to `mkdtemp`. + +## Running individual layers + +From `frontend`: + +```sh +# Rebuild protocol peers and run scripted tests (name filter optional). +npm run test --workspace deterministic-tests -- --filter v4- --concurrency 2 + +# Adapter/oracle/lifecycle self-tests and strict recovery regressions. +npm run test:harness + +# After rebuilding: real worker death, server SIGKILL/restart, CAS/WS ordering, +# and host filesystem contract checks. +npm run test:protocol + +npm run build --workspace sync-client --workspace test-client +node test-client/dist/cli.js --seed 42 --iterations 25 +node test-client/dist/cli.js --replay /path/to/trace-42.json ``` -The `verify` callback receives an `AssertableState` object with chainable assertion methods: +`test:harness` includes strict regressions for **engine** bugs. +A failing default run is not an expected-failure test or a harness skip. See +[`../test-support/README.md`](../test-support/README.md) for their scope. -```typescript -s.assertFileCount(n); // exact file count -s.assertFileExists("path"); // file must exist -s.assertFileNotExists("path"); // file must not exist -s.assertContent("path", "expected"); // exact content match -s.assertContains("path", "a", "b"); // all substrings present in file -s.assertContainsAny("path", "a", "b"); // at least one substring present -s.assertAnyFileContains("text"); // substring present in some file -s.assertNoFileContains("text"); // substring absent from every file -s.assertSubstringCount("path", "x", 3); // substring appears exactly N times -s.assertContentInAtMostOneFile("text"); // no duplicate content -s.ifFileExists("path", (s) => { /* … */ }); // conditional block -s.getContent("path"); // raw content (or "" if missing) -``` +## What the scripted runner checks -2. Register it in `src/test-registry.ts`: +Real `SyncClient` instances use the shared v4 adapter in `../test-support`. +External editor operations and engine filesystem operations are separate: engine +writes never generate self-notifications. Editor identity notifications continue +while transport is disabled. Delayed notifications can be explicitly queued and +flushed. Snapshots and persisted state are deep copies, never shared references. -```typescript -import { myScenarioTest } from "./tests/my-scenario.test"; +Scripted and seeded clients explicitly set `syncIntervalMs: 0`, with assertions +that it stays disabled. Barriers observe convergence; they do not wake the engine. +Network retry timers remain enabled. Separate fake-clock tests in +`test-support/polling.test.ts` exercise optional periodic reconciliation. -const TESTS = { - // ... - "my-scenario": myScenarioTest -}; -``` +A barrier checks byte-for-byte replica equality **and** the server's canonical +UUID/path map, file bytes, independently computed base/observed hashes, content +and manifest CAS parents, event watermark, +contiguous event log, and absence of pending requests/application journals. +Internal recovery files are excluded from the user-visible file list, but are +present in failure disk images. Scenario assertions separately check intended +content, document identity, or preservation markers; equality alone is not an +adequate data-loss oracle. + +Clients start disabled. `enable-sync` waits for the initial sync attempt/retries, +except while the server is intentionally paused: that attempt is tracked so a +later script step can resume the server. Thus sequential enables really establish +a winner. To test _independent creates_, first bootstrap both clients empty, then +disable them and create files. Initial adoption of pre-existing files during +bootstrap is a different policy from two already-initialized clients creating +distinct UUIDs at one path. + +## Controls and assertions + +- `create`, `update`, `create-bytes`, `rename`, `delete`: external editor actions. + Editor rename may explicitly replace its target; engine rename never may. +- `sync`, `barrier`, `enable-sync`, `disable-sync`. +- `reset`: restart transports; **not** a crash and not a database reset. +- `pause-websocket`, `resume-websocket`: buffer only WS delivery. HTTP catchup + remains active; this is deliberately not called "disconnect". +- `pause-observation`, `wait-for-observation`, `resume-observation`: explicitly + gate HTTP catchup as well as WS, including replies already in flight. The + checkpoint must be exercised and released, and held reads respect cancellation. +- `delay-notifications`, `flush-notifications`: delay editor notifications without + disabling preservation assertions. +- `drop-response`: `{client, kind: "create" | "content" | "manifest", point: +"before" | "after"}`. After faults fire only on an `Accepted` CAS response. +- `wait-for-response-drop`: explicit checkpoint; every armed fault must fire. + Every interrupted request must be retried with an identical request ID, target + and body before a different CAS is sent. Fault kinds match exactly. +- `rename-next-write`: inject an editor rename after the engine's durable install + rename. The historical name is retained for fixtures; v4 never writes in place. +- `pause-server`, `resume-server`: SIGSTOP/SIGCONT, not a crash. +- `crash-server`, `restart-server`: SIGKILL and restart with the same database/port. + Server-control scenarios have dedicated processes. +- `remember-identity`, `assert-identity`, `assert-files`, `assert-markers`. +- `assert-documents`: a complete expected set of `{key, path, content, conflict?}` + records. Content is text or a byte array. Keys bind UUIDs on the first successful + check and must retain them on later checks; new keys cannot reuse deleted UUIDs. + `conflict: true` resolves the UUID-derived conflict name for `path`. +- `assert-consistent` can provide `verify(state)`: exact text/bytes, UUID lookups, + `assertIdentity`, and `conflictPath(original)` for UUID-derived conflict names. + +Legacy timing scenarios are retained. Prefer explicit fault checkpoints and +barriers for new tests. Only known coherent-scan races and actually injected +transient network errors may retry; they still must converge before the deadline. +Other background errors, unhandled rejections, and cleanup failures fail the run. + +## Reproducibility limits + +Seeds and JSON traces reproduce harness choices and logical actions. They do not +control the OS scheduler, server thread scheduling, or production-generated UUIDs. +The recovery sweep enumerates adapter/save boundaries; worker termination proves +abrupt runtime loss. Neither is a proof covering every real filesystem or hardware +power-loss behavior. Run multiple seeds, repeat timing-sensitive cases, and keep +the scenario's independent oracle when reducing a failing trace. diff --git a/frontend/deterministic-tests/package.json b/frontend/deterministic-tests/package.json index 4bd82c74..4e7ee563 100644 --- a/frontend/deterministic-tests/package.json +++ b/frontend/deterministic-tests/package.json @@ -8,7 +8,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "npm run build && node dist/cli.js" + "test": "cargo build --release --manifest-path ../../sync-server/Cargo.toml && npm run build --workspace sync-client && npm run build && node dist/cli.js" }, "devDependencies": { "commander": "^14.0.2", diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts index 1f5f4706..2c1f495c 100644 --- a/frontend/deterministic-tests/src/cli.ts +++ b/frontend/deterministic-tests/src/cli.ts @@ -9,10 +9,11 @@ import { runWithConcurrency } from "./run-with-concurrency"; import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts"; import * as path from "node:path"; import * as fs from "node:fs"; -import { debugging, Logger } from "sync-client"; +import * as os from "node:os"; +import { Logger } from "sync-client"; const logger = new Logger(); -debugging.logToConsole(logger, { useColors: true }); +logger.onLogEmitted.add((line) => console.log(line.message)); process.on("unhandledRejection", (reason) => { logger.error(`Unhandled Rejection: ${reason}`); @@ -32,6 +33,8 @@ function testUsesPauseServer(test: TestDefinition): boolean { (step) => step.type === "pause-server" || step.type === "resume-server" || + step.type === "crash-server" || + step.type === "restart-server" || step.type === "resume-server-until-history-then-pause" ); } @@ -42,7 +45,7 @@ function testUsesPauseServer(test: TestDefinition): boolean { */ function findProjectRoot(): string { let dir = path.dirname(__filename); - const {root} = path.parse(dir); + const { root } = path.parse(dir); while (dir !== root) { if ( fs.existsSync(path.join(dir, "sync-server")) && @@ -114,11 +117,7 @@ async function runDedicatedServerTest( } return { name, result }; } finally { - try { - await server.stop(); - } catch { - // best-effort cleanup - } + await server.stop(); serverManager.untrack(server); } } @@ -138,6 +137,11 @@ async function main(): Promise { } const { filter, concurrency } = parseArgs(process.argv); + const artifacts = + process.env.E2E_ARTIFACTS ?? + fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-deterministic-")); + fs.mkdirSync(artifacts, { recursive: true }); + logger.info(`Artifacts: ${artifacts}`); const testsToRun: [string, TestDefinition][] = []; for (const [key, test] of Object.entries(TESTS)) { @@ -193,13 +197,7 @@ async function main(): Promise { allResults.push(...results); } finally { - try { - await sharedServer.stop(); - } catch (error) { - logger.warn( - `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}` - ); - } + await sharedServer.stop(); serverManager.untrack(sharedServer); } } @@ -221,6 +219,10 @@ async function main(): Promise { const passed = allResults.filter((r) => r.result.success); const failed = allResults.filter((r) => !r.result.success); + fs.writeFileSync( + path.join(artifacts, "deterministic-results.json"), + JSON.stringify(allResults, null, 2) + ); logger.info( `\n--- Results: ${passed.length}/${allResults.length} passed ---` diff --git a/frontend/deterministic-tests/src/crash-worker.ts b/frontend/deterministic-tests/src/crash-worker.ts new file mode 100644 index 00000000..c015fac6 --- /dev/null +++ b/frontend/deterministic-tests/src/crash-worker.ts @@ -0,0 +1,93 @@ +import { parentPort, workerData } from "node:worker_threads"; +import assert from "node:assert/strict"; +import { SyncClient } from "sync-client"; +import { + MemoryDisk, + MemoryPersistence, + type DiskImage, + type StoredClient +} from "../../test-support/storage"; +import { + classifyRequest, + type RequestRecord, + type RequestKind +} from "../../test-support/network"; + +export interface CrashWorkerInput { + disk: DiskImage; + stored: StoredClient; + stopAt?: RequestKind; + editAfterSync?: { path: string; content: string }; +} +const input = workerData as CrashWorkerInput; +const disk = new MemoryDisk(); +disk.restore(input.disk); +const persistence = new MemoryPersistence(input.stored); +const requests: RequestRecord[] = []; +let editedBase: { documentId: string; vaultUpdateId: number } | undefined; +const snapshot = () => ({ + disk: disk.image(), + stored: persistence.snapshot(), + requests, + editedBase +}); + +async function main() { + const client = await SyncClient.create({ + fs: disk.session(), + persistence, + fetch: async (url, init) => { + const record = classifyRequest(url, init); + if (record) requests.push(record); + const response = await fetch(url, init); + if (record && response.ok && input.stopAt === record.kind) { + const receipt = (await response.clone().json()) as { + type: string; + }; + if (receipt.type === "Accepted") { + parentPort!.postMessage({ + type: "checkpoint", + ...snapshot() + }); + // Parent terminates this worker. No destroy(), no state flush, + // no finally block, and no old runtime survives the restart. + await new Promise(() => {}); + } + } + return response; + } + }); + assert.equal( + client.getSettings().syncIntervalMs, + 0, + "Crash recovery must preserve the disabled polling setting" + ); + await client.start(); + await client.waitUntilFinished(); + if (input.editAfterSync) { + const state = persistence.snapshot().database!; + const documentId = Object.keys(state.local!).find( + (id) => state.local![id] === input.editAfterSync!.path + ); + assert(documentId, "Edit must target an established document"); + editedBase = { + documentId, + vaultUpdateId: state.documents![documentId].base!.vaultUpdateId + }; + await disk.userWrite( + input.editAfterSync.path, + new TextEncoder().encode(input.editAfterSync.content) + ); + await client.syncLocallyUpdatedFile({ + relativePath: input.editAfterSync.path + }); + await client.waitUntilFinished(); + } + await client.destroy(); + parentPort!.postMessage({ type: "finished", ...snapshot() }); + parentPort!.close(); +} +main().catch((error: unknown) => { + parentPort!.postMessage({ type: "error", error: String(error) }); + process.exit(1); +}); diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts index fb7fe582..36a029e5 100644 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -1,483 +1,340 @@ -import type { - HistoryEntry, - StoredDatabase, - SyncSettings, - RelativePath, - TextWithCursors -} from "sync-client"; import { SyncClient, - SyncResetError, - debugging, LogLevel, - utils + type HistoryEntry, + type SyncSettings } from "sync-client"; +import { MemoryDisk, MemoryPersistence } from "../../test-support/storage"; +import { NetworkFaults, type RequestKind } from "../../test-support/network"; import { assert } from "./utils/assert"; import { sleep } from "./utils/sleep"; import { withTimeout } from "./utils/with-timeout"; -import { - IS_SYNC_ENABLED_BY_DEFAULT, - WAIT_TIMEOUT_MS, - WEBSOCKET_CONNECT_TIMEOUT_MS, - WEBSOCKET_POLL_INTERVAL_MS -} from "./consts"; +import { WAIT_TIMEOUT_MS, WEBSOCKET_CONNECT_TIMEOUT_MS } from "./consts"; import { ManagedWebSocketFactory } from "./managed-websocket"; -export class DeterministicAgent extends debugging.InMemoryFileSystem { - public readonly clientId: number; - private readonly logger: (msg: string) => void; - private client!: SyncClient; - private data: Partial<{ - settings: Partial; - database: Partial; - }> = {}; - private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT; - private readonly syncErrors: Error[] = []; - private readonly pendingSyncOperations = new Set>(); +/** User actions live here; the engine only receives disk.session(). */ +export class DeterministicAgent { + public readonly disk = new MemoryDisk(); + public readonly persistence: MemoryPersistence; + public readonly network = new NetworkFaults(); + private client?: SyncClient; + private readonly errors: Error[] = []; + private readonly pending = new Set>(); private readonly wsFactory = new ManagedWebSocketFactory(); - private nextWriteRename: - | { - oldPath: RelativePath; - newPath: RelativePath; - } - | undefined; - private nextCreateResponseDrop: - | { - dropped: Promise; - resolveDropped: () => void; - } - | undefined; + private nextWriteRename?: { oldPath: string; newPath: string }; + private delayedNotifications = false; + private disposed = false; + private readonly notifications: (() => Promise)[] = []; public constructor( - clientId: number, - initialSettings: Partial, - logger: (msg: string) => void + public readonly clientId: number, + settings: Partial, + private readonly logger: (msg: string) => void ) { - super(); - this.clientId = clientId; - this.logger = logger; - this.data.settings = { ...initialSettings }; + this.persistence = new MemoryPersistence({ + settings: { + networkRetryIntervalMs: 25, + webSocketRetryIntervalMs: 50, + requestTimeoutMs: 5_000, + ...settings, + // Scripted and seeded scenarios must exercise notification delivery. + syncIntervalMs: 0 + } + }); + this.disk.boundary = async (label) => { + const rename = this.nextWriteRename; + // v4 installs through an exclusive rename, never write-in-place. + if ( + rename && + label.startsWith("durable:rename:") && + label.endsWith(`->${rename.oldPath}`) + ) { + this.nextWriteRename = undefined; + await this.rename(rename.oldPath, rename.newPath); + } + }; } public async init( - fetchImplementation: typeof globalThis.fetch + fetchImplementation: typeof globalThis.fetch = fetch ): Promise { this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: this.wrapFetch(fetchImplementation), + fs: this.disk.session(), + persistence: this.persistence, + fetch: this.network.wrap(fetchImplementation), webSocket: this.wsFactory.constructorFn }); - - this.client.logger.onLogEmitted.add((line) => { - const prefix = `[Client ${this.clientId}]`; - switch (line.level) { - case LogLevel.ERROR: - this.logger(`${prefix} ERROR: ${line.message}`); - break; - case LogLevel.WARNING: - this.logger(`${prefix} WARN: ${line.message}`); - break; - case LogLevel.INFO: - this.logger(`${prefix} INFO: ${line.message}`); - break; - case LogLevel.DEBUG: - this.logger(`${prefix} DEBUG: ${line.message}`); - break; - } - }); - - await this.client.start(); - - const connectionCheck = await this.client.checkConnection(); assert( - connectionCheck.isSuccessful, + this.client.getSettings().syncIntervalMs === 0, + "Correctness scenarios must run with periodic polling disabled" + ); + this.client.logger.onLogEmitted.add((line) => { + this.logger( + `[Client ${this.clientId}] ${line.level}: ${line.message}` + ); + if (this.disposed && line.level === LogLevel.ERROR) + throw new Error( + `Client ${this.clientId} error after cleanup: ${line.message}` + ); + if ( + line.level === LogLevel.ERROR && + !this.isRetryableFailure(line.message) + ) + this.errors.push(new Error(line.message)); + }); + await this.client.start(); + assert( + (await this.client.checkConnection()).isSuccessful, `Client ${this.clientId} connection check failed` ); + } - if (this.isSyncEnabled) { - await this.waitForWebSocket(); + public database() { + return this.persistence.snapshot().database; + } + public files(): Map { + return this.disk.userFiles(); + } + public async listFilesRecursively(): Promise { + return [...this.files().keys()].sort(); + } + public async getFileContent(path: string): Promise { + const bytes = this.files().get(path); + assert(bytes !== undefined, `Missing file: ${path}`); + return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + } + public async read(path: string): Promise { + const snapshot = await this.disk.readSnapshot(path); + assert(snapshot !== undefined, `Missing file: ${path}`); + return snapshot.content; + } + public async write(path: string, content: Uint8Array): Promise { + const created = !(await this.disk.exists(path)); + await this.disk.userWrite(path, content); + this.notify(() => + created + ? this.client!.syncLocallyCreatedFile(path) + : this.client!.syncLocallyUpdatedFile({ relativePath: path }) + ); + } + public async rename(oldPath: string, relativePath: string): Promise { + await this.disk.userRename(oldPath, relativePath); + this.notify(() => + this.client!.syncLocallyUpdatedFile({ oldPath, relativePath }) + ); + } + public async delete(path: string): Promise { + await this.disk.userDelete(path); + this.notify(() => this.client!.syncLocallyDeletedFile(path)); + } + public delayNotifications(): void { + this.delayedNotifications = true; + } + public flushNotifications(): void { + this.delayedNotifications = false; + for (const notify of this.notifications.splice(0)) this.enqueue(notify); + } + private notify(operation: () => Promise): void { + // Offline means transport disabled, not that editor identity events vanish. + if (this.delayedNotifications) this.notifications.push(operation); + else this.enqueue(operation); + } + private enqueue(operation: () => Promise): void { + const promise = operation().catch((error: unknown) => { + if (!this.isRetryableFailure(String(error))) + this.errors.push( + error instanceof Error ? error : new Error(String(error)) + ); + }); + this.pending.add(promise); + void promise.then(() => this.pending.delete(promise)); + } + private async drain(): Promise { + while (this.pending.size) await Promise.all([...this.pending]); + } + private isRetryableFailure(message: string): boolean { + // A coherent scan rejecting an editor race is a required adapter + // behavior. It must subsequently converge; never blanket-ignore errors. + return ( + this.network.isExpectedFailure(message) || + /^(Sync paused for retry: )?Error: File changed during scan: .+$/.test( + message + ) + ); + } + private assertHealthy(): void { + if (this.errors.length) { + const errors = this.errors.splice(0); + throw new AggregateError( + errors, + `Client ${this.clientId}: ${errors.map(String).join("; ")}` + ); } } - - public pauseWebSocket(): void { - this.log("Pausing WebSocket message delivery"); - this.wsFactory.pause(); - } - - public resumeWebSocket(): void { - this.log("Resuming WebSocket message delivery"); - this.wsFactory.resume(); - } - - public dropNextCreateResponse(): void { + public async waitForSync(): Promise { assert( - this.nextCreateResponseDrop === undefined, - `Client ${this.clientId} already has a create response drop armed` - ); - let resolveDropped!: () => void; - const dropped = new Promise((resolve) => { - resolveDropped = resolve; - }); - this.nextCreateResponseDrop = { - dropped, - resolveDropped - }; - this.log("Armed next create response drop"); - } - - public async waitForDroppedCreateResponse(): Promise { - assert( - this.nextCreateResponseDrop !== undefined, - `Client ${this.clientId} has no create response drop armed` + this.client!.getSettings().syncIntervalMs === 0, + "Periodic polling was enabled during a correctness scenario" ); await withTimeout( - this.nextCreateResponseDrop.dropped, + (async () => { + await this.drain(); + const deadline = Date.now() + WAIT_TIMEOUT_MS; + for (;;) { + try { + await this.client!.waitUntilFinished(); + break; + } catch (error) { + if ( + !this.isRetryableFailure(String(error)) || + Date.now() >= deadline + ) + throw error; + await sleep(30); + } + } + this.assertHealthy(); + })(), WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for create response drop` + `Client ${this.clientId} sync timeout` ); - this.log("Create response was dropped after server commit"); } - + public async disableSync(): Promise { + await this.drain(); + await this.client!.setSetting("isSyncEnabled", false); + this.assertHealthy(); + } + public async enableSync(wait = true): Promise { + // setSetting waits for a sync attempt in v4. Track it, but don't block + // the script from resuming a paused server or releasing a checkpoint. + this.enqueue(() => this.client!.setSetting("isSyncEnabled", true)); + const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; + while ( + !this.persistence.snapshot().settings?.isSyncEnabled && + Date.now() < deadline + ) + await sleep(1); + assert( + !!this.persistence.snapshot().settings?.isSyncEnabled, + "Enable did not persist" + ); + if (wait) await this.waitForSync(); + } + public async reset(): Promise { + await this.drain(); + await this.client!.reset(); + } + public pauseWebSocket(): void { + this.wsFactory.pause(); + } + public pauseObservation(): void { + this.pauseWebSocket(); + this.network.pauseObservation(); + } + public resumeObservation(): void { + this.network.resumeObservation(); + this.resumeWebSocket(); + } + public async waitForObservation(): Promise { + await withTimeout( + this.network.waitForObservation(), + WAIT_TIMEOUT_MS, + "HTTP observation checkpoint never reached" + ); + } + public resumeWebSocket(): void { + this.wsFactory.resume(); + } + public dropNextCreateResponse(): void { + this.network.arm("create"); + } + public dropNextResponse( + kind: RequestKind, + point: "before" | "after" = "after" + ): void { + this.network.arm(kind, point); + } + public async waitForDroppedCreateResponse(): Promise { + await withTimeout( + this.network.wait(), + WAIT_TIMEOUT_MS, + "Armed response drop never fired" + ); + } + public renameNextWrite(oldPath: string, newPath: string): void { + assert( + !this.nextWriteRename, + "Previous install/rename hook never fired" + ); + this.nextWriteRename = { oldPath, newPath }; + } public async waitForHistoryEntry( matches: (entry: HistoryEntry) => boolean, onMatch?: (entry: HistoryEntry) => void ): Promise { - const existing = this.client.getHistoryEntries().find(matches); - if (existing !== undefined) { - onMatch?.(existing); - return; - } - - await withTimeout( - new Promise((resolve) => { - const unsubscribe = this.client.onSyncHistoryUpdated.add(() => { - const entry = this.client - .getHistoryEntries() - .find(matches); - if (entry === undefined) { - return; - } - - unsubscribe(); - onMatch?.(entry); - resolve(); - }); - }), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for history entry` - ); - } - - public async waitForSync(): Promise { - this.log("Waiting for sync to complete..."); - // Drain agent-level sync operations first. These are the fire-and-forget - // promises from enqueueSync() that call into the SyncClient's methods. - // Without this, waitUntilFinished() might return before the SyncClient - // has even been told about the operation. - await this.drainPendingSyncOperations(); - await withTimeout( - this.client.waitUntilFinished(), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms` - ); - if (this.syncErrors.length > 0) { - const errors = this.syncErrors.splice(0); - throw new Error( - `Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}` - ); - } - this.log("Sync complete"); - } - - public async reset(): Promise { - this.log("Resetting client (clears tracked state, keeps disk files)"); - await this.drainPendingSyncOperations(); - await this.client.reset(); - if (this.isSyncEnabled) { - await this.waitForWebSocket(); - } - } - - public async disableSync(): Promise { - this.log("Disabling sync"); - // Drain pending enqueued operations before disabling so the SyncClient - // knows about all operations that were enqueued while sync was enabled. - await this.drainPendingSyncOperations(); - await this.client.setSetting("isSyncEnabled", false); - this.isSyncEnabled = false; - // Wait for in-flight operations to drain. Disabling sync triggers - // a reset, which aborts in-flight fetches with SyncResetError. + let unsubscribe = () => {}; try { await withTimeout( - this.client.waitUntilFinished(), + new Promise((resolve) => { + const check = () => { + const entry = + this.client!.getHistoryEntries().find(matches); + if (entry) { + onMatch?.(entry); + resolve(); + } + }; + unsubscribe = this.client!.onSyncHistoryUpdated.add(check); + check(); + }), WAIT_TIMEOUT_MS, - `Client ${this.clientId} disableSync drain timed out` + "History checkpoint not reached" ); - } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log("Disable sync drain interrupted by reset (expected)"); - } else { - throw error; - } + } finally { + unsubscribe(); } } - - public async enableSync(): Promise { - this.log("Enabling sync"); - await this.client.setSetting("isSyncEnabled", true); - this.isSyncEnabled = true; - await this.waitForWebSocket(); - } - - public async getFileContent(path: string): Promise { - const bytes = await this.read(path); - return new TextDecoder().decode(bytes); - } - - public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void { - assert( - this.nextWriteRename === undefined, - `Client ${this.clientId} already has a next-write rename armed` - ); - this.nextWriteRename = { oldPath, newPath }; - this.log(`Armed next write rename: ${oldPath} -> ${newPath}`); - } - public async cleanup(): Promise { - this.log("Cleaning up..."); - // Guard against uninitialized client (init() failed partway). - // The class field uses `!:` so TS thinks this is always defined, - // but at runtime it can be undefined when init() throws partway. - const maybeClient = this.client as SyncClient | undefined; - if (maybeClient === undefined) { - this.log("Client not initialized, nothing to clean up"); - return; + if (!this.client) return; + const errors: unknown[] = []; + this.resumeObservation(); + this.flushNotifications(); + try { + await this.waitForSync(); + } catch (error) { + errors.push(error); + } + try { + this.network.assertConsumed(); + assert(!this.nextWriteRename, "Install/rename hook never fired"); + } catch (error) { + errors.push(error); } try { - await this.drainPendingSyncOperations(); await withTimeout( - this.client.waitUntilFinished(), + this.client.destroy(), WAIT_TIMEOUT_MS, - `Client ${this.clientId} cleanup waitUntilFinished timed out` + "Destroy timed out" + ); + await withTimeout( + this.wsFactory.finish(), + WAIT_TIMEOUT_MS, + "WebSocket callbacks did not finish during cleanup" ); } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Cleanup interrupted by reset (expected): ${error}`); - } else { - this.log(`Cleanup waitUntilFinished failed: ${error}`); - } + errors.push(error); } - // Surface any background sync errors that arrived after the last - // waitForSync (e.g. between the final assert-consistent and here). - // Without this, regressions that fault the engine during the very - // last step of a test would be silently swallowed. - const pendingErrors = this.syncErrors.splice(0); - await this.client.destroy(); - this.log("Cleanup complete"); - if (pendingErrors.length > 0) { - throw new Error( - `Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}` - ); - } - } - - public override async read(path: RelativePath): Promise { - await Promise.resolve(); - return super.read(path); - } - - public override async write( - path: RelativePath, - content: Uint8Array - ): Promise { - await Promise.resolve(); - const isNew = !this.files.has(path); - await super.write(path, content); - - if (this.isSyncEnabled && isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyCreatedFile(path); - }); - } - - const { nextWriteRename } = this; - if ( - nextWriteRename !== undefined && - nextWriteRename.oldPath === path - ) { - this.nextWriteRename = undefined; - await super.rename( - nextWriteRename.oldPath, - nextWriteRename.newPath - ); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath: nextWriteRename.oldPath, - relativePath: nextWriteRename.newPath - }); - }); - } - // The rename consumed `path`. Skip the post-update enqueue below - // — it would send a syncLocallyUpdatedFile for a path that no - // longer exists. - return; - } - - if (!this.isSyncEnabled) { - return; - } - - if (!isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - } - - public override async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const result = await super.atomicUpdateText(path, updater); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - return result; - } - - public override async delete(path: RelativePath): Promise { - await super.delete(path); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyDeletedFile(path); - }); - } - } - - public override async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - await super.rename(oldPath, newPath); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - }); - } - } - - private async waitForWebSocket(): Promise { - const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; - while (!this.client.isWebSocketConnected && Date.now() < deadline) { - await sleep(WEBSOCKET_POLL_INTERVAL_MS); - } - assert( - this.client.isWebSocketConnected, - `Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms` - ); - } - - /** - * Wait until all agent-level enqueued sync operations have completed. - * Uses a loop because completing one operation can trigger new enqueues. - */ - private async drainPendingSyncOperations(): Promise { - while (this.pendingSyncOperations.size > 0) { - await utils.awaitAll([...this.pendingSyncOperations]); - } - } - - private enqueueSync(operation: () => Promise): void { - const promise = this.executeSyncOperation(operation).catch( - (error: unknown) => { - const err = - error instanceof Error ? error : new Error(String(error)); - this.log(`Background sync failed: ${err.message}`); - this.syncErrors.push(err); - } - ); - this.pendingSyncOperations.add(promise); - void promise.finally(() => { - this.pendingSyncOperations.delete(promise); - }); - } - - private async executeSyncOperation( - operation: () => Promise - ): Promise { + this.client = undefined; + this.disposed = true; try { - await operation(); + this.assertHealthy(); } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Sync operation interrupted by reset: ${error}`); - return; - } - if ( - error instanceof Error && - error.message.includes("has been destroyed") - ) { - this.log(`Sync operation interrupted by destroy: ${error}`); - return; - } - - throw error; + errors.push(error); } - } - - private log(message: string): void { - this.logger(`[Client ${this.clientId}] ${message}`); - } - - private wrapFetch( - fetchImplementation: typeof globalThis.fetch - ): typeof globalThis.fetch { - return async (input, init) => { - const response = await fetchImplementation(input, init); - const drop = this.nextCreateResponseDrop; - if ( - drop !== undefined && - DeterministicAgent.isCreateDocumentRequest(input, init) - ) { - this.nextCreateResponseDrop = undefined; - try { - await response.body?.cancel(); - } catch { - // Best-effort — body may already be consumed/closed. - } - drop.resolveDropped(); - throw new SyncResetError(); - } - return response; - }; - } - - private static isCreateDocumentRequest( - input: RequestInfo | URL, - init: RequestInit | undefined - ): boolean { - const method = - init?.method ?? - (typeof Request !== "undefined" && input instanceof Request - ? input.method - : "GET"); - if (method.toUpperCase() !== "POST") { - return false; - } - - const url = - input instanceof URL - ? input - : new URL(typeof input === "string" ? input : input.url); - return /\/documents\/?$/.test(url.pathname); + if (errors.length) + throw new AggregateError( + errors, + `Client ${this.clientId} cleanup failed: ${errors.map(String).join("; ")}` + ); } } diff --git a/frontend/deterministic-tests/src/managed-websocket.ts b/frontend/deterministic-tests/src/managed-websocket.ts index c759891b..7d1159b5 100644 --- a/frontend/deterministic-tests/src/managed-websocket.ts +++ b/frontend/deterministic-tests/src/managed-websocket.ts @@ -209,8 +209,13 @@ export class ManagedWebSocketFactory { // WS before the agent reconnects would silently see the new socket // start un-paused and miss the messages it meant to buffer. private currentlyPaused = false; + private disposed = false; public get constructorFn(): typeof globalThis.WebSocket { + const ensureActive = () => { + if (this.disposed) + throw new Error("WebSocket created after harness cleanup"); + }; const trackInstance = (instance: ManagedWebSocket): void => { this.instances.push(instance); if (this.currentlyPaused) { @@ -222,6 +227,7 @@ export class ManagedWebSocketFactory { url: string | URL, protocols?: string | string[] ) { + ensureActive(); super(url, protocols); trackInstance(this); } @@ -229,6 +235,21 @@ export class ManagedWebSocketFactory { return TrackedManagedWebSocket; } + public async finish(): Promise { + this.disposed = true; + await Promise.all( + this.instances.map((socket) => + socket.readyState === WebSocket.CLOSED + ? Promise.resolve() + : new Promise((resolve) => + socket.addEventListener("close", () => resolve(), { + once: true + }) + ) + ) + ); + } + public pause(): void { this.currentlyPaused = true; for (const ws of this.instances) { diff --git a/frontend/deterministic-tests/src/parse-args.ts b/frontend/deterministic-tests/src/parse-args.ts index 11c56f19..a6968d13 100644 --- a/frontend/deterministic-tests/src/parse-args.ts +++ b/frontend/deterministic-tests/src/parse-args.ts @@ -7,8 +7,8 @@ export interface CliArgs { } function parsePositiveInt(value: string): number { - const n = parseInt(value, 10); - if (isNaN(n) || n <= 0) { + const n = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(n) || n <= 0) { throw new InvalidArgumentError("must be a positive integer"); } return n; diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts index d3c5387d..03c3f418 100644 --- a/frontend/deterministic-tests/src/server-control.ts +++ b/frontend/deterministic-tests/src/server-control.ts @@ -20,10 +20,13 @@ export class ServerControl { private _port: number | undefined; private tempDir: string | undefined; private _isPaused = false; + private preserveOnStop = false; + private expectedExit = false; + private unexpectedExit?: Error; public constructor(serverPath: string, configPath: string, logger: Logger) { - this.serverPath = serverPath; - this.baseConfigPath = configPath; + this.serverPath = path.resolve(serverPath); + this.baseConfigPath = path.resolve(configPath); this.logger = logger; } @@ -38,6 +41,15 @@ export class ServerControl { return `http://localhost:${this.port}`; } + public get databaseDirectory(): string { + if (!this.tempDir) throw new Error("Server has no database directory"); + return path.join(this.tempDir, "databases"); + } + + public get isPaused(): boolean { + return this._isPaused; + } + public async start(): Promise { if (this.process !== null) { throw new Error("Server is already running"); @@ -66,14 +78,25 @@ export class ServerControl { } private async startOnce(): Promise { - const reservation = await findFreePort(); - this._port = reservation.port; + const reservation = + this._port === undefined ? await findFreePort() : undefined; + this._port ??= reservation!.port; const tmpBase = os.tmpdir(); - this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-")); - const tempConfigPath = path.join(this.tempDir, "config.yml"); - const dbDir = path.join(this.tempDir, "databases"); - - this.writeConfigFile(tempConfigPath, dbDir); + let tempConfigPath: string; + try { + this.tempDir ??= fs.mkdtempSync( + path.join(tmpBase, "vault-link-test-") + ); + tempConfigPath = path.join(this.tempDir, "config.yml"); + this.writeConfigFile( + tempConfigPath, + path.join(this.tempDir, "databases") + ); + } catch (error) { + reservation?.release(); + this.cleanupTempDir(); + throw error; + } this.logger.info( `Starting server: ${this.serverPath} (port ${this._port})` @@ -81,12 +104,15 @@ export class ServerControl { // Release the port reservation right before spawning to minimize // the TOCTOU window between port discovery and server binding. - reservation.release(); + reservation?.release(); this.process = spawn(this.serverPath, [tempConfigPath], { + cwd: this.tempDir, stdio: ["ignore", "pipe", "pipe"], detached: false }); + this.expectedExit = false; + this.unexpectedExit = undefined; this.process.stdout?.on("data", (data: Buffer) => { this.logger.info(`[SERVER] ${data.toString().trim()}`); @@ -102,6 +128,10 @@ export class ServerControl { const currentProcess = this.process; currentProcess.on("exit", (code, signal) => { + if (!this.expectedExit) + this.unexpectedExit = new Error( + `Owned server exited unexpectedly (code ${code}, signal ${signal})` + ); this.logger.info( `Server exited with code ${code}, signal ${signal}` ); @@ -139,7 +169,9 @@ export class ServerControl { ); } try { - const response = await fetch(pingUrl); + const response = await fetch(pingUrl, { + signal: AbortSignal.timeout(1_000) + }); if (response.ok) { this.logger.info("[SERVER] Ready"); return; @@ -194,11 +226,13 @@ export class ServerControl { public async stop(): Promise { const proc = this.process; if (proc?.pid === undefined) { - this.cleanupTempDir(); + if (!this.preserveOnStop) this.cleanupTempDir(); + if (this.unexpectedExit) throw this.unexpectedExit; return; } + this.expectedExit = true; - // Resume if paused — a SIGSTOP'd process ignores SIGKILL + // SIGKILL terminates even a stopped process; SIGCONT is harmless here. if (this._isPaused) { try { process.kill(proc.pid, "SIGCONT"); @@ -212,7 +246,7 @@ export class ServerControl { // Set up a promise that resolves when the process actually exits. const exitPromise = new Promise((resolve) => { - if (proc.exitCode !== null) { + if (proc.exitCode !== null || proc.signalCode !== null) { resolve(); return; } @@ -229,11 +263,49 @@ export class ServerControl { // Wait for the process to actually exit before cleaning up, // with a 5s safety timeout to avoid hanging forever. - await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]); + let timer: ReturnType | undefined; + try { + await Promise.race([ + exitPromise, + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + "Owned server did not exit after SIGKILL" + ) + ), + STOP_TIMEOUT_MS + ); + }) + ]); + } finally { + clearTimeout(timer); + } this.process = null; this._isPaused = false; - this.cleanupTempDir(); + if (!this.preserveOnStop) this.cleanupTempDir(); + if (this.unexpectedExit) throw this.unexpectedExit; + } + + /** A real process death: retain the exact database and port for restart. */ + public async crash(): Promise { + if (!this.isRunning()) throw new Error("Cannot crash a stopped server"); + this.preserveOnStop = true; + try { + await this.stop(); + } finally { + this.preserveOnStop = false; + } + } + + public async restart(): Promise { + if (!this.tempDir || this.isRunning()) + throw new Error( + "Restart requires a crashed server with retained storage" + ); + await this.startOnce(); } public isRunning(): boolean { @@ -273,23 +345,27 @@ export class ServerControl { // one `databases_directory_path:` (under `server:` and `database:` // respectively) const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8"); + for (const key of ["port", "databases_directory_path"]) { + if ( + (baseConfig.match(new RegExp(`^ ${key}:`, "gm")) ?? []) + .length !== 1 + ) + throw new Error(`Expected exactly one config key: ${key}`); + } const config = baseConfig .replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`) .replace( /^\s*databases_directory_path:\s*.+/m, - ` databases_directory_path: ${dbDir}` + ` databases_directory_path: ${JSON.stringify(dbDir)}` ); fs.writeFileSync(destPath, config); } private cleanupTempDir(): void { if (this.tempDir !== undefined) { - try { - fs.rmSync(this.tempDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup - } + fs.rmSync(this.tempDir, { recursive: true, force: true }); this.tempDir = undefined; + this._port = undefined; } } } diff --git a/frontend/deterministic-tests/src/server-manager.ts b/frontend/deterministic-tests/src/server-manager.ts index 76c624f7..53862fae 100644 --- a/frontend/deterministic-tests/src/server-manager.ts +++ b/frontend/deterministic-tests/src/server-manager.ts @@ -59,9 +59,8 @@ export class ServerManager { // Last-resort synchronous cleanup. Runs even when the process is // exiting via process.exit() from unhandledRejection / // uncaughtException — paths where async stopAll() cannot complete. - // SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the - // kernel keeps them as zombies holding the test's tmpdir, and the - // next CI run can't reuse the port. + // Kill only the server processes owned by this runner. SIGKILL also + // terminates a SIGSTOP'd process; forceKillSync's SIGCONT is harmless. process.on("exit", () => { for (const server of this.activeServers) { server.forceKillSync(); diff --git a/frontend/deterministic-tests/src/test-definition.ts b/frontend/deterministic-tests/src/test-definition.ts index bd832a50..3e4c5fe0 100644 --- a/frontend/deterministic-tests/src/test-definition.ts +++ b/frontend/deterministic-tests/src/test-definition.ts @@ -3,6 +3,18 @@ import type { AssertableState } from "./utils/assertable-state"; export interface ClientState { files: Map; clientFiles: Map[]; + manifests: Record[]; + canonical: Record; + bytes: Map; +} + +/** A scenario's independent, complete expected state. Keys survive moves; + * a new key must receive a previously unseen UUID, including after deletion. */ +export interface ExpectedDocument { + key: string; + path: string; + conflict?: boolean; + content: string | number[]; } export type TestStep = @@ -31,10 +43,41 @@ export type TestStep = | { type: "assert-consistent"; verify?: (state: AssertableState) => void } | { type: "pause-websocket"; client: number } | { type: "resume-websocket"; client: number } + | { + type: + | "pause-observation" + | "resume-observation" + | "wait-for-observation"; + client: number; + } | { type: "drop-next-create-response"; client: number } | { type: "wait-for-dropped-create-response"; client: number } | { type: "sleep"; ms: number } - | { type: "reset"; client: number }; + | { type: "reset"; client: number } + | { type: "create-bytes"; client: number; path: string; bytes: number[] } + | { + type: "drop-response"; + client: number; + kind: "create" | "content" | "manifest"; + point?: "before" | "after"; + } + | { type: "wait-for-response-drop"; client: number } + | { type: "delay-notifications" | "flush-notifications"; client: number } + | { + type: "remember-identity" | "assert-identity"; + path: string; + key: string; + } + | { + type: "assert-files"; + expected: Record; + absent?: string[]; + count?: number; + } + | { type: "assert-markers"; markers: string[]; removed?: string[] } + | { type: "assert-documents"; expected: ExpectedDocument[] } + | { type: "crash-server" } + | { type: "restart-server" }; export interface TestDefinition { description?: string; @@ -46,4 +89,5 @@ export interface TestResult { success: boolean; error?: string; duration?: number; + diagnostics?: unknown; } diff --git a/frontend/deterministic-tests/src/test-registry.ts b/frontend/deterministic-tests/src/test-registry.ts index 1a07b411..322fe049 100644 --- a/frontend/deterministic-tests/src/test-registry.ts +++ b/frontend/deterministic-tests/src/test-registry.ts @@ -1,4 +1,5 @@ import type { TestDefinition } from "./test-definition"; +import { V4_TESTS } from "./v4-regressions"; import { renameCreateConflictTest } from "./tests/rename-create-conflict.test"; import { renameChainTest } from "./tests/rename-chain.test"; import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test"; @@ -106,6 +107,7 @@ import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-wri import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test"; export const TESTS: Partial> = { + ...V4_TESTS, "rename-create-conflict": renameCreateConflictTest, "rename-chain": renameChainTest, "rename-update-conflict": renameUpdateConflictTest, diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts index cf99f7f2..2ea7db3a 100644 --- a/frontend/deterministic-tests/src/test-runner.ts +++ b/frontend/deterministic-tests/src/test-runner.ts @@ -13,6 +13,8 @@ import { IS_SYNC_ENABLED_BY_DEFAULT } from "./consts"; import { randomUUID } from "node:crypto"; +import { assertCanonical } from "../../test-support/canonical"; +import { ContentLedger } from "../../test-support/oracles"; export class TestRunner { private agents: DeterministicAgent[] = []; @@ -20,6 +22,8 @@ export class TestRunner { private readonly token: string; private readonly remoteUri: string; private readonly logger: Logger; + private vaultName = ""; + private readonly identities = new Map(); public constructor( serverControl: ServerControl, @@ -44,6 +48,8 @@ export class TestRunner { } this.logger.info(`Clients: ${test.clients}`); this.logger.info(`Steps: ${test.steps.length}`); + this.identities.clear(); + let diagnostics: TestResult["diagnostics"]; try { assert( @@ -61,6 +67,11 @@ export class TestRunner { await this.executeStep(step); } + assert( + this.serverControl.isRunning(), + "Server died before test completion" + ); + diagnostics = this.captureDiagnostics(); await this.cleanup(); const duration = Date.now() - startTime; @@ -71,25 +82,46 @@ export class TestRunner { duration }; } catch (error) { + diagnostics ??= this.captureDiagnostics(); const duration = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : String(error); this.logger.info(`\n✗ Test failed: ${name}`); this.logger.info(`Error: ${errorMessage}`); - await this.cleanup(); + try { + await this.cleanup(); + } catch (cleanupError) { + return { + success: false, + error: `${errorMessage}\nCleanup: ${String(cleanupError)}`, + duration, + diagnostics + }; + } return { success: false, error: errorMessage, + diagnostics, duration }; } } + private captureDiagnostics(): unknown { + return this.agents.map((agent) => ({ + client: agent.clientId, + database: agent.database(), + disk: agent.disk.image(), + requests: structuredClone(agent.network.requests) + })); + } + private async initializeAgents(count: number): Promise { assert(count > 0, `Client count must be positive, got ${count}`); const vaultName = `test-${randomUUID()}`; + this.vaultName = vaultName; this.logger.info( `Initializing ${count} agents with vault: ${vaultName}` ); @@ -129,6 +161,87 @@ export class TestRunner { private async executeStep(step: TestStep): Promise { switch (step.type) { + case "pause-observation": + this.getAgent(step.client).pauseObservation(); + break; + case "resume-observation": + this.getAgent(step.client).resumeObservation(); + break; + case "wait-for-observation": + await this.getAgent(step.client).waitForObservation(); + break; + case "create-bytes": + await this.getAgent(step.client).write( + step.path, + new Uint8Array(step.bytes) + ); + break; + case "drop-response": + this.getAgent(step.client).dropNextResponse( + step.kind, + step.point + ); + break; + case "wait-for-response-drop": + await this.getAgent(step.client).waitForDroppedCreateResponse(); + break; + case "delay-notifications": + this.getAgent(step.client).delayNotifications(); + break; + case "flush-notifications": + this.getAgent(step.client).flushNotifications(); + break; + case "remember-identity": + case "assert-identity": + await this.assertConsistent((state) => { + if (step.type === "remember-identity") + this.identities.set( + step.key, + state.documentId(step.path) + ); + else { + const id = this.identities.get(step.key); + assert( + id !== undefined, + `Identity ${step.key} was never recorded` + ); + state.assertIdentity(step.path, id); + } + }); + break; + case "assert-files": + await this.assertConsistent((state) => { + for (const [path, value] of Object.entries(step.expected)) + state.assertContent(path, value); + for (const path of step.absent ?? []) + state.assertFileNotExists(path); + if (step.count !== undefined) + state.assertFileCount(step.count); + }); + break; + case "assert-documents": + await this.assertConsistent((state) => + state.assertDocuments(step.expected, this.identities) + ); + break; + case "assert-markers": { + const ledger = new ContentLedger(); + for (const marker of step.markers) ledger.add(marker); + for (const marker of step.removed ?? []) + ledger.removedBy( + marker, + "trace records explicit deletion/overwrite" + ); + for (const agent of this.agents) + ledger.assertPreserved(agent.files()); + break; + } + case "crash-server": + await this.serverControl.crash(); + break; + case "restart-server": + await this.serverControl.restart(); + break; case "create": case "update": await this.getAgent(step.client).write( @@ -170,7 +283,9 @@ export class TestRunner { break; case "enable-sync": - await this.getAgent(step.client).enableSync(); + await this.getAgent(step.client).enableSync( + !this.serverControl.isPaused + ); break; case "pause-server": @@ -190,7 +305,9 @@ export class TestRunner { (entry) => entry.details.type === step.syncType && entry.details.relativePath === step.path, - () => { this.serverControl.pause(); } + () => { + this.serverControl.pause(); + } ); this.serverControl.resume(); await historySeen; @@ -305,6 +422,11 @@ export class TestRunner { // Snapshot all agents' file states upfront to minimize the window // where background sync could mutate state between reads. + const canonical = await assertCanonical( + this.agents, + `${this.remoteUri}/vaults/${encodeURIComponent(this.vaultName)}`, + this.token + ); const clientFiles: Map[] = []; for (const agent of this.agents) { const sortedFiles = (await agent.listFilesRecursively()).sort(); @@ -360,7 +482,12 @@ export class TestRunner { verify( new AssertableState({ files: clientFiles[0], - clientFiles + clientFiles, + manifests: this.agents.map( + (agent) => agent.database()!.local! + ), + canonical: canonical.fileManifest.entries, + bytes: this.agents[0].files() }) ); } catch (error) { @@ -384,10 +511,12 @@ export class TestRunner { } this.logger.info("\nCleaning up agents..."); - for (const agent of this.agents) { + const errors: unknown[] = []; + for (const agent of this.agents.splice(0)) { try { await agent.cleanup(); } catch (error) { + errors.push(error); this.logger.warn( `Agent cleanup error: ${error instanceof Error ? error.message : String(error)}` ); @@ -395,5 +524,10 @@ export class TestRunner { } this.agents = []; this.logger.info("Cleanup complete"); + if (errors.length) + throw new AggregateError( + errors, + `Cleanup failed: ${errors.map(String).join("; ")}` + ); } } diff --git a/frontend/deterministic-tests/src/utils/assertable-state.ts b/frontend/deterministic-tests/src/utils/assertable-state.ts index 7c6f192c..c208fa13 100644 --- a/frontend/deterministic-tests/src/utils/assertable-state.ts +++ b/frontend/deterministic-tests/src/utils/assertable-state.ts @@ -1,12 +1,103 @@ -import type { ClientState } from "../test-definition"; +import assert from "node:assert/strict"; +import type { ClientState, ExpectedDocument } from "../test-definition"; export class AssertableState { public readonly files: Map; public readonly clientFiles: Map[]; + public readonly manifests: Record[]; + public readonly canonical: Record; + public readonly bytes: Map; public constructor(state: ClientState) { this.files = state.files; this.clientFiles = state.clientFiles; + this.manifests = state.manifests; + this.canonical = state.canonical; + this.bytes = state.bytes; + } + + public documentId(path: string): string { + const id = Object.keys(this.canonical).find( + (id) => this.canonical[id] === path + ); + if (!id) throw new Error(`No document identity for ${path}`); + return id; + } + + public assertDocuments( + expected: readonly ExpectedDocument[], + identities: Map + ): this { + this.assertFileCount(expected.length); + assert.equal( + new Set(expected.map((doc) => doc.key)).size, + expected.length, + "Duplicate expected identity key" + ); + const next = new Map(identities); + const paths = new Set(); + for (const doc of expected) { + const path = doc.conflict ? this.conflictPath(doc.path) : doc.path; + assert( + !paths.has(path), + `Two expected documents resolved to ${path}` + ); + paths.add(path); + this.assertBytes( + path, + typeof doc.content === "string" + ? new TextEncoder().encode(doc.content) + : new Uint8Array(doc.content) + ); + const id = this.documentId(path); + const recorded = next.get(doc.key); + if (recorded !== undefined) this.assertIdentity(path, recorded); + else { + assert( + ![...next.values()].includes(id), + `New document ${doc.key} reused an existing or deleted UUID` + ); + next.set(doc.key, id); + } + } + // A failed check must not establish a partial identity oracle. + for (const [key, id] of next) identities.set(key, id); + return this; + } + + public conflictPath(original: string): string { + const dot = original.lastIndexOf("."); + const split = + dot > original.lastIndexOf("/") + 1 ? dot : original.length; + const matches = Object.entries(this.canonical).filter( + ([id, path]) => + path === + `${original.slice(0, split)} (conflict ${id})${original.slice(split)}` + ); + if (matches.length !== 1) + throw new Error( + `Expected one UUID-derived conflict path for ${original}, found ${matches.length}` + ); + return matches[0][1]; + } + + public assertIdentity(path: string, id: string): this { + if (this.documentId(path) !== id) + throw new Error( + `Document identity changed at ${path}: expected ${id}, got ${this.documentId(path)}` + ); + return this; + } + + public assertBytes(path: string, expected: Uint8Array): this { + const actual = this.bytes.get(path); + if ( + !actual || + actual.length !== expected.length || + actual.some((byte, i) => byte !== expected[i]) + ) + throw new Error(`Byte mismatch at ${path}`); + return this; } public assertFileCount(expected: number): this { diff --git a/frontend/deterministic-tests/src/v4-regressions.ts b/frontend/deterministic-tests/src/v4-regressions.ts new file mode 100644 index 00000000..75841fc4 --- /dev/null +++ b/frontend/deterministic-tests/src/v4-regressions.ts @@ -0,0 +1,253 @@ +import type { TestDefinition, TestStep } from "./test-definition"; + +const online: TestStep[] = [ + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" } +]; +const offline: TestStep[] = [ + { type: "disable-sync", client: 0 }, + { type: "disable-sync", client: 1 } +]; +export const V4_TESTS: Record = { + "v4-text-byte-and-component-boundaries": { + clients: 2, + steps: [ + ...online, + { type: "create", client: 0, path: "empty.md", content: "" }, + { + type: "create", + client: 0, + path: "bom.md", + content: "\ufefffirst\r\nlast" + }, + { + type: "create", + client: 0, + path: `${"a".repeat(252)}.md`, + content: "255 ASCII bytes" + }, + { + type: "create", + client: 0, + path: `${"é".repeat(126)}.md`, + content: "255 UTF-8 bytes" + }, + { type: "barrier" }, + { + type: "assert-files", + count: 4, + expected: { + "empty.md": "", + "bom.md": "\ufefffirst\r\nlast", + [`${"a".repeat(252)}.md`]: "255 ASCII bytes", + [`${"é".repeat(126)}.md`]: "255 UTF-8 bytes" + } + } + ] + }, + "v4-binary-bytes-and-independent-identities": { + clients: 2, + steps: [ + ...online, + ...offline, + { + type: "create-bytes", + client: 0, + path: "nested/blob.bin", + bytes: [0, 255, 254, 128, 13, 10] + }, + { + type: "create-bytes", + client: 1, + path: "nested/blob.bin", + bytes: [0, 254, 255, 129, 10, 13] + }, + ...online, + { + type: "assert-consistent", + verify: (state) => { + state + .assertFileCount(2) + .assertBytes( + "nested/blob.bin", + new Uint8Array([0, 255, 254, 128, 13, 10]) + ); + state.assertBytes( + state.conflictPath("nested/blob.bin"), + new Uint8Array([0, 254, 255, 129, 10, 13]) + ); + } + } + ] + }, + "v4-three-file-cycle-preserves-identities": { + clients: 2, + steps: [ + ...online, + { type: "create", client: 0, path: "a.md", content: "A" }, + { type: "create", client: 0, path: "b.md", content: "B" }, + { type: "create", client: 0, path: "c.md", content: "C" }, + { type: "barrier" }, + { type: "remember-identity", key: "a", path: "a.md" }, + { type: "remember-identity", key: "b", path: "b.md" }, + { type: "remember-identity", key: "c", path: "c.md" }, + { type: "disable-sync", client: 0 }, + { type: "delay-notifications", client: 0 }, + { type: "rename", client: 0, oldPath: "a.md", newPath: "temp.md" }, + { type: "rename", client: 0, oldPath: "b.md", newPath: "a.md" }, + { type: "rename", client: 0, oldPath: "c.md", newPath: "b.md" }, + { type: "rename", client: 0, oldPath: "temp.md", newPath: "c.md" }, + { + type: "update", + client: 0, + path: "c.md", + content: "A edited offline" + }, + { type: "flush-notifications", client: 0 }, + { type: "enable-sync", client: 0 }, + { type: "barrier" }, + { type: "assert-identity", key: "a", path: "c.md" }, + { type: "assert-identity", key: "b", path: "a.md" }, + { type: "assert-identity", key: "c", path: "b.md" }, + { + type: "assert-files", + count: 3, + expected: { + "a.md": "B", + "b.md": "C", + "c.md": "A edited offline" + }, + absent: ["temp.md"] + } + ] + }, + "v4-case-alias-and-file-directory-conflicts": { + clients: 2, + steps: [ + ...online, + ...offline, + { + type: "create", + client: 0, + path: "folder/note.md", + content: "nested file" + }, + { + type: "create", + client: 0, + path: "README.md", + content: "uppercase file" + }, + { + type: "create", + client: 1, + path: "folder", + content: "blocking file" + }, + { + type: "create", + client: 1, + path: "readme.md", + content: "lowercase file" + }, + ...online, + { + type: "assert-consistent", + verify: (state) => { + state + .assertFileCount(4) + .assertContent("folder/note.md", "nested file") + .assertContent("README.md", "uppercase file"); + state.assertAnyFileContains( + "blocking file", + "lowercase file" + ); + } + }, + { + type: "assert-markers", + markers: [ + "nested file", + "blocking file", + "uppercase file", + "lowercase file" + ] + } + ] + }, + "v4-server-sigkill-retains-acknowledged-state": { + clients: 2, + steps: [ + ...online, + { + type: "create", + client: 0, + path: "a.md", + content: "durable bytes" + }, + { type: "barrier" }, + { type: "remember-identity", key: "a", path: "a.md" }, + ...offline, + { type: "crash-server" }, + { type: "restart-server" }, + ...online, + { type: "assert-identity", key: "a", path: "a.md" }, + { + type: "assert-files", + count: 1, + expected: { "a.md": "durable bytes" } + } + ] + } +}; + +for (const kind of ["create", "content", "manifest"] as const) { + for (const point of ["before", "after"] as const) { + V4_TESTS[`v4-${kind}-${point}-commit-network-loss`] = { + clients: 2, + steps: [ + ...online, + ...(kind === "content" + ? ([ + { + type: "create", + client: 0, + path: "file.md", + content: "old bytes" + }, + { type: "barrier" }, + { + type: "remember-identity", + key: "updated-file", + path: "file.md" + } + ] as TestStep[]) + : []), + { type: "drop-response", client: 0, kind, point }, + { + type: kind === "content" ? "update" : "create", + client: 0, + path: "file.md", + content: "must survive retry" + }, + { type: "wait-for-response-drop", client: 0 }, + { type: "barrier" }, + ...(kind === "content" + ? [ + { + type: "assert-identity", + key: "updated-file", + path: "file.md" + } as TestStep + ] + : []), + { + type: "assert-files", + count: 1, + expected: { "file.md": "must survive retry" } + } + ] + }; + } +} diff --git a/frontend/deterministic-tests/tsconfig.json b/frontend/deterministic-tests/tsconfig.json index 7558871d..47e92325 100644 --- a/frontend/deterministic-tests/tsconfig.json +++ b/frontend/deterministic-tests/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "baseUrl": ".", "strict": true, + "skipLibCheck": true, "target": "ES2022", "module": "CommonJS", "esModuleInterop": true, diff --git a/frontend/deterministic-tests/webpack.config.js b/frontend/deterministic-tests/webpack.config.js index 6aee1547..12c1653f 100644 --- a/frontend/deterministic-tests/webpack.config.js +++ b/frontend/deterministic-tests/webpack.config.js @@ -2,8 +2,9 @@ const path = require("path"); const webpack = require("webpack"); module.exports = { - entry: "./src/cli.ts", + entry: { cli: "./src/cli.ts", "crash-worker": "./src/crash-worker.ts" }, target: "node", + externals: { "sync-client": "commonjs sync-client" }, mode: "production", optimization: { minimize: false @@ -21,7 +22,7 @@ module.exports = { }, output: { globalObject: "this", - filename: "cli.js", + filename: "[name].js", path: path.resolve(__dirname, "dist") }, plugins: [ diff --git a/frontend/local-client-cli/src/node-filesystem.test.ts b/frontend/local-client-cli/src/node-filesystem.test.ts index 85fb7a0f..55b286a4 100644 --- a/frontend/local-client-cli/src/node-filesystem.test.ts +++ b/frontend/local-client-cli/src/node-filesystem.test.ts @@ -51,7 +51,7 @@ test("NodeFileSystemOperations - exists with forward slashes", async () => { } }); -test("NodeFileSystemOperations - delete with forward slashes", async () => { +test("NodeFileSystemOperations - deleteFile with forward slashes", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); @@ -59,8 +59,10 @@ test("NodeFileSystemOperations - delete with forward slashes", async () => { await fsOps.write("test.txt", new TextEncoder().encode("test")); assert.equal(await fsOps.exists("test.txt"), true); - await fsOps.delete("test.txt"); + await fsOps.deleteFile("test.txt"); assert.equal(await fsOps.exists("test.txt"), false); + + await fsOps.deleteFile("test.txt"); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -154,9 +156,46 @@ test("NodeFileSystemOperations - handles paths with forward slashes on all platf const readContent = await fsOps.read(testPath); assert.equal(new TextDecoder().decode(readContent), "test"); - await fsOps.delete(testPath); + await fsOps.deleteFile(testPath); assert.equal(await fsOps.exists(testPath), false); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test("NodeFileSystemOperations - delete prunes empty directory trees", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); + const fsOps = new NodeFileSystemOperations(tempDir); + + try { + await fs.mkdir(path.join(tempDir, "parent", "child"), { + recursive: true + }); + + await fsOps.delete("parent"); + assert.equal(await fsOps.exists("parent"), false); + + await fsOps.delete("parent"); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test("NodeFileSystemOperations - delete does not remove files", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); + const fsOps = new NodeFileSystemOperations(tempDir); + + try { + await fsOps.write( + "parent/file.txt", + new TextEncoder().encode("preserve me") + ); + + await assert.rejects(async () => fsOps.delete("parent")); + assert.equal(await fsOps.exists("parent/file.txt"), true); + await assert.rejects(async () => fsOps.delete("parent/file.txt")); + assert.equal(await fsOps.exists("parent/file.txt"), true); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index ba95ab6a..521f76bd 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -15,7 +15,7 @@ import { toUnixPath } from "./path-utils"; export const VAULTLINK_DIR = ".vaultlink"; export class NodeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly basePath: string) { } + public constructor(private readonly basePath: string) {} public async listFilesRecursively( directory: RelativePath | undefined @@ -105,9 +105,36 @@ export class NodeFileSystemOperations implements FileSystemOperations { } public async delete(relativePath: RelativePath): Promise { + if (!relativePath) { + throw new Error("Cannot delete the vault root"); + } + const fullPath = path.join(this.basePath, relativePath); try { - await fs.unlink(fullPath); + await this.deleteDirectoryTree(fullPath); + } catch (error) { + throw new Error( + `Failed to delete directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + public async deleteFile(relativePath: RelativePath): Promise { + const fullPath = path.join(this.basePath, relativePath); + try { + const entry = await fs.lstat(fullPath).catch((error: unknown) => { + if (this.isMissing(error)) return undefined; + throw error; + }); + if (!entry) return; + if (!entry.isFile() || entry.nlink !== 1) { + throw new Error("Cannot unlink a non-regular or linked file"); + } + + await fs.unlink(fullPath).catch((error: unknown) => { + if (!this.isMissing(error)) throw error; + }); + await this.syncDirectory(path.dirname(fullPath)); } catch (error) { throw new Error( `Failed to delete file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` @@ -171,6 +198,40 @@ export class NodeFileSystemOperations implements FileSystemOperations { } } + private async deleteDirectoryTree(fullPath: string): Promise { + const entry = await fs.lstat(fullPath).catch((error: unknown) => { + if (this.isMissing(error)) return undefined; + throw error; + }); + if (!entry) return; + if (!entry.isDirectory()) { + throw new Error("Cannot delete a regular file as a directory"); + } + + for (const child of await fs.readdir(fullPath, { + withFileTypes: true + })) { + if (!child.isDirectory()) { + throw new Error( + `Directory contains a non-directory entry: ${path.join(fullPath, child.name)}` + ); + } + await this.deleteDirectoryTree(path.join(fullPath, child.name)); + } + + await fs.rmdir(fullPath); + await this.syncDirectory(path.dirname(fullPath)); + } + + private isMissing(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ); + } + private async walkDirectory( relativePath: string, files: RelativePath[] diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index ceb8bc2a..fec25c59 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -152,9 +152,20 @@ export class ObsidianFileSystemOperations implements FileSystemOperations { } public async delete(path: RelativePath): Promise { - if (!(await this.vault.adapter.trashSystem(normalizePath(path)))) { - return this.vault.adapter.remove(normalizePath(path)); + if (!path) { + throw new Error("Cannot delete the vault root"); } + await this.deleteDirectoryTree(normalizePath(path)); + } + + public async deleteFile(path: RelativePath): Promise { + path = normalizePath(path); + const entry = await this.vault.adapter.stat(path); + if (!entry) return; + if (entry.type !== "file") { + throw new Error(`Cannot delete non-file: ${path}`); + } + await this.vault.adapter.remove(path); } public async rename( @@ -173,4 +184,21 @@ export class ObsidianFileSystemOperations implements FileSystemOperations { return file; } + + private async deleteDirectoryTree(path: string): Promise { + const entry = await this.vault.adapter.stat(path); + if (!entry) return; + if (entry.type !== "folder") { + throw new Error(`Cannot delete non-directory: ${path}`); + } + + const children = await this.vault.adapter.list(path); + if (children.files.length > 0) { + throw new Error(`Cannot delete non-empty directory: ${path}`); + } + for (const folder of children.folders) { + await this.deleteDirectoryTree(folder); + } + await this.vault.adapter.rmdir(path, false); + } } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4d8218ba..5aa0bee9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,6 +9,7 @@ "sync-client", "obsidian-plugin", "test-client", + "deterministic-tests", "local-client-cli" ], "devDependencies": { @@ -21,26 +22,102 @@ "typescript-eslint": "8.41.0" } }, + "deterministic-tests": { + "version": "0.14.0", + "bin": { + "deterministic-tests": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^25.0.2", + "commander": "^14.0.2", + "sync-client": "file:../sync-client", + "ts-loader": "^9.5.4", + "tslib": "2.8.1", + "typescript": "5.9.3", + "webpack": "^5.103.0", + "webpack-cli": "^6.0.1" + } + }, + "deterministic-tests/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "deterministic-tests/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "deterministic-tests/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "local-client-cli": { "version": "0.14.0", - "dependencies": { - "commander": "^14.0.2", - "watcher": "^2.3.1" - }, "bin": { "vaultlink": "dist/cli.js" }, "devDependencies": { - "@types/node": "^24.8.1", + "@types/node": "^25.0.2", + "commander": "^14.0.2", "sync-client": "file:../sync-client", - "ts-loader": "^9.5.2", + "ts-loader": "^9.5.4", "tslib": "2.8.1", - "tsx": "^4.20.6", - "typescript": "5.8.3", - "webpack": "^5.99.9", + "tsx": "^4.21.0", + "typescript": "5.9.3", + "watcher": "^2.3.1", + "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } }, + "local-client-cli/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "local-client-cli/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "local-client-cli/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "node_modules/@codemirror/state": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz", @@ -52,20 +129,6 @@ "@marijn/find-cluster-break": "^1.0.0" } }, - "node_modules/@codemirror/view": { - "version": "6.38.1", - "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.1.tgz", - "integrity": "sha512-RmTOkE7hRU3OVREqFVITWHz6ocgBjv08GoePscAakgVQfciA3SGCEk7mb9IzwW61cKKmlTpHXG6DUE5Ubx+MGQ==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@codemirror/state": "^6.5.0", - "crelt": "^1.0.6", - "style-mod": "^4.1.0", - "w3c-keyname": "^2.2.4" - } - }, "node_modules/@discoveryjs/json-ext": { "version": "0.6.3", "dev": true, @@ -75,9 +138,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", - "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", "cpu": [ "ppc64" ], @@ -92,9 +155,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", - "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", "cpu": [ "arm" ], @@ -109,9 +172,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", - "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", "cpu": [ "arm64" ], @@ -126,9 +189,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", - "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", "cpu": [ "x64" ], @@ -143,9 +206,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", - "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", "cpu": [ "arm64" ], @@ -160,9 +223,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", - "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", "cpu": [ "x64" ], @@ -177,9 +240,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", - "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", "cpu": [ "arm64" ], @@ -194,9 +257,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", - "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", "cpu": [ "x64" ], @@ -211,9 +274,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", - "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", "cpu": [ "arm" ], @@ -228,9 +291,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", - "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", "cpu": [ "arm64" ], @@ -245,9 +308,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", - "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", "cpu": [ "ia32" ], @@ -262,9 +325,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", - "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", "cpu": [ "loong64" ], @@ -279,9 +342,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", - "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", "cpu": [ "mips64el" ], @@ -296,9 +359,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", - "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", "cpu": [ "ppc64" ], @@ -313,9 +376,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", - "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", "cpu": [ "riscv64" ], @@ -330,9 +393,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", - "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", "cpu": [ "s390x" ], @@ -347,9 +410,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", - "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", "cpu": [ "x64" ], @@ -364,9 +427,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", - "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", "cpu": [ "arm64" ], @@ -381,9 +444,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", - "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", "cpu": [ "x64" ], @@ -398,9 +461,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", - "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", "cpu": [ "arm64" ], @@ -415,9 +478,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", - "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", "cpu": [ "x64" ], @@ -432,9 +495,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", - "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", "cpu": [ "arm64" ], @@ -449,9 +512,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", - "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", "cpu": [ "x64" ], @@ -466,9 +529,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", - "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", "cpu": [ "arm64" ], @@ -483,9 +546,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", - "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", "cpu": [ "ia32" ], @@ -500,9 +563,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", - "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", "cpu": [ "x64" ], @@ -723,6 +786,8 @@ }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", "engines": { @@ -730,7 +795,9 @@ } }, "node_modules/@jridgewell/source-map": { - "version": "0.3.6", + "version": "0.3.11", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.11.tgz", + "integrity": "sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==", "dev": true, "license": "MIT", "dependencies": { @@ -739,14 +806,16 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.0", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.30", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", - "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, "license": "MIT", "dependencies": { @@ -759,7 +828,8 @@ "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz", "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -961,26 +1031,10 @@ "@types/tern": "*" } }, - "node_modules/@types/eslint": { - "version": "9.6.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/eslint-scope": { - "version": "3.7.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/eslint": "*", - "@types/estree": "*" - } - }, "node_modules/@types/estree": { - "version": "1.0.7", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "dev": true, "license": "MIT" }, @@ -1013,7 +1067,6 @@ "integrity": "sha512-8fz6oa6wEKZrhXWro/S3n2eRJqlRcIa6SlDh59FXJ5Wp5XRZ8B9ixpJDcjadHq47hMx0u+HW6SNa6LjJQ6NLtw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.41.0", @@ -1052,7 +1105,6 @@ "integrity": "sha512-gTtSdWX9xiMPA/7MV9STjJOOYtWwIJIYxkQxnSV1U3xcE+mnJSH3f6zI0RYP+ew66WSlZ5ed+h0VCxsvdC1jJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.41.0", "@typescript-eslint/types": "8.41.0", @@ -1453,7 +1505,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1485,7 +1536,6 @@ "version": "6.12.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1763,6 +1813,19 @@ "dev": true, "license": "MIT" }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.25", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.25.tgz", + "integrity": "sha512-gMmEShwwq7FJqMwvfRwvCl00v4kN+KOfJqXn+f4nrufak5gNHJOksd/60Dvjuz7sI8Y5WiSFBa8FEYr+zoyqCw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/big.js": { "version": "5.2.2", "dev": true, @@ -1804,7 +1867,9 @@ } }, "node_modules/browserslist": { - "version": "4.24.4", + "version": "4.29.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", + "integrity": "sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==", "dev": true, "funding": [ { @@ -1821,12 +1886,12 @@ } ], "license": "MIT", - "peer": true, "dependencies": { - "caniuse-lite": "^1.0.30001688", - "electron-to-chromium": "^1.5.73", - "node-releases": "^2.0.19", - "update-browserslist-db": "^1.1.1" + "baseline-browser-mapping": "^2.11.23", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.427", + "node-releases": "^2.0.55", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -1860,6 +1925,8 @@ }, "node_modules/buffer-from": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true, "license": "MIT" }, @@ -1983,7 +2050,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001707", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -2038,14 +2107,16 @@ } }, "node_modules/chokidar": { - "version": "4.0.3", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", "dev": true, "license": "MIT", "dependencies": { - "readdirp": "^4.0.1" + "readdirp": "^5.0.0" }, "engines": { - "node": ">= 14.16.0" + "node": ">= 20.19.0" }, "funding": { "url": "https://paulmillr.com/funding/" @@ -2232,6 +2303,7 @@ "version": "14.0.2", "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=20" @@ -2286,7 +2358,8 @@ "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -2442,10 +2515,15 @@ "node": ">=0.10" } }, + "node_modules/deterministic-tests": { + "resolved": "deterministic-tests", + "link": true + }, "node_modules/dettle": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/dettle/-/dettle-1.0.5.tgz", "integrity": "sha512-ZVyjhAJ7sCe1PNXEGveObOH9AC8QvMga3HJIghHawtG7mE4K5pW9nz/vDGAr/U7a3LWgdOzEE7ac9MURnyfaTA==", + "dev": true, "license": "MIT" }, "node_modules/dunder-proto": { @@ -2771,7 +2849,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.127", + "version": "1.5.433", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.433.tgz", + "integrity": "sha512-5lCAbyZBjtmUt/RAGHRqrL2q0oEFRThDAsZHHDn9XHa89Qw7gMYOeSicBTy+AHfvo0r6vwsZvqNJTQIQy1BLzA==", "dev": true, "license": "ISC" }, @@ -2889,12 +2969,14 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.18.1", + "version": "5.25.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.25.1.tgz", + "integrity": "sha512-nGXts5znJzmWPu+mIE9izCOzdg63oJca2mDzGWWTth7sr4aCToKcoyFVBQwN75Ij5Pf6p510EwkTqViTRzDV+w==", "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" + "tapable": "^2.3.3" }, "engines": { "node": ">=10.13.0" @@ -2928,7 +3010,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.6.0", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, "license": "MIT" }, @@ -2944,9 +3028,9 @@ } }, "node_modules/esbuild": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", - "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -2957,32 +3041,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.9", - "@esbuild/android-arm": "0.25.9", - "@esbuild/android-arm64": "0.25.9", - "@esbuild/android-x64": "0.25.9", - "@esbuild/darwin-arm64": "0.25.9", - "@esbuild/darwin-x64": "0.25.9", - "@esbuild/freebsd-arm64": "0.25.9", - "@esbuild/freebsd-x64": "0.25.9", - "@esbuild/linux-arm": "0.25.9", - "@esbuild/linux-arm64": "0.25.9", - "@esbuild/linux-ia32": "0.25.9", - "@esbuild/linux-loong64": "0.25.9", - "@esbuild/linux-mips64el": "0.25.9", - "@esbuild/linux-ppc64": "0.25.9", - "@esbuild/linux-riscv64": "0.25.9", - "@esbuild/linux-s390x": "0.25.9", - "@esbuild/linux-x64": "0.25.9", - "@esbuild/netbsd-arm64": "0.25.9", - "@esbuild/netbsd-x64": "0.25.9", - "@esbuild/openbsd-arm64": "0.25.9", - "@esbuild/openbsd-x64": "0.25.9", - "@esbuild/openharmony-arm64": "0.25.9", - "@esbuild/sunos-x64": "0.25.9", - "@esbuild/win32-arm64": "0.25.9", - "@esbuild/win32-ia32": "0.25.9", - "@esbuild/win32-x64": "0.25.9" + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" } }, "node_modules/escalade": { @@ -3010,7 +3094,6 @@ "integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3179,11 +3262,6 @@ "node": ">=0.10.0" } }, - "node_modules/eventemitter3": { - "version": "5.0.1", - "dev": true, - "license": "MIT" - }, "node_modules/events": { "version": "3.3.0", "dev": true, @@ -3555,7 +3633,9 @@ } }, "node_modules/fs-extra": { - "version": "11.3.0", + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", "dev": true, "license": "MIT", "dependencies": { @@ -3667,19 +3747,6 @@ "node": ">=6" } }, - "node_modules/get-tsconfig": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz", - "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -3759,11 +3826,6 @@ "node": ">=0.10.0" } }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "dev": true, - "license": "BSD-2-Clause" - }, "node_modules/globals": { "version": "14.0.0", "dev": true, @@ -4240,7 +4302,9 @@ } }, "node_modules/immutable": { - "version": "5.1.1", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.9.tgz", + "integrity": "sha512-m8nVez3rwrgmWxtLMt1ZYXB2Lv7OKYn/disyxAlSDYAlKSlFoPPfIAmAM/M5xqL4m4C/wAPw7S2/CNaUii1Hxg==", "dev": true, "license": "MIT" }, @@ -4548,6 +4612,21 @@ "node": ">=0.10.0" } }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -4566,11 +4645,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "dev": true, - "license": "MIT" - }, "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, @@ -4694,14 +4768,6 @@ "iconv-lite": "^0.4.15" } }, - "node_modules/loader-runner": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.11.5" - } - }, "node_modules/loader-utils": { "version": "2.0.4", "dev": true, @@ -4812,6 +4878,8 @@ }, "node_modules/merge-stream": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true, "license": "MIT" }, @@ -4838,24 +4906,15 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -4867,7 +4926,9 @@ } }, "node_modules/mini-css-extract-plugin": { - "version": "2.9.2", + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.10.2.tgz", + "integrity": "sha512-AOSS0IdEB95ayVkxn5oGzNQwqAi2J0Jb/kKm43t7H73s8+f5873g0yuj0PNvK4dO75mu5DHg4nlgp4k6Kga8eg==", "dev": true, "license": "MIT", "dependencies": { @@ -4889,7 +4950,6 @@ "version": "8.17.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4956,6 +5016,154 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minimizer-webpack-plugin": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.11.0.tgz", + "integrity": "sha512-2JVy2DY+iBtfRyrQB5cRM7lfurnJIi6jFTgLip/7FxDu0WxswrIRzr4fBySQ0X7HqM7BcwE28zm+HfQ5V6y4Cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.3", + "terser": "^5.51.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, + "@napi-rs/image": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "html-minifier-terser": { + "optional": true + }, + "imagemin": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, + "sharp": { + "optional": true + }, + "svgo": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimizer-webpack-plugin/node_modules/schema-utils": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -5038,22 +5246,16 @@ "license": "MIT", "optional": true }, - "node_modules/node-gyp-build": { - "version": "4.8.4", + "node_modules/node-releases": { + "version": "2.0.56", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.56.tgz", + "integrity": "sha512-x0InOIyzgdk+eyaWaRJFH5snEtiImgBgblZ2CyPrLmqqcuMQkEvcDPHbzqbD8eDsSeJbVOjn+crzyzHaM4D+/A==", "dev": true, "license": "MIT", - "optional": true, - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" + "engines": { + "node": ">=18" } }, - "node_modules/node-releases": { - "version": "2.0.19", - "dev": true, - "license": "MIT" - }, "node_modules/normalize-path": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", @@ -5170,21 +5372,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obsidian": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.10.2.tgz", - "integrity": "sha512-bX03YCHf06OTzI/D+QK71ajCPCmwr/cjxzlVXjQa10DjK5iHRWhtJJpp83arSCyayFMp23u+UHcY7hxcEx2Mvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/codemirror": "5.60.8", - "moment": "2.29.4" - }, - "peerDependencies": { - "@codemirror/state": "6.5.0", - "@codemirror/view": "6.38.1" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5329,32 +5516,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-queue": { - "version": "8.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^5.0.1", - "p-timeout": "^6.1.2" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-timeout": { - "version": "6.1.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-try": { "version": "2.2.0", "dev": true, @@ -5531,7 +5692,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.8", "picocolors": "^1.1.1", @@ -5648,6 +5808,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/promise-make-counter/-/promise-make-counter-1.0.2.tgz", "integrity": "sha512-FJAxTBWQuQoAs4ZOYuKX1FHXxEgKLEzBxUvwr4RoOglkTpOjWuM+RXsK3M9q5lMa8kjqctUrhwYeZFT4ygsnag==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-naked": "^3.0.2" @@ -5657,6 +5818,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/promise-make-naked/-/promise-make-naked-3.0.2.tgz", "integrity": "sha512-B+b+kQ1YrYS7zO7P7bQcoqqMUizP06BOyNSBEnB5VJKDSWo8fsVuDkfSmwdjF0JsRtaNh83so5MMFJ95soH5jg==", + "dev": true, "license": "MIT" }, "node_modules/pseudomap": { @@ -5743,14 +5905,6 @@ ], "license": "MIT" }, - "node_modules/randombytes": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -5775,11 +5929,13 @@ "license": "MIT" }, "node_modules/readdirp": { - "version": "4.1.2", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", + "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", "dev": true, "license": "MIT", "engines": { - "node": ">= 14.18.0" + "node": ">= 20.19.0" }, "funding": { "type": "individual", @@ -5944,16 +6100,6 @@ "node": ">= 0.10" } }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, "node_modules/resolve-url-loader": { "version": "5.0.0", "dev": true, @@ -6039,22 +6185,21 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.91.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.91.0.tgz", - "integrity": "sha512-aFOZHGf+ur+bp1bCHZ+u8otKGh77ZtmFyXDo4tlYvT7PWql41Kwd8wdkPqhhT+h2879IVblcHFglIMofsFd1EA==", + "version": "1.104.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.104.1.tgz", + "integrity": "sha512-yDA+1aIG3EHgN4V/BvuhCvu61FF6hEd4e+9DxikUm9U0CAGuvdIZ/UYy7qbOxjhbbWQraoyLlMuzdRGOSV5Bmw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "chokidar": "^4.0.0", - "immutable": "^5.0.2", + "chokidar": "^5.0.0", + "immutable": "^5.1.5", "source-map-js": ">=0.6.2 <2.0.0" }, "bin": { "sass": "sass.js" }, "engines": { - "node": ">=14.0.0" + "node": ">=20.19.0" }, "optionalDependencies": { "@parcel/watcher": "^2.4.1" @@ -6129,14 +6274,6 @@ "node": ">=10" } }, - "node_modules/serialize-javascript": { - "version": "6.0.2", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" - } - }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -6326,6 +6463,17 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -6415,14 +6563,16 @@ "node_modules/stubborn-fs": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", - "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==", + "dev": true }, "node_modules/style-mod": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/supports-color": { "version": "8.1.1", @@ -6454,11 +6604,17 @@ "link": true }, "node_modules/tapable": { - "version": "2.2.1", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true, "license": "MIT", "engines": { "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/term-size": { @@ -6552,12 +6708,14 @@ } }, "node_modules/terser": { - "version": "5.39.0", + "version": "5.51.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.51.2.tgz", + "integrity": "sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "@jridgewell/source-map": "^0.3.3", - "acorn": "^8.8.2", + "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, @@ -6569,14 +6727,15 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.14", + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", - "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "engines": { @@ -6590,22 +6749,50 @@ "webpack": "^5.1.0" }, "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, "@swc/core": { "optional": true }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, "esbuild": { "optional": true }, + "html-minifier-terser": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, "uglify-js": { "optional": true } } }, "node_modules/terser-webpack-plugin/node_modules/ajv": { - "version": "8.17.1", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6617,8 +6804,28 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/terser-webpack-plugin/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/terser-webpack-plugin/node_modules/ajv-keywords": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "dev": true, "license": "MIT", "dependencies": { @@ -6628,32 +6835,23 @@ "ajv": "^8.8.2" } }, - "node_modules/terser-webpack-plugin/node_modules/jest-worker": { - "version": "27.5.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, "node_modules/terser-webpack-plugin/node_modules/json-schema-traverse": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, "node_modules/terser-webpack-plugin/node_modules/schema-utils": { - "version": "4.3.0", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "ajv-keywords": "^5.1.0" }, "engines": { @@ -6671,15 +6869,6 @@ "dev": true, "license": "MIT" }, - "node_modules/terser/node_modules/source-map-support": { - "version": "0.5.21", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "node_modules/test-client": { "resolved": "test-client", "link": true @@ -6720,6 +6909,7 @@ "version": "2.7.4", "resolved": "https://registry.npmjs.org/tiny-readdir/-/tiny-readdir-2.7.4.tgz", "integrity": "sha512-721U+zsYwDirjr8IM6jqpesD/McpZooeFi3Zc6mcjy1pse2C+v19eHPFRqz4chGXZFw7C3KITDjAtHETc2wj7Q==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-counter": "^1.0.2" @@ -6795,22 +6985,41 @@ } }, "node_modules/ts-loader": { - "version": "9.5.2", + "version": "9.6.2", + "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.6.2.tgz", + "integrity": "sha512-R4iuczmtgxvtuI556s+hTZ6/7Ee03VCAk/l/M8LY1OAsUgB7YydsCxkgq9D9pKRaD7GJqUi2u8fp9zZP/ufjKA==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.0", - "enhanced-resolve": "^5.0.0", - "micromatch": "^4.0.0", - "semver": "^7.3.4", + "picomatch": "^4.0.0", "source-map": "^0.7.4" }, "engines": { "node": ">=12.0.0" }, "peerDependencies": { + "loader-utils": "*", "typescript": "*", - "webpack": "^5.0.0" + "webpack": "^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "loader-utils": { + "optional": true + } + } + }, + "node_modules/ts-loader/node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/ts-loader/node_modules/source-map": { @@ -6827,14 +7036,13 @@ "license": "0BSD" }, "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" + "esbuild": "~0.28.0" }, "bin": { "tsx": "dist/cli.mjs" @@ -6861,7 +7069,6 @@ "version": "5.8.3", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6931,7 +7138,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.3", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -7088,12 +7297,14 @@ "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/watcher": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/watcher/-/watcher-2.3.1.tgz", "integrity": "sha512-d3yl+ey35h05r5EFP0TafE2jsmQUJ9cc2aernRVyAkZiu0J3+3TbNugNcqdUJDoWOfL2p+bNsN427stsBC/HnA==", + "dev": true, "dependencies": { "dettle": "^1.0.2", "stubborn-fs": "^1.2.5", @@ -7101,11 +7312,12 @@ } }, "node_modules/watchpack": { - "version": "2.4.2", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", + "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", "dev": true, "license": "MIT", "dependencies": { - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" }, "engines": { @@ -7113,35 +7325,29 @@ } }, "node_modules/webpack": { - "version": "5.99.9", + "version": "5.111.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.111.1.tgz", + "integrity": "sha512-cNypaz0RP+S4cvQVi1M/3bRUkvNP0xZpL0TjFx+c6jg64VbxD+2weWDgY9UnjRI6dhZgtaj8DU76GGFcJ79xPQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@types/eslint-scope": "^3.7.7", - "@types/estree": "^1.0.6", + "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", - "acorn": "^8.14.0", - "browserslist": "^4.24.0", + "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.1", - "es-module-lexer": "^1.2.1", - "eslint-scope": "5.1.1", + "enhanced-resolve": "^5.25.0", + "es-module-lexer": "^2.1.0", "events": "^3.2.0", - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.2.0", - "mime-types": "^2.1.27", - "neo-async": "^2.6.2", - "schema-utils": "^4.3.2", - "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.3.11", - "watchpack": "^2.4.1", - "webpack-sources": "^3.2.3" + "mime-db": "^1.54.0", + "minimizer-webpack-plugin": "^5.7.0", + "schema-utils": "^4.5.0", + "tapable": "^2.3.0", + "watchpack": "^2.5.2", + "webpack-sources": "^3.5.1" }, "bin": { "webpack": "bin/webpack.js" @@ -7163,7 +7369,6 @@ "version": "6.0.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -7228,7 +7433,9 @@ } }, "node_modules/webpack-sources": { - "version": "3.2.3", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.5.1.tgz", + "integrity": "sha512-jyuiGJdtvY434z5bUZrjz67v76/ePNvFZTp9Mdz29IlH4+GPsgyGjiv0fKI+M7BdkU6ADjulUcKAd3tUK3WlEw==", "dev": true, "license": "MIT", "engines": { @@ -7236,10 +7443,11 @@ } }, "node_modules/webpack/node_modules/ajv": { - "version": "8.17.1", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -7251,8 +7459,28 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/webpack/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/webpack/node_modules/ajv-keywords": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "dev": true, "license": "MIT", "dependencies": { @@ -7262,39 +7490,23 @@ "ajv": "^8.8.2" } }, - "node_modules/webpack/node_modules/eslint-scope": { - "version": "5.1.1", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/webpack/node_modules/estraverse": { - "version": "4.3.0", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, "node_modules/webpack/node_modules/json-schema-traverse": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, "node_modules/webpack/node_modules/schema-utils": { - "version": "4.3.2", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "ajv-keywords": "^5.1.0" }, "engines": { @@ -7467,28 +7679,95 @@ "version": "0.14.0", "license": "MIT", "devDependencies": { - "@types/node": "^24.8.1", + "@types/node": "^25.0.2", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", - "fs-extra": "^11.3.0", - "mini-css-extract-plugin": "^2.9.2", - "obsidian": "1.10.2", - "reconcile-text": "^0.8.0", + "fs-extra": "^11.3.2", + "mini-css-extract-plugin": "^2.9.4", + "obsidian": "1.11.0", + "reconcile-text": "^0.11.0", "resolve-url-loader": "^5.0.0", - "sass": "^1.91.0", + "sass": "^1.96.0", "sass-loader": "^16.0.6", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.14", - "ts-loader": "^9.5.2", + "terser-webpack-plugin": "^5.3.16", + "ts-loader": "^9.5.4", "tslib": "2.8.1", - "tsx": "^4.20.6", - "typescript": "5.8.3", + "tsx": "^4.21.0", + "typescript": "5.9.3", "url": "^0.11.4", - "webpack": "^5.99.9", + "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } }, + "obsidian-plugin/node_modules/@codemirror/view": { + "version": "6.38.6", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.6.tgz", + "integrity": "sha512-qiS0z1bKs5WOvHIAC0Cybmv4AJSkAXgX5aD6Mqd2epSLlVJsQl8NG23jCVouIgkh4All/mrbdsf2UOLFnJw0tw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@codemirror/state": "^6.5.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "obsidian-plugin/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "obsidian-plugin/node_modules/obsidian": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.11.0.tgz", + "integrity": "sha512-lVqN9AmDWHzhNATi2tDnjqVgI6WUYKeT+lIsAycAyLt4XCC6zRsWzb+tFCiB7Rn3PpttefjoovilhYwvS4Iqxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/codemirror": "5.60.8", + "moment": "2.29.4" + }, + "peerDependencies": { + "@codemirror/state": "6.5.0", + "@codemirror/view": "6.38.6" + } + }, + "obsidian-plugin/node_modules/reconcile-text": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/reconcile-text/-/reconcile-text-0.11.0.tgz", + "integrity": "sha512-a3sy3obazoc1BMEHx6IQn8ESZKnakVWZuRLi7OSEB56E8evRtrXBMj7yuo10fMoG4JkcZC6tokOfzpwZAKX+PQ==", + "dev": true, + "license": "MIT" + }, + "obsidian-plugin/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "obsidian-plugin/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "sync-client": { "version": "0.14.0", "devDependencies": { @@ -7496,7 +7775,6 @@ "@types/node": "^24.8.1", "byte-base64": "^1.1.0", "minimatch": "^10.0.1", - "p-queue": "^8.1.0", "reconcile-text": "^0.8.0", "ts-loader": "^9.5.2", "tslib": "2.8.1", diff --git a/frontend/package.json b/frontend/package.json index 0dd9057d..0bbf94af 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,9 @@ "build": "npm run build --workspaces", "dev": "concurrently --kill-others \"npm run dev -w sync-client\" \"npm run dev -w obsidian-plugin\"", "test": "npm run test --workspaces", + "check:harness": "tsc --noEmit -p test-support/tsconfig.json && tsc --noEmit -p deterministic-tests/tsconfig.json && tsc --noEmit -p test-client/tsconfig.json", + "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/journal-interleavings.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", + "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/journal-protocol.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts'", "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md new file mode 100644 index 00000000..06d680ad --- /dev/null +++ b/frontend/sync-client/README.md @@ -0,0 +1,137 @@ +# Sync client (API v5) + +The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket +messages wake it; complete disk scans and the durable server event log are the +sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a +positive interval to repair missed notifications. Unset or `0` disables polling. +`syncLocallyCreatedFile`, `syncLocallyUpdatedFile`, and `syncLocallyDeletedFile` +durably record logical file changes before resolving, including while offline. +Shutdown waits for those saves. Notifications describe logical external changes: adapters suppress +self-generated create/delete/move notifications, and report atomic editor saves +as content updates. This distinguishes a deliberate delete/recreate from an edit. +Use `waitUntilFinished()` to await the active attempt; errors leave +pending work durable and are reported to the caller. Transient failures retry; +authentication and protocol errors wait for an explicit wake or settings change. + +## State and policy + +The persisted state contains the incorporated server file manifest, the current +local identity map, and the exact pending submission. A received server file +manifest is not a merge base until its local application is durable. Successful +submissions advance the base to the submitted content snapshot, preserving later +local edits. Unknown outcomes are retried unchanged before incorporating newer +remote state. Each contiguous event batch is folded into its final manifest and +content heads before merging. Intermediate changes that the server has already +reverted must not overwrite local edits. + +A permanent upload rejection is recorded locally, with content retained for +recovery. An unchanged rejected payload is skipped so other files can sync; the +failure remains visible in history and `waitUntilFinished()`. Editing it, resetting +the client, or changing settings permits a fresh attempt. Earlier attempts that +committed despite a lost reply are incorporated through normal event replay. + +For each document ID (absence means deletion), use the server path if local is +unchanged, use local if the server is unchanged, and use the server when both +changed. Then resolve cross-document path conflicts: a retained server path wins, +UUID ordering breaks other ties, and displaced documents receive deterministic +`(conflict UUID)` names. This also handles file/ancestor and portable-name aliases. +Nonportable local names are repaired client-side before a file manifest is published. +Allocated filename components fit within 255 UTF-8 bytes, including conflict suffixes. +The protocol does not impose a universal total-path byte limit; filesystem-specific +limits surface from the adapter instead of silently flattening a name. Directories +are implicit; empty directories are not synced. + +- Content is uploaded before new membership; the client chooses and persists UUIDs. +- Text uses three-way `reconcile`. First contact at a matching path adopts the + remote ID and reconciles differing text with an empty parent. +- Concurrent binary/unmergeable edits keep the server version and retain + displaced local bytes in recovery. If only one side changed, keep that side. +- Deletion follows the file manifest decision. Displaced edited bytes are retained + locally, without automatically restoring membership. +- Notified moves preserve IDs. Offline inference requires a unique nonempty hash + match on both sides; uncertain cases become delete/create. +- Ignored and oversized files preserve their disk paths and remote membership. + The original manifest base is retained while excluded, so becoming eligible + resumes reconciliation without losing local renames. Not-yet-downloaded files + are not mistaken for local deletions. +- Bytes and line endings are preserved. File modification times are not tracked + or synchronized. Optional editor cursors are repositioned when text is reconciled. + +## Restored server histories + +Successful HTTP responses carry an `X-Vault-Link-History` checkpoint. The client +persists it before consuming the response and supplies it on subsequent reads and +writes. WebSocket events wake HTTP replay, which validates that checkpoint. A +restored database cannot silently reuse a version number: a missing or different +event incarnation returns a history mismatch before executing the request. + +On mismatch the client first recovers its filesystem journal and scans local +work. It durably records a reset before clearing the old checkpoint, archives +pending content requests, and clears old receipts, cursors, bases and diff-cache +entries. Clean files reconcile against the restored snapshot. Unsent edits with +an old base become separate local documents; path conflicts preserve them under +conflict names instead of applying a diff to an unrelated restored version. +Recovery resumes after interrupted state saves. Existing API v4 client state +without a checkpoint takes this same conservative bootstrap on upgrade. + +Status observers cannot stop the background engine by throwing or rejecting. +Their failures are reported, and remaining observers continue receiving updates. + +## Required adapter contract + +`FileSystemOperations` and `PersistenceProvider` define the adapter operations +needed by the recovery model. The Obsidian and CLI adapters have intentionally +not been updated in this rewrite. + +The sync client does not coordinate separate processes accessing the same local +vault. + +The filesystem adapter must provide: + +- Complete scans and coherent content snapshots. Missing and unreadable must + be distinguished; reject read races rather than return a partial snapshot. +- Vault-relative operations that reject symlinks, hardlinks, special files, and + ancestor symlink traversal. Staging and visible files share one filesystem. +- Atomic, complete, exclusive file creation (a crash leaves absent or complete + bytes, never a partial file) and atomic renames that **never replace** destinations. + Writes apply optional cursor metadata. +- `flushPaths` flushes existing files and ancestor directories, including parents + of absent paths. Recovery uses it to complete interrupted rename durability. +- Durable recursive directory creation, and directory pruning using only `rmdir` + operations (never unlink descendant files). All affected data/directory entries + must be flushed before a mutation resolves, including intermediate ancestors. + +Keep the persistence store outside the user file namespace or inside the reserved +`.vault-link-sync` directory, so writing sync state cannot generate user edits. + +Persistence `save` atomically replaces the complete state and resolves only after +its data and directory entry are durable. A failed save may leave either complete +value; the engine reloads before retrying. Settings saves are assumed to succeed +and do not reload after errors. Settings and engine saves are serialized. +Changing a client's vault once it holds identities, offline notifications, +pending requests, or bootstrap/application state requires a separate state store +and client. Lifecycle operations are serialized. + +Before changing visible files, the engine persists an application journal. It +stages every affected source before installing destinations, so swaps and cycles +work. Recovery runs before scanning, including while the server is unavailable. +Prepared content is persisted before installation, preventing repeated merges +following a crash. Unexpected occupants are preserved and given conflict names. +Notifications received while staging rebind the actual source identity before +content is merged. Editor saves to a temporarily staged path continue the original +document's journal, including saves racing preparation and installation. Ignored +or oversized occupants are reserved before conflict allocation. + +`.vault-link-sync/transactions` contains only active recovery artifacts. Staged +sources and prepared output are removed after every install is durably recorded; +completed journals are not archived. `.vault-link-sync/recovery` retains displaced +original bytes and their document/path metadata before staging files are deleted. +`.vault-link-sync/requests` retains rejected submission snapshots for explicit +recovery. These directories are never scanned +as user files. Pending requests and active recovery journals must not be +independently discarded. + +These guarantees assume the adapters/storage honor their durability contracts. +Arbitrary external writers can change a file between observations; detected races +are preserved/reconciled, but unobserved writes and offline identity cannot be +reconstructed perfectly. There is no automated v2/v3 persisted-state migration. diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index aa369fa7..bdc094cb 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -15,7 +15,6 @@ "devDependencies": { "byte-base64": "^1.1.0", "minimatch": "^10.0.1", - "p-queue": "^8.1.0", "reconcile-text": "^0.8.0", "uuid": "^13.0.0", "@types/node": "^24.8.1", diff --git a/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts b/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts new file mode 100644 index 00000000..22ed4c8e --- /dev/null +++ b/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts @@ -0,0 +1,428 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../persistence/database"; +import type { ServerConfig } from "../services/server-config"; +import { toStoredSnapshot } from "../sync-operations/content"; +import { isInternalPath } from "../utils/portable-path"; +import { Logger } from "../tracing/logger"; +import { FileOperations, type FileWrite } from "./file-operations"; +import type { + FileSnapshot, + FileSystemOperations +} from "./filesystem-operations"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const text = (snapshot: FileSnapshot) => + new TextDecoder().decode(snapshot.content); +const parent = (path: string) => path.split("/").slice(0, -1).join("/"); + +/** Atomic mutations with an injectable interruption before or after durability. */ +class MemoryFiles implements FileSystemOperations { + public readonly files = new Map(); + public readonly directories = new Set([""]); + public boundary: (label: string) => void = () => {}; + + public change(label: string, action: () => void): void { + this.boundary(`before ${label}`); + action(); + this.boundary(`after ${label}`); + } + + public put(path: string, value: string): void { + for ( + let directory = parent(path); + directory; + directory = parent(directory) + ) + this.directories.add(directory); + this.files.set(path, { content: bytes(value) }); + } + + public async listFilesRecursively(root = ""): Promise { + assert(this.directories.has(root), `Cannot list non-directory ${root}`); + return [...this.files.keys()].filter( + (path) => !root || path.startsWith(`${root}/`) + ); + } + + public async readSnapshot(path: string): Promise { + assert(!this.directories.has(path), `Cannot read directory ${path}`); + return structuredClone(this.files.get(path)); + } + + public async stat(path: string) { + for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor)) + assert( + !this.files.has(ancestor), + `Cannot traverse file ${ancestor}` + ); + const snapshot = this.files.get(path); + if (snapshot) + return { kind: "file" as const, size: snapshot.content.length }; + if (this.directories.has(path)) + return { kind: "directory" as const, size: 0 }; + return undefined; + } + + public async exists(path: string): Promise { + return this.files.has(path) || this.directories.has(path); + } + + public async createDirectory(path: string): Promise { + this.change(`mkdir ${path}`, () => { + for ( + let directory = path; + directory; + directory = parent(directory) + ) { + assert( + !this.files.has(directory), + `File blocks directory ${directory}` + ); + this.directories.add(directory); + } + }); + } + + public async write(path: string, snapshot: FileSnapshot): Promise { + this.change(`write ${path}`, () => { + assert( + !this.files.has(path) && !this.directories.has(path), + `Destination exists: ${path}` + ); + assert( + this.directories.has(parent(path)), + `Parent missing: ${path}` + ); + this.files.set(path, structuredClone(snapshot)); + }); + } + + public async rename(from: string, to: string): Promise { + this.change(`rename ${from} -> ${to}`, () => { + assert(this.files.has(from), `Source missing: ${from}`); + assert( + !this.files.has(to) && !this.directories.has(to), + `Destination exists: ${to}` + ); + assert(this.directories.has(parent(to)), `Parent missing: ${to}`); + this.files.set(to, this.files.get(from)!); + this.files.delete(from); + }); + } + + public async flushPaths(paths: readonly string[]): Promise { + this.change(`flush ${paths.join(",")}`, () => {}); + } + + public async delete(path: string): Promise { + this.change(`rmdir ${path}`, () => { + assert(!this.files.has(path)); + assert( + ![...this.files.keys()].some((file) => + file.startsWith(`${path}/`) + ) + ); + for (const directory of this.directories) + if (directory === path || directory.startsWith(`${path}/`)) + this.directories.delete(directory); + }); + } + + public async deleteFile(path: string): Promise { + this.change(`unlink ${path}`, () => { + assert(!this.directories.has(path)); + this.files.delete(path); + }); + } +} + +async function setup( + contents: Record, + local: Record +) { + const fs = new MemoryFiles(); + for (const [path, value] of Object.entries(contents)) fs.put(path, value); + let durable: StoredDatabase = emptyState("test"); + durable.local = local; + for (const [id, path] of Object.entries(local)) + durable.documents[id] = { + materialized: true, + observedHash: (await toStoredSnapshot(fs.files.get(path)!)).hash + }; + const config = { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig; + const restart = () => { + const database = new Database( + new Logger(), + structuredClone(durable), + async (next) => { + fs.change("save", () => { + durable = structuredClone(next); + }); + }, + "test", + async () => structuredClone(durable) + ); + return { + database, + operations: new FileOperations(fs, database, config) + }; + }; + return { fs, restart, ...restart() }; +} + +async function scenario() { + const fixture = await setup( + { + "a.md": "A", + "b.md": "B", + "c.md": "C", + "d.md": "Unmanaged", + "e.md": "E", + "folder/keep.md": "Keep", + "deleted.md": "Deleted" + }, + { a: "a.md", b: "b.md", c: "c.md", e: "e.md", deleted: "deleted.md" } + ); + const next = structuredClone(fixture.database.state); + next.local = { + a: "b.md", + b: "a.md", + c: "d.md", + e: "folder", + added: "new.md" + }; + next.documents.added = { materialized: false }; + const writes: Record = { + a: { + expected: await fixture.operations.snapshot("a.md"), + replacement: await toStoredSnapshot({ content: bytes("A updated") }) + }, + added: { + replacement: await toStoredSnapshot({ + content: bytes("Downloaded") + }) + } + }; + return { ...fixture, next, writes }; +} + +async function verifyScenario(fs: MemoryFiles, database: Database) { + assert.equal(database.state.application, undefined); + for (const [id, expected] of Object.entries({ + a: "A updated", + b: "B", + c: "C", + e: "E", + added: "Downloaded" + })) { + const snapshot = fs.files.get(database.state.local[id])!; + assert.equal(text(snapshot), expected, id); + assert.equal(database.state.documents[id].materialized, true, id); + assert.equal( + database.state.documents[id].observedHash, + (await toStoredSnapshot(snapshot)).hash, + id + ); + } + assert.equal(database.state.local.a, "b.md"); + assert.equal(database.state.local.b, "a.md"); + assert.equal(database.state.local.c, "d.md"); + assert.notEqual(database.state.local.e, "folder"); + assert.equal(text(fs.files.get("folder/keep.md")!), "Keep"); + assert.equal(fs.files.has("deleted.md"), false); + assert.equal(database.state.local.deleted, undefined); + const visible = [...fs.files].filter(([path]) => !isInternalPath(path)); + assert.deepEqual( + visible.map(([, value]) => text(value)).sort(), + ["A updated", "B", "C", "Downloaded", "E", "Keep", "Unmanaged"].sort() + ); + assert.equal( + [...fs.files.keys()].some((path) => + path.startsWith(".vault-link-sync/transactions/") + ), + false + ); +} + +describe("filesystem journal recovery", () => { + it("stages swaps, preserves unexpected files/directories, and cleans recovery artifacts", async () => { + const { operations, database, fs, next, writes } = await scenario(); + await operations.apply(next, writes); + await verifyScenario(fs, database); + fs.boundary = () => { + throw new Error("Completed recovery must not write"); + }; + await operations.recover(); + }); + + it("resumes after interruption at every filesystem and metadata mutation boundary", async (t) => { + const baseline = await scenario(); + const boundaries: string[] = []; + baseline.fs.boundary = (label) => boundaries.push(label); + await baseline.operations.apply(baseline.next, baseline.writes); + for (const [index, label] of boundaries.entries()) { + await t.test(`${index}: ${label}`, async () => { + const fixture = await scenario(); + const interrupted = new Error("Interrupted"); + let count = 0; + fixture.fs.boundary = () => { + if (count++ === index) throw interrupted; + }; + await assert.rejects( + fixture.operations.apply(fixture.next, fixture.writes), + (error) => error === interrupted + ); + fixture.fs.boundary = () => {}; + const { database, operations } = fixture.restart(); + if ( + !database.state.application && + database.state.local.a === "a.md" + ) + await operations.apply( + structuredClone(fixture.next), + fixture.writes + ); + else await operations.recover(); + await verifyScenario(fixture.fs, database); + }); + } + }); + + it("merges an edit made after planning exactly once across an interrupted install", async () => { + const base = "First paragraph.\n\nLast paragraph.\n"; + const local = "Local first paragraph.\n\nLast paragraph.\n"; + const remote = "First paragraph.\n\nRemote last paragraph.\n"; + const fixture = await setup({ "note.md": base }, { note: "note.md" }); + const expected = await fixture.operations.snapshot("note.md"); + let edited = false; + const interrupted = new Error("Interrupted after install"); + fixture.fs.boundary = (label) => { + if (!edited && label === "after save") { + fixture.fs.put("note.md", local); + edited = true; + } + if ( + label.startsWith("after rename ") && + label.endsWith(" -> note.md") + ) + throw interrupted; + }; + await assert.rejects( + fixture.operations.apply(structuredClone(fixture.database.state), { + note: { + expected, + replacement: await toStoredSnapshot({ + content: bytes(remote) + }) + } + }), + (error) => error === interrupted + ); + fixture.fs.boundary = () => {}; + const { operations, database } = fixture.restart(); + await operations.recover(); + assert.equal( + text(fixture.fs.files.get("note.md")!), + "Local first paragraph.\n\nRemote last paragraph.\n" + ); + assert.equal( + database.state.documents.note.observedHash, + (await operations.snapshot("note.md"))!.hash + ); + assert.equal( + [...fixture.fs.files.keys()].some((path) => + path.startsWith(".vault-link-sync/transactions/") + ), + false + ); + }); + + it("preserves an external deletion between planning and staging", async () => { + const fixture = await setup( + { "note.md": "Local" }, + { note: "note.md" } + ); + const expected = await fixture.operations.snapshot("note.md"); + fixture.fs.boundary = (label) => { + if (label === "after save") fixture.fs.files.delete("note.md"); + }; + await fixture.operations.apply( + structuredClone(fixture.database.state), + { + note: { + expected, + replacement: await toStoredSnapshot({ + content: bytes("Remote") + }) + } + } + ); + assert.equal(fixture.database.state.local.note, undefined); + assert.equal(fixture.fs.files.has("note.md"), false); + }); + + it("preserves a file created after the destination check", async () => { + const fixture = await setup( + { "source.md": "Incoming" }, + { doc: "source.md" } + ); + const next = structuredClone(fixture.database.state); + next.local.doc = "target.md"; + fixture.fs.boundary = (label) => { + if ( + label.startsWith("before rename ") && + label.endsWith(" -> target.md") + ) + fixture.fs.put("target.md", "Concurrent"); + }; + await assert.rejects( + fixture.operations.apply(next), + /Destination exists/ + ); + fixture.fs.boundary = () => {}; + const { operations, database } = fixture.restart(); + await operations.recover(); + assert.equal(text(fixture.fs.files.get("target.md")!), "Incoming"); + const displaced = Object.entries(database.state.local).find( + ([id]) => id !== "doc" + )!; + assert.equal(text(fixture.fs.files.get(displaced[1])!), "Concurrent"); + }); + + it("preserves an unexpected file blocking a destination's ancestor", async () => { + const fixture = await setup( + { "source.md": "Incoming", target: "Occupant" }, + { doc: "source.md" } + ); + const next = structuredClone(fixture.database.state); + next.local.doc = "target/child.md"; + await fixture.operations.apply(next); + assert.equal( + text(fixture.fs.files.get("target/child.md")!), + "Incoming" + ); + const displaced = Object.entries(fixture.database.state.local).find( + ([id]) => id !== "doc" + )!; + assert.equal(text(fixture.fs.files.get(displaced[1])!), "Occupant"); + }); + + it("handles paths becoming directories and directories becoming files", async () => { + const fixture = await setup( + { a: "A", "b/deep/child": "B" }, + { a: "a", b: "b/deep/child" } + ); + const next = structuredClone(fixture.database.state); + next.local = { a: "b", b: "a/deep/child" }; + await fixture.operations.apply(next); + assert.equal(text(fixture.fs.files.get("b")!), "A"); + assert.equal(text(fixture.fs.files.get("a/deep/child")!), "B"); + }); +}); diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 2864bd20..81d1a0f6 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,291 +1,916 @@ -import type { Logger } from "../tracing/logger"; -import type { FileSystemOperations } from "./filesystem-operations"; -import type { Database, RelativePath } from "../persistence/database"; -import { SafeFileSystemOperations } from "./safe-filesystem-operations"; -import type { TextWithCursors } from "reconcile-text"; -import { reconcile } from "reconcile-text"; -import { isFileTypeMergable } from "../utils/is-file-type-mergable"; -import { isBinary } from "../utils/is-binary"; +import { v4 as uuid } from "uuid"; +import type { + FileSnapshot, + FileSystemOperations +} from "./filesystem-operations"; +import type { + ApplicationJournal, + Database, + EngineState, + FileStep, + StoredSnapshot +} from "../persistence/database"; +import { + allocatePortablePath, + INTERNAL_DIRECTORY, + isInternalPath +} from "../utils/portable-path"; +import { + mergeContent, + fromStoredSnapshot, + toStoredSnapshot +} from "../sync-operations/content"; import type { ServerConfig } from "../services/server-config"; +import { FileNotFoundError } from "../errors/errors"; +import { + applyLocalChange, + unappliedChanges, + type LocalChange +} from "../sync-operations/local-changes"; +import { resolvePaths } from "../sync-operations/file-manifest"; +export interface FileWrite { + expected?: StoredSnapshot; + replacement: StoredSnapshot; +} + +/** The persisted journal bridges the filesystem and the metadata store. Every + * source is staged before any destination is installed. A phase is saved BEFORE + * an ambiguous rename, so recovery never reapplies a content merge to its output. + */ export class FileOperations { - private static readonly PARENTHESES_REGEX = / \((?\d+)\)$/; - private readonly fs: SafeFileSystemOperations; - public constructor( - private readonly logger: Logger, + public readonly fs: FileSystemOperations, private readonly database: Database, - fs: FileSystemOperations, private readonly serverConfig: ServerConfig, - private readonly nativeLineEndings = "\n" - ) { - this.fs = new SafeFileSystemOperations(fs, logger); + private readonly filesApplied: ( + paths: readonly string[] + ) => void = () => {}, + private readonly localChanges: { + entries: () => readonly LocalChange[]; + flush: () => Promise; + } = { entries: () => [], flush: async () => {} }, + private readonly protectedFile: ( + path: string, + size: number + ) => boolean = () => false + ) {} + + public async read(path: string): Promise { + const snapshot = await this.fs.readSnapshot(path); + if (!snapshot) + throw new FileNotFoundError(`File not found: ${path}`, path); + return snapshot.content; } - private static getParentDirAndFile( - path: RelativePath - ): [RelativePath, RelativePath] { - const pathParts = path.split("/"); - const fileName = pathParts.pop(); - if (fileName == null || fileName === "") { - throw new Error(`Path '${path}' cannot be empty`); - } - - return [pathParts.join("/"), fileName]; + public async listFilesRecursively(): Promise { + return (await this.fs.listFilesRecursively()).filter( + (path) => !isInternalPath(path) + ); } - public async listFilesRecursively( - root: RelativePath | undefined = undefined - ): Promise { - return this.fs.listFilesRecursively(root); + public async snapshot(path: string): Promise { + const value = await this.fs.readSnapshot(path); + return value && toStoredSnapshot(value); } - public async read(path: RelativePath): Promise { - return this.fromNativeLineEndings(await this.fs.read(path)); - } - - /** - * Create a file at the specified path. - * - * If a file with the same name already exists, it is moved before creating the new one. - * Parent directories are created if necessary. - */ - public async create( - path: RelativePath, - newContent: Uint8Array - ): Promise { - await this.ensureClearPath(path); - return this.fs.write(path, this.toNativeLineEndings(newContent)); - } - - public async ensureClearPath(path: RelativePath): Promise { - if (await this.fs.exists(path)) { - const deconflictedPath = await this.deconflictPath(path); - try { - this.logger.debug( - `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` - ); - - this.database.move(path, deconflictedPath); - await this.fs.rename(path, deconflictedPath, true); - } finally { - this.fs.unlock(deconflictedPath); + /** Project journal phases onto the physical namespace before binding an + * editor notification. Content hashes must never choose a notified identity. */ + private async physicalState( + journal = this.database.state.application + ): Promise { + const state = structuredClone(journal?.next ?? this.database.state); + if (journal) { + for (const step of journal.steps) { + if (step.superseded) continue; + const origin = step.replacement ? step.output : step.staged; + const path = + step.phase === "installed" + ? step.to + : step.phase === "installing" && + !(await this.fs.exists(origin)) + ? step.to + : (await this.fs.exists(step.staged)) + ? step.staged + : step.from; + if (path) state.local[step.documentId] = path; + else delete state.local[step.documentId]; } - } else { - await this.createParentDirectories(path); } + return state; } - /** - * Update the file at the given path. - * - * Performs a 3-way merge before writing if the file's content differs from `expectedContent`. - * Does not recreate the file if it no longer exists, returning an empty array instead. - */ - public async write( - path: RelativePath, - expectedContent: Uint8Array, - newContent: Uint8Array + public async captureChange( + change: LocalChange, + previous: readonly LocalChange[] ): Promise { - if (!(await this.fs.exists(path))) { - this.logger.debug( - `The caller assumed ${path} exists, but it no longer, so we wont recreate it` - ); - return; - } - + const state = await this.physicalState(); + // No notifications may bind provisional identities before the initial + // snapshot is available. Replay the whole ordered queue at bootstrap, + // including moves/deletions after creates. + if (!state.initialized) return; + for (const pending of unappliedChanges(state, previous)) + applyLocalChange(state, pending, () => false); if ( - !isFileTypeMergable( - path, - (await this.serverConfig.getConfig()).mergeableFileExtensions - ) || - isBinary(expectedContent) || - isBinary(newContent) + change.type === "update" && + !Object.values(state.local).includes(change.path) ) { - this.logger.debug( - `The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it` + // An editor may save to the logical path while its original is in + // staging. Namespace creates/deletes still bind to physical paths. + const step = this.database.state.application?.steps.find( + (step) => + step.from === change.path && step.phase !== "installed" ); - await this.fs.write( - path, - // `newContent` might not be binary so we still have to ensure the line endings are correct - this.toNativeLineEndings(newContent) + if (step && isInternalPath(state.local[step.documentId] ?? "")) + state.local[step.documentId] = change.path; + } + applyLocalChange(state, change, () => false); + } + + private async incorporateLocalChanges( + journal: ApplicationJournal + ): Promise { + await this.localChanges.flush(); + const changes = unappliedChanges( + journal.next, + this.localChanges.entries() + ); + if (!changes.length) return; + const physical = await this.physicalState(journal); + for (const change of changes) { + if (change.type === "update") { + const info = await this.fs.stat(change.path); + for (const id of Object.keys(change.identities ?? {})) { + // A save immediately before the rename has already moved + // to staging by the time its notification is incorporated. + const stagedPath = physical.local[id]; + const saved = + info ?? + (stagedPath && isInternalPath(stagedPath) + ? await this.fs.stat(stagedPath) + : undefined); + if ( + saved?.kind === "file" && + this.protectPath( + journal, + physical, + id, + change.path, + saved.size + ) + ) + journal.next.local[id] = change.path; + } + await this.incorporateContentSave(journal, physical, change); + journal.next.excluded = physical.excluded; + journal.next.lastAppliedLocalChangeId = change.changeId; + continue; + } + const before = { ...physical.local }; + const path = + change.type === "move" ? change.relativePath : change.path; + const info = await this.fs.stat(path); + if (info?.kind === "file" && this.protectedFile(path, info.size)) + journal.next.protectedPaths = [ + ...new Set([...(journal.next.protectedPaths ?? []), path]) + ]; + applyLocalChange( + physical, + change, + (path) => + isInternalPath(path) || + this.protectedFile(path, 0) || + journal.next.protectedPaths?.includes(path) === true || + Object.keys(journal.next.excluded ?? {}).some( + (id) => before[id] === path + ) ); - return; - } - - const expectedText = new TextDecoder().decode(expectedContent); // this comes from a previous read which must only have \n line endings - const newText = new TextDecoder().decode(newContent); // this comes from the server which stores text with \n line endings - - await this.fs.atomicUpdateText( - path, - ({ text, cursors }: TextWithCursors): TextWithCursors => { - this.logger.debug( - `Performing a 3-way merge for ${path} with the expected content` - ); - - text = text.replaceAll(this.nativeLineEndings, "\n"); - const merged = reconcile( - expectedText, - { text, cursors }, - newText - ); - - const resultText = merged.text.replaceAll( - "\n", - this.nativeLineEndings - ); - - return { - text: resultText, - cursors: merged.cursors - }; + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(physical.local) + ])) { + if (before[id] === physical.local[id]) continue; + if (physical.local[id] === undefined) + delete journal.next.local[id]; + else { + const path = physical.local[id]; + journal.next.local[id] = path; + const info = await this.fs.stat(path); + this.protectPath( + journal, + physical, + id, + path, + info?.size ?? 0 + ); + } + journal.next.documents[id] ??= physical.documents[id]; } + journal.next.excluded = physical.excluded; + journal.next.lastAppliedLocalChangeId = change.changeId; + } + journal.next.local = resolvePaths( + journal.next.local, + journal.next.fileManifest.entries, + Object.keys(journal.next.excluded ?? {}), + journal.next.protectedPaths ); - } - - public async delete(path: RelativePath): Promise { - if (await this.exists(path)) { - await this.fs.delete(path); - await this.deletingEmptyParentDirectoriesOfDeletedFile(path); - } else { - this.logger.debug(`No need to delete '${path}', it doesn't exist`); - } - } - - public async getFileSize(path: RelativePath): Promise { - return this.fs.getFileSize(path); - } - - public async exists(path: RelativePath): Promise { - return this.fs.exists(path); - } - - public async move( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - if (oldPath === newPath) { - return; - } - - await this.ensureClearPath(newPath); - - this.database.move(oldPath, newPath); - await this.fs.rename(oldPath, newPath); - await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath); - } - - public reset(): void { - this.fs.reset(); - } - - private async deletingEmptyParentDirectoriesOfDeletedFile( - path: RelativePath - ): Promise { - let directory = path; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - [directory] = FileOperations.getParentDirAndFile(directory); - if (directory.length === 0) { - break; - } - - const remainingContent = - await this.fs.listFilesRecursively(directory); - if (remainingContent.length === 0) { - this.logger.debug( - `Folder (${directory}) is now empty, deleting` - ); - await this.fs.delete(directory); - } else { - break; - } - } - } - - private fromNativeLineEndings(content: Uint8Array): Uint8Array { - if (isBinary(content)) { - return content; - } - - const decoder = new TextDecoder("utf-8"); - let text = decoder.decode(content); - text = text.replaceAll(this.nativeLineEndings, "\n"); - return new TextEncoder().encode(text); - } - - private toNativeLineEndings(content: Uint8Array): Uint8Array { - if (isBinary(content)) { - return content; - } - - const decoder = new TextDecoder("utf-8"); - let text = decoder.decode(content); - text = text.replaceAll("\n", this.nativeLineEndings); - return new TextEncoder().encode(text); - } - - private async createParentDirectories(path: string): Promise { - const components = path.split("/"); - if (components.length === 1) { - return; - } - for (let i = 1; i < components.length; i++) { - const parentDir = components.slice(0, i).join("/"); - if (!(await this.fs.exists(parentDir))) { - await this.fs.createDirectory(parentDir); - } - } - } - - /** - * Deconflicts the given path by appending (1), (2), etc. before the file extension until a non-existent path is found. - * The returned path has a lock acquired on it; it must be released by the caller when no longer needed. - * - * @param path The starting path to deconflict - * @returns a non-existent path with a lock acquired on it - */ - private async deconflictPath(path: RelativePath): Promise { - // eslint-disable-next-line prefer-const - let [directory, fileName] = FileOperations.getParentDirAndFile(path); - - if (directory) { - directory += "/"; - } - - const nameParts = fileName.split("."); - // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should have ".json" - const isDotfile = fileName.startsWith(".") && nameParts[0] === ""; - const extension = - nameParts.length > 1 && !(isDotfile && nameParts.length === 2) - ? "." + nameParts[nameParts.length - 1] - : ""; - let stem = extension ? nameParts.slice(0, -1).join(".") : fileName; - let currentCount = Number.parseInt( - FileOperations.PARENTHESES_REGEX.exec(stem)?.groups?.count ?? "0" - ); - stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); - - let newName = path; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - currentCount++; - newName = `${directory}${stem} (${currentCount})${extension}`; - - // Avoid multiple deconflictPath calls returning the same path - if (this.fs.tryLock(newName)) { - const newDocument = - this.database.getLatestDocumentByRelativePath(newName); + for (const step of journal.steps) { + if (step.superseded) continue; + // A notified move can exclude an input after this transaction was + // planned. Preserve its bytes as well as its chosen physical path. + if (journal.next.excluded?.[step.documentId] !== undefined) { + const origin = step.replacement ? step.output : step.staged; if ( - newDocument?.isDeleted === false || // the document might have been confirmed by the server at a new path but haven't yet moved there locally - (await this.fs.exists(newName, true)) + step.phase === "installing" && + !(await this.fs.exists(origin)) + ) + step.phase = "installed"; + if (step.replacement && step.phase !== "installed") { + // contentWrite advanced the proposed merge context before + // applying bytes. Cancelling that application must retain + // the last incorporated base so reenabling the file still + // reconciles the deferred remote head. An installed output, + // including one whose acknowledgement was lost, keeps its + // new base for subsequent local edits. + const previous = + this.database.state.documents[step.documentId]; + Object.assign( + journal.next.documents[step.documentId], + structuredClone({ + base: previous?.base, + bootstrap: previous?.bootstrap, + recoveryBase: previous?.recoveryBase + }) + ); + } + step.replacement = undefined; + } + if (step.phase === "installed") { + step.to = physical.local[step.documentId]; + continue; + } + if (step.phase === "planned") { + const staged = await this.fs.exists(step.staged); + const from = physical.local[step.documentId]; + if (staged && from !== step.staged) { + // A notified move/delete and path reuse raced the source + // rename. Bind the staged object to the new occupant, then + // stage the original identity from its notified location. + const occupant = Object.keys(physical.local).find( + (id) => physical.local[id] === step.from + ); + const originalStaged = step.staged; + step.staged = `${step.staged}-${uuid()}`; + if (occupant) { + journal.steps.push({ + documentId: occupant, + from: step.from, + to: journal.next.local[occupant], + staged: originalStaged, + output: `${originalStaged}.output`, + phase: "staged" + }); + physical.local[occupant] = originalStaged; + } else { + // Retain an ambiguous object without merging it into a + // different document. Cleanup archives this source. + journal.steps.push({ + ...step, + staged: originalStaged, + to: undefined, + replacement: undefined, + phase: "installed", + superseded: true + }); + } + } + if ( + !staged || + (from !== undefined && !isInternalPath(from)) || + from === undefined ) { - this.fs.unlock(newName); - } else { - return newName; + if (step.from && !from) step.replacement = undefined; + step.from = from; } } + step.to = journal.next.local[step.documentId]; + } + for (const [id, from] of Object.entries(physical.local)) { + const to = journal.next.local[id]; + if ( + from === to || + isInternalPath(from) || + journal.steps.some( + (step) => + step.documentId === id && step.phase !== "installed" + ) + ) + continue; + const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${id}-${uuid()}`; + journal.steps.push({ + documentId: id, + from, + to, + expected: await this.snapshot(from), + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + } + await this.persist(journal); + } + + private protectPath( + journal: ApplicationJournal, + physical: EngineState, + id: string, + path: string, + size: number + ): boolean { + if ( + !this.protectedFile(path, size) && + !journal.next.protectedPaths?.includes(path) + ) + return false; + physical.excluded ??= {}; + physical.excluded[id] ??= + this.database.state.fileManifest.entries[id] ?? null; + journal.next.protectedPaths = [ + ...new Set([...(journal.next.protectedPaths ?? []), path]) + ]; + return true; + } + + private async incorporateContentSave( + journal: ApplicationJournal, + physical: EngineState, + change: LocalChange & { type: "update" } + ): Promise { + for (const id of Object.keys(change.identities ?? {})) { + const step = journal.steps.find( + (step) => + step.documentId === id && + step.phase !== "installed" && + !step.superseded + ); + if ( + !step || + !(await this.fs.exists(step.staged)) || + (await this.fs.stat(change.path))?.kind !== "file" + ) + continue; + const moved = + step.phase === "installing" && + !(await this.fs.exists( + step.replacement ? step.output : step.staged + )); + if (moved && step.to === change.path) continue; + // Stage the editor's save as a continuation of the SAME identity. + // The previous source stays in the journal until durable cleanup. + const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${id}-${uuid()}`; + if (moved && step.to) { + // Installation won the race with a save to the old editor path. + // Stage its output before reinstalling this identity's merge. + journal.steps.push({ + documentId: id, + from: step.to, + staged: `${prefix}.previous`, + output: `${prefix}.unused`, + phase: "planned", + superseded: true + }); + } + journal.steps.push({ + documentId: id, + from: change.path, + to: step.to, + expected: step.expected, + replacement: step.replacement, + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + step.to = undefined; + step.replacement = undefined; + step.phase = "installed"; + step.superseded = true; + physical.local[id] = change.path; } } + + private directory(path: string): string { + return path.split("/").slice(0, -1).join("/"); + } + + private async writeOnce( + path: string, + snapshot: FileSnapshot + ): Promise { + const existing = await this.fs.readSnapshot(path); + if (existing) { + const actual = await toStoredSnapshot(existing); + const expected = await toStoredSnapshot(snapshot); + if (actual.contentBase64 !== expected.contentBase64) + throw new Error(`Recovery artifact changed: ${path}`); + await this.fs.flushPaths([path]); + return; + } + await this.ensureParents(path); + await this.fs.write(path, snapshot); + } + + private async writeJsonOnce(path: string, value: unknown): Promise { + await this.writeOnce(path, { + content: new TextEncoder().encode(JSON.stringify(value)) + }); + } + + private async persist(journal: ApplicationJournal): Promise { + await this.database.commit({ + ...this.database.state, + application: journal + }); + } + + public async retainSnapshot( + requestId: string, + documentId: string, + path: string | undefined, + snapshot: StoredSnapshot + ): Promise { + const prefix = `${INTERNAL_DIRECTORY}/requests/${requestId}`; + await this.writeOnce(`${prefix}.content`, fromStoredSnapshot(snapshot)); + const metadata = await this.fs.readSnapshot(`${prefix}.json`); + if (metadata) { + const saved: unknown = JSON.parse( + new TextDecoder().decode(metadata.content) + ); + if ( + typeof saved !== "object" || + saved === null || + !("documentId" in saved) || + saved.documentId !== documentId || + !("hash" in saved) || + saved.hash !== snapshot.hash + ) + throw new Error(`Recovery artifact changed: ${prefix}.json`); + await this.fs.flushPaths([`${prefix}.json`]); + return; // originalPath is a hint recorded once, not request identity. + } + await this.writeJsonOnce(`${prefix}.json`, { + documentId, + originalPath: path, + hash: snapshot.hash + }); + } + + public async apply( + next: EngineState, + writes: Record = {}, + validatePlan: () => void = () => {} + ): Promise { + if (this.database.state.application) + throw new Error( + "Recover the existing filesystem transaction first" + ); + + const id = uuid(); + const steps: FileStep[] = []; + for (const documentId of new Set([ + ...Object.keys(this.database.state.local), + ...Object.keys(next.local) + ])) { + const oldPath = this.database.state.local[documentId]; + const newPath = next.local[documentId]; + const materialized = + this.database.state.documents[documentId]?.materialized; + const from = materialized ? oldPath : undefined; + const write = writes[documentId]; + if (!write && (from === undefined || from === newPath)) continue; + const expected = + write?.expected ?? + (from ? await this.snapshot(from) : undefined); + // An external deletion is a local change, not permission to resurrect + // the file using a stale network response. + if (from && !expected && !write) { + delete next.local[documentId]; + continue; + } + const prefix = `${INTERNAL_DIRECTORY}/transactions/${id}/${documentId}`; + steps.push({ + documentId, + from, + to: newPath, + expected, + replacement: write?.replacement, + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + } + validatePlan(); + if (steps.length === 0) { + const changedPaths = this.changedPaths( + this.database.state.local, + next.local + ); + await this.database.commit(next); + this.filesApplied(changedPaths); + return; + } + + const journal = { + id, + next: structuredClone(next), + steps, + extensions: (await this.serverConfig.getConfig()) + .mergeableFileExtensions + }; + validatePlan(); + await this.persist(journal); + await this.recover(); + } + + public async recover(): Promise { + const saved = this.database.state.application; + if (!saved) return; + + const journal = structuredClone(saved); + await this.fs.createDirectory( + `${INTERNAL_DIRECTORY}/transactions/${journal.id}` + ); + + for (;;) { + await this.incorporateLocalChanges(journal); + // Stage every pending source before installing anything, including + // occupants discovered during an earlier installation attempt. + await this.stageSources(journal); + const step = journal.steps.find( + (step) => step.phase !== "installed" + ); + if (!step) break; + await this.prepareContent(journal, step); + await this.incorporateLocalChanges(journal); + if (step.superseded && step.phase === "installed") continue; + try { + await this.install(journal, step); + } catch (error) { + const code = + typeof error === "object" && + error !== null && + "code" in error && + typeof error.code === "string" + ? error.code + : undefined; + if ( + step.to === undefined || + ![ + "ENAMETOOLONG", + "EINVAL", + "ENOTSUP", + "EOPNOTSUPP", + "EACCES", + "EPERM" + ].includes(code ?? "") + ) + throw error; + const origin = step.replacement ? step.output : step.staged; + // A rename that took effect must be acknowledged at its original + // destination. Never infer failure from a failed durability flush. + if ( + step.phase === "installing" && + !(await this.fs.exists(origin)) + ) + throw error; + if (step.replanned === true) throw error; + const extension = + /\.[^.]*$/u.exec(step.to.split("/").at(-1) ?? "")?.[0] ?? + ""; + const wanted = `Recovered ${step.documentId}${extension}`; + step.to = await this.conflictPath( + journal, + step.documentId, + wanted + ); + step.replanned = true; + journal.next.local[step.documentId] = step.to; + // The payload has already been prepared; persist the new target + // before retrying so crashes cannot reapply its content merge. + step.phase = "prepared"; + await this.persist(journal); + } + } + + // Once every install intent is durable, the journal can be replayed + // without its staged inputs. Remove transient bytes before committing + // the next engine state so an interrupted cleanup is retried. + await this.cleanup(journal); + await this.database.commit(journal.next); + this.filesApplied( + [ + ...new Set( + journal.steps.flatMap((step) => [step.from, step.to]) + ) + ].filter((path): path is string => path !== undefined) + ); + } + + private async stageSources(journal: ApplicationJournal): Promise { + for (;;) { + await this.incorporateLocalChanges(journal); + const step = journal.steps.find((step) => step.phase === "planned"); + if (!step) break; + // An existing staging file proves the move completed before a crash. + if (step.from && !(await this.fs.exists(step.staged))) { + const { from } = step, + { staged } = step; + const present = await this.fs.exists(from); + await this.incorporateLocalChanges(journal); + if ( + step.from !== from || + step.staged !== staged || + step.phase !== "planned" + ) + continue; + if (present) { + await this.fs.rename(from, staged); + await this.incorporateLocalChanges(journal); + if ( + step.from !== from || + step.staged !== staged || + step.phase !== "planned" + ) + continue; + } else { + // Preserve an external deletion instead of resurrecting it. + delete journal.next.local[step.documentId]; + step.to = undefined; + step.replacement = undefined; + } + } + if (step.from) await this.fs.flushPaths([step.from, step.staged]); + step.phase = "staged"; + await this.persist(journal); + } + // Directory cleanup is also retried if staging completed before a crash. + for (const step of journal.steps) + await this.removeEmptyParents(step.from); + } + + private async prepareContent( + journal: ApplicationJournal, + step: FileStep + ): Promise { + if (step.phase !== "staged") return; + const source = await this.snapshot(step.staged); + if (source) { + if (step.replacement && step.expected) + step.replacement = await mergeContent( + step.to ?? step.from ?? "", + step.expected, + source, + step.replacement, + journal.extensions + ); + // The prepared replacement now includes this source's edits. A + // later save must be merged relative to this source, or superseded + // local text is reintroduced from the already merged replacement. + step.expected = source; + } + // Save the merged result before writing it, so recovery never merges twice. + step.phase = "prepared"; + await this.persist(journal); + } + + private async install( + journal: ApplicationJournal, + step: FileStep + ): Promise { + if (step.to) { + const origin = step.replacement ? step.output : step.staged; + const alreadyMoved = + step.phase === "installing" && !(await this.fs.exists(origin)); + if (alreadyMoved) { + if (!(await this.fs.exists(step.to))) { + // The installed file was externally deleted before + // recovery. Preserve that deletion instead of failing or + // resurrecting bytes from metadata. + delete journal.next.local[step.documentId]; + step.to = undefined; + step.replacement = undefined; + step.phase = "installed"; + await this.persist(journal); + return; + } + await this.fs.flushPaths([origin, step.to]); + } else { + if (step.replacement) + await this.writeOnce( + step.output, + fromStoredSnapshot(step.replacement) + ); + await this.incorporateLocalChanges(journal); + if (step.superseded) return; + if (!(await this.clearDestination(journal, step, step.to))) + return; + await this.ensureParents(step.to); + await this.incorporateLocalChanges(journal); + if (step.superseded) return; + // Persist intent first; the no-replace rename protects against + // another writer taking the destination after we checked it. + step.phase = "installing"; + await this.persist(journal); + await this.fs.rename(origin, step.to); + await this.incorporateLocalChanges(journal); + if (step.superseded) return; + } + const document = journal.next.documents[step.documentId]; + if (document) { + document.materialized = true; + document.observedHash = + step.replacement?.hash ?? + (await this.snapshot(step.to))?.hash; + } + } + // Deletions and completed renames share this path whether their + // acknowledgement was received or lost. Transient bytes are removed + // only after every step reaches this phase. + step.phase = "installed"; + await this.persist(journal); + } + + private async clearDestination( + journal: ApplicationJournal, + step: FileStep, + path: string + ): Promise { + const obstruction = await this.obstruction(path); + if (!obstruction) return true; + if (obstruction.kind === "file") { + const info = await this.fs.stat(obstruction.path); + if ( + journal.next.protectedPaths?.includes(obstruction.path) === + true || + (info !== undefined && + this.protectedFile(obstruction.path, info.size)) + ) { + (journal.next.protectedPaths ??= []).push(obstruction.path); + step.to = await this.conflictPath( + journal, + step.documentId, + path + ); + journal.next.local[step.documentId] = step.to; + await this.persist(journal); + } else await this.displace(journal, obstruction.path); + } else if ( + (await this.fs.listFilesRecursively(obstruction.path)).length + ) { + // Preserve unmanaged descendants by changing the incoming file's name. + step.to = await this.conflictPath(journal, step.documentId, path); + journal.next.local[step.documentId] = step.to; + await this.persist(journal); + } else { + // The adapter only removes empty directories, including on a race. + await this.fs.delete(obstruction.path); + return true; + } + // The journal now contains a new source or destination. Retry it through + // the main loop so newly discovered sources are staged before installing. + return false; + } + + private async obstruction( + path: string + ): Promise<{ path: string; kind: "file" | "directory" } | undefined> { + const parts = path.split("/"); + for (let i = 0; i < parts.length; i++) { + const prefix = parts.slice(0, i + 1).join("/"); + const info = await this.fs.stat(prefix); + if (info && (info.kind === "file" || i === parts.length - 1)) + return { path: prefix, kind: info.kind }; + } + return undefined; + } + + private async conflictPath( + journal: ApplicationJournal, + documentId: string, + path: string + ): Promise { + const occupied = { ...journal.next.local }; + delete occupied[documentId]; + for (const [index, diskPath] of ( + await this.listFilesRecursively() + ).entries()) + occupied[`disk-${index}`] = diskPath; + return allocatePortablePath(path, documentId, occupied); + } + + private async displace( + journal: ApplicationJournal, + path: string + ): Promise { + const unexpected = await this.snapshot(path); + if (!unexpected) + throw new Error(`Destination changed during recovery: ${path}`); + const documentId = uuid(); + const to = await this.conflictPath(journal, documentId, path); + const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${documentId}`; + journal.steps.push({ + documentId, + from: path, + to, + expected: unexpected, + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + journal.next.local[documentId] = to; + journal.next.documents[documentId] = { + materialized: true, + observedHash: unexpected.hash + }; + await this.persist(journal); + } + + private async ensureParents(path: string): Promise { + const parent = this.directory(path); + if (parent) await this.fs.createDirectory(parent); + } + + private async removeEmptyParents(path?: string): Promise { + if (!path) return; + let parent = this.directory(path); + while (parent && !isInternalPath(parent)) { + // An earlier installation may have turned an ancestor into a file. + const info = await this.obstruction(parent); + if (!info) { + parent = this.directory(parent); + continue; + } + if (info.kind !== "directory") break; + if ((await this.fs.listFilesRecursively(parent)).length) break; + // delete() must reject non-empty directories, including unmanaged entries. + try { + await this.fs.delete(parent); + } catch { + break; + } + parent = this.directory(parent); + } + } + + private async cleanup(journal: ApplicationJournal): Promise { + for (const step of journal.steps) { + const source = await this.snapshot(step.staged); + if ( + source && + (!step.to || + (step.replacement && source.hash !== step.replacement.hash)) + ) { + const prefix = `${INTERNAL_DIRECTORY}/recovery/${journal.id}-${step.staged.split("/").at(-1)}`; + await this.writeOnce( + `${prefix}.content`, + fromStoredSnapshot(source) + ); + await this.writeJsonOnce(`${prefix}.json`, { + documentId: step.documentId, + originalPath: step.from, + hash: source.hash + }); + } + for (const path of [ + step.staged, + step.output, + `${step.staged}.recovery`, + `${step.staged}.json` + ]) { + if (await this.fs.exists(path)) await this.fs.deleteFile(path); + } + } + const completed = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/completed.json`; + if (await this.fs.exists(completed)) + await this.fs.deleteFile(completed); + await this.fs.delete( + `${INTERNAL_DIRECTORY}/transactions/${journal.id}` + ); + } + + private changedPaths( + before: Readonly>, + after: Readonly> + ): string[] { + const result = new Set(); + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(after) + ])) { + if (before[id] === after[id]) continue; + if (before[id] !== undefined) result.add(before[id]); + if (after[id] !== undefined) result.add(after[id]); + } + return [...result]; + } } diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 36dddfe6..f4e79397 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -1,37 +1,45 @@ -import type { RelativePath } from "../persistence/database"; +import type { CursorPosition } from "reconcile-text"; -import type { TextWithCursors } from "reconcile-text"; - +export interface FileSnapshot { + content: Uint8Array; + /** Optional editor selections, moved with the file and applied with its bytes. */ + cursors?: CursorPosition[]; +} + +/** Required adapter contract for API v4; there is deliberately no weak fallback. + * Paths are vault-relative. Never follow symlinks (including ancestor symlinks). + * Only independent regular files are supported; reject hardlinks/special files. + * Every mutation must flush file data and affected directories before resolving. + * A successful scan is complete: unreadable directories must reject, not vanish. + */ export interface FileSystemOperations { - // List all files under root that should be synced. If root is undefined, return every file. - listFilesRecursively: ( - root: RelativePath | undefined - ) => Promise; - - // Read the content of a file. - read: (path: RelativePath) => Promise; - - // Create or overwrite a file with the given content. - write: (path: RelativePath, content: Uint8Array) => Promise; - - // Atomically update the content of a text file. - atomicUpdateText: ( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ) => Promise; - - // Get the size of a file in bytes. - getFileSize: (path: RelativePath) => Promise; - - // Check if a file exists. - exists: (path: RelativePath) => Promise; - - // Create a directory at the specified path. All parent directories must already exist. - createDirectory: (path: RelativePath) => Promise; - - // Delete a file. It is expected that the path points to an existing file. - delete: (path: RelativePath) => Promise; - - // Rename a file. It is expected that the oldPath points to an existing file and the newPath does not exist. - rename: (oldPath: RelativePath, newPath: RelativePath) => Promise; + listFilesRecursively: (root?: string) => Promise; + /** Coherent content snapshot; undefined only for an absent file. Reject read races. */ + readSnapshot: (path: string) => Promise; + /** Reject symlinks and special files rather than following them. */ + stat: ( + path: string + ) => Promise<{ kind: "file" | "directory"; size: number } | undefined>; + /** Includes directories. */ + exists: (path: string) => Promise; + /** Recursive, idempotent, durable directory creation. */ + createDirectory: (path: string) => Promise; + /** Atomically create a complete file: never replace an existing object. + * A crash leaves the path absent or complete, never a partial file. + * Flush data and directory entries before resolving. */ + write: (path: string, snapshot: FileSnapshot) => Promise; + /** Atomic, durable rename on the same filesystem; fail if destination exists. + * Flush the source's data and both affected directories before resolving. */ + rename: (from: string, to: string) => Promise; + /** Flush existing files and all affected ancestor directories, even when a + * named file is absent. Recovery uses this after a mutation became visible + * before its caller learned that it was durable. */ + flushPaths: (paths: readonly string[]) => Promise; + /** Idempotently unlink one regular file and flush its parent directory. + * The sync engine uses this only for its own internal recovery artifacts. */ + deleteFile: (path: string) => Promise; + /** Prune an implicit directory tree using rmdir only, durably. Reject + * regular files and never unlink descendant files; reject if files remain. + * Partial removal of empty child directories on failure is harmless. */ + delete: (path: string) => Promise; } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 86b2845c..750eb667 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,374 +1,235 @@ +import type { CursorPosition } from "reconcile-text"; import type { Logger } from "../tracing/logger"; -import { EMPTY_HASH } from "../utils/hash"; -import { CoveredValues } from "../utils/data-structures/min-covered"; -import { awaitAll } from "../utils/await-all"; -import { removeFromArray } from "../utils/remove-from-array"; +import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { FileManifestUpdateResponse } from "../services/types/FileManifestUpdateResponse"; +import type { PushFileManifest } from "../services/types/PushFileManifest"; +import type { PutFileContent } from "../services/types/PutFileContent"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import type { EventRecord } from "../services/types/EventRecord"; +import type { FileManifestEntries } from "../types/file-manifest-entries"; export type VaultUpdateId = number; export type DocumentId = string; export type RelativePath = string; +export interface StoredSnapshot { + contentBase64: string; + hash: string; + cursors?: CursorPosition[]; +} + export interface DocumentMetadata { - parentVersionId: VaultUpdateId; + parentVersionId: number; hash: string; - remoteRelativePath?: RelativePath; + remoteRelativePath?: string; } -export interface StoredDocumentMetadata { - relativePath: RelativePath; - documentId: DocumentId; - parentVersionId: VaultUpdateId; - remoteRelativePath?: RelativePath; - hash: string; +export interface DocumentState { + /** Local clean base retained while rebasing after a server restore. */ + recoveryBase?: StoredSnapshot; + base?: DocumentVersionWithoutContent & { hash: string }; + observedHash?: string; + materialized: boolean; + /** First contact with existing local content: there is no common ancestor. */ + bootstrap?: boolean; + /** Do not resubmit an unchanged payload rejected by the server. */ + rejected?: { hash: string; message: string }; } -export interface StoredDatabase { - documents: StoredDocumentMetadata[]; - lastSeenUpdateId: VaultUpdateId | undefined; - hasInitialSyncCompleted: boolean; +export type PendingRequest = { rejection?: string } & ( + | { + type: "content"; + documentId: string; + request: PutFileContent; + snapshot: StoredSnapshot; + response?: DocumentUpdateResponse; + } + | { + type: "fileManifest"; + request: PushFileManifest; + response?: FileManifestUpdateResponse; + } +); + +export interface EngineState { + /** Durable reset intent; clear the transport checkpoint before bootstrap. */ + historyRecovery?: boolean; + vaultKey: string; + initialized: boolean; + fileManifest: FileManifest; + local: FileManifestEntries; + documents: Record; + lastSeenUpdateId: number; + remoteHeads: Record; + bootstrap?: VaultSnapshot; + /** Fold pages durably without applying intermediate remote snapshots. */ + eventReplay?: { + after: number; + manifest: FileManifest; + heads: Record; + receipts: EventRecord[]; + }; + pending?: PendingRequest; + /** A rejected retry does not prove an earlier attempt failed to commit. */ + unconfirmed?: PendingRequest[]; + /** Commits consumption of the durable notification queue with the identity map. */ + lastAppliedLocalChangeId?: string; + /** Physical exclusions, including files with no sync identity. */ + protectedPaths?: string[]; + /** Original server paths while local files are excluded; null means absent. */ + excluded?: Record; + rejectedManifest?: { entries: FileManifestEntries; message: string }; } -/** - * Represents a document in the database. - * - * It is mutable and its content should always represent the latest - * state of the document on disk based on the update events we have seen. - */ +export interface FileStep { + documentId: string; + from?: string; + to?: string; + expected?: StoredSnapshot; + replacement?: StoredSnapshot; + staged: string; + output: string; + /** A permanent destination error has already selected a root fallback. */ + replanned?: boolean; + /** Retained input of a later step for this identity; never projected as live. */ + superseded?: boolean; + phase: "planned" | "staged" | "prepared" | "installing" | "installed"; +} + +export interface ApplicationJournal { + id: string; + extensions: string[]; + next: EngineState; + steps: FileStep[]; +} + +export interface StoredDatabase extends EngineState { + application?: ApplicationJournal; +} + +// Read-only view used by cursor tracking and status reporting. export interface DocumentRecord { - relativePath: RelativePath; - documentId: DocumentId; - metadata: DocumentMetadata | undefined; + documentId: string; + relativePath: string; + metadata?: DocumentMetadata; isDeleted: boolean; - updates: Promise[]; - parallelVersion: number; +} + +export function emptyState(vaultKey: string): StoredDatabase { + return { + vaultKey, + initialized: false, + fileManifest: { fileManifestId: 0, entries: {} }, + local: {}, + documents: {}, + remoteHeads: {}, + lastSeenUpdateId: 0 + }; +} + +function canRebindEmpty(state: Partial | undefined): boolean { + return ( + state !== undefined && + !state.initialized && + !state.pending && + !state.application && + !state.bootstrap && + !state.eventReplay && + (state.unconfirmed?.length ?? 0) === 0 && + Object.keys(state.local ?? {}).length === 0 + ); } export class Database { - private documents: DocumentRecord[]; - private lastSeenUpdateIds: CoveredValues; - private hasInitialSyncCompleted: boolean; - + public state: StoredDatabase; + private needsReload = false; public constructor( private readonly logger: Logger, - initialState: Partial | undefined, - private readonly saveData: (data: StoredDatabase) => Promise + initial: Partial | undefined, + private readonly saveData: (data: StoredDatabase) => Promise, + vaultKey: string, + private readonly loadData: () => Promise ) { - initialState ??= {}; - - this.documents = - initialState.documents?.map( - ({ relativePath, documentId, ...metadata }) => ({ - relativePath, - documentId, - metadata, - isDeleted: false, - updates: [], - parallelVersion: 0 - }) - ) ?? []; - - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.length} documents`); - - const { lastSeenUpdateId } = initialState; - this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`); - this.lastSeenUpdateIds = new CoveredValues( - Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1 - ); - - this.documents.forEach((doc) => { - this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId); - }); - - this.hasInitialSyncCompleted = - initialState.hasInitialSyncCompleted ?? false; - this.logger.debug( - `Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}` - ); - } - - public get length(): number { - return this.documents.length; - } - - public get resolvedDocuments(): DocumentRecord[] { - const paths = new Map(); - this.documents - // eslint-disable-next-line no-restricted-syntax -- Type narrowing, not removing a specific item - .filter(({ metadata }) => metadata !== undefined) - .forEach((record) => - paths.set(record.relativePath, [ - record, - ...(paths.get(record.relativePath) ?? []) - ]) - ); - - return Array.from(paths.values()).map((records) => { - records.sort( - (a, b) => b.parallelVersion - a.parallelVersion // descending - ); - - if ( - records.length > 1 && - records.some((current, i) => - i === 0 - ? false - : records[i - 1].parallelVersion === - current.parallelVersion - ) - ) { - throw new Error( - `Multiple documents with the same parallel version and path at ${records[0].relativePath}` - ); - } - return records[0]; - }); - } - - public updateDocumentMetadata( - metadata: { - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath: RelativePath; - }, - toUpdate: DocumentRecord - ): void { - if (!this.documents.includes(toUpdate)) { - throw new Error("Document not found in database"); - } - - toUpdate.metadata = metadata; - - this.saveInTheBackground(); - } - - public removeDocumentPromise(promise: Promise): void { - const entry = this.documents.find(({ updates }) => - updates.includes(promise) - ); - - if (entry === undefined) { - // This method should be idempotent and tolerant of - // stragglers calling it after the databse has been reset. - return; - } - - removeFromArray(entry.updates, promise); - // No need to save as Promises don't get serialized - } - - public removeDocument(find: DocumentRecord): void { - removeFromArray(this.documents, find); - this.saveInTheBackground(); - } - - public getLatestDocumentByRelativePath( - find: RelativePath - ): DocumentRecord | undefined { - const candidates = this.documents.filter( - ({ relativePath }) => relativePath === find - ); - candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending - return candidates[0]; - } - - public async getResolvedDocumentByRelativePath( - relativePath: RelativePath, - promise: Promise - ): Promise { - const entry = this.getLatestDocumentByRelativePath(relativePath); - - if (entry === undefined) { + const hasState = initial && Object.keys(initial).length > 0; + if ( + hasState && + initial.vaultKey !== vaultKey && + !canRebindEmpty(initial) + ) { throw new Error( - `Document not found by relative path: ${relativePath}, ${JSON.stringify( - this.documents, - null, - 2 - )}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); } - - const currentPromises = entry.updates; - entry.updates = [...currentPromises, promise]; - await awaitAll(currentPromises); - - return entry; + this.state = hasState + ? { ...(initial as StoredDatabase), vaultKey } + : emptyState(vaultKey); } - public createNewPendingDocument( - documentId: DocumentId, - relativePath: RelativePath, - promise: Promise - ): DocumentRecord { - this.logger.debug( - `Creating new pending document: ${relativePath} (${documentId})` - ); - const previousEntry = - this.getLatestDocumentByRelativePath(relativePath); - - const entry = { - relativePath, - documentId, - metadata: undefined, - isDeleted: false, - updates: [promise], - parallelVersion: - previousEntry?.parallelVersion === undefined - ? 0 - : previousEntry.parallelVersion + 1 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public createNewEmptyDocument( - documentId: DocumentId, - parentVersionId: VaultUpdateId, - relativePath: RelativePath - ): DocumentRecord { - const entry = { - relativePath, - documentId, - metadata: { - parentVersionId, - hash: EMPTY_HASH, - remoteRelativePath: relativePath - }, - isDeleted: false, - updates: [], - parallelVersion: 0 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public getDocumentByDocumentId( - find: DocumentId - ): DocumentRecord | undefined { - return this.documents.find(({ documentId }) => documentId === find); - } - - public move( - oldRelativePath: RelativePath, - newRelativePath: RelativePath - ): void { - const oldDocument = - this.getLatestDocumentByRelativePath(oldRelativePath); - - if (oldDocument === undefined) { - return; - } - - const newDocument = - this.getLatestDocumentByRelativePath(newRelativePath); - if (newDocument?.isDeleted === false) { + /** Bind an empty database to the configured vault. */ + public async bindVault(vaultKey: string): Promise { + if (this.state.vaultKey === vaultKey) return; + if (!canRebindEmpty(this.state)) throw new Error( - `Document already exists at new location: ${newRelativePath}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); - } - - oldDocument.relativePath = newRelativePath; - // We're in a strange state where the target of the move has just got deleted, - // however, its metadata might already have a bunch of updates queued up for - // the document at the new location. We need to keep these updates. - oldDocument.parallelVersion = - newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - - this.saveInTheBackground(); + await this.commit({ ...this.state, vaultKey }); } - public delete(relativePath: RelativePath): void { - const candidate = this.getLatestDocumentByRelativePath(relativePath); - if (candidate === undefined) { + public async commit(next: StoredDatabase): Promise { + const snapshot = structuredClone(next); + if (this.needsReload) throw new Error( - `Document not found by relative path: ${relativePath}` + "Reload durable state before retrying an uncertain save" ); - } - candidate.isDeleted = true; + this.needsReload = true; + await this.saveData(snapshot); + this.state = snapshot; + this.needsReload = false; } - public getHasInitialSyncCompleted(): boolean { - return this.hasInitialSyncCompleted; - } - - public setHasInitialSyncCompleted(value: boolean): void { - this.hasInitialSyncCompleted = value; - this.saveInTheBackground(); - } - - public getLastSeenUpdateId(): VaultUpdateId { - return this.lastSeenUpdateIds.min; - } - - public addSeenUpdateId(value: number): void { - const previousMin = this.lastSeenUpdateIds.min; - this.lastSeenUpdateIds.add(value); - if (previousMin !== this.lastSeenUpdateIds.min) { - this.saveInTheBackground(); - } - } - - public setLastSeenUpdateId(value: number): void { - this.lastSeenUpdateIds.min = value; - this.saveInTheBackground(); - } - - public reset(): void { - this.documents = []; - this.lastSeenUpdateIds = new CoveredValues( - 0 // the first updateId will be 1 which is the first integer after -1 - ); - this.hasInitialSyncCompleted = false; - this.saveInTheBackground(); + public async recoverPersistence(): Promise { + if (!this.needsReload) return; + const saved = await this.loadData(); + if (saved) this.state = saved; + this.needsReload = false; } public async save(): Promise { - return this.saveData({ - documents: this.resolvedDocuments.map( - ({ relativePath, documentId, metadata }) => ({ - documentId, - relativePath, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ...metadata! // `resolvedDocuments` only returns docs with metadata set - }) - ), - lastSeenUpdateId: this.lastSeenUpdateIds.min, - hasInitialSyncCompleted: this.hasInitialSyncCompleted - }); + await this.commit(this.state); } - private ensureConsistency(): void { - const idToPath = new Map(); - - this.resolvedDocuments.forEach(({ relativePath, documentId }) => { - idToPath.set(documentId, [ - ...(idToPath.get(documentId) ?? []), - relativePath - ]); - }); - - const duplicates = Array.from(idToPath.entries()) - .filter(([_, paths]) => paths.length > 1) - .map(([id, paths]) => `${id} (${paths.join(", ")})`); - - if (duplicates.length > 0) { - throw new Error( - "Document IDs are not unique, found duplicates: " + - duplicates.join("; ") - ); - } + public get length(): number { + return Object.keys(this.state.local).length; } - private saveInTheBackground(): void { - this.ensureConsistency(); - void this.save().catch((error: unknown) => { - this.logger.error(`Error saving data: ${error}`); - }); + public getLastSeenUpdateId(): number { + return this.state.lastSeenUpdateId; + } + + public getDocumentByDocumentId(id: string): DocumentRecord | undefined { + const relativePath = this.state.local[id]; + if (relativePath === undefined) return undefined; + const base = this.state.documents[id]?.base; + return { + documentId: id, + relativePath, + isDeleted: false, + metadata: base && { + parentVersionId: base.vaultUpdateId, + hash: base.hash, + remoteRelativePath: this.state.fileManifest.entries[id] + } + }; + } + + public getLatestDocumentByRelativePath( + path: string + ): DocumentRecord | undefined { + const id = Object.keys(this.state.local).find( + (id) => this.state.local[id] === path + ); + return id === undefined ? undefined : this.getDocumentByDocumentId(id); } } diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 3c5f604c..48629d49 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -3,6 +3,7 @@ import type { FetchController } from "./fetch-controller"; import type { Logger } from "../tracing/logger"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; import type { DocumentVersion } from "./types/DocumentVersion"; +import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { EventBatch } from "./types/EventBatch"; import type { FileManifest } from "./types/FileManifest"; import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse"; @@ -10,7 +11,11 @@ import type { PutFileContent } from "./types/PutFileContent"; import type { PushFileManifest } from "./types/PushFileManifest"; import type { VaultSnapshot } from "./types/VaultSnapshot"; import type { PingResponse } from "./types/PingResponse"; -import { AuthenticationError, PermanentSyncError } from "../errors/errors"; +import { + AuthenticationError, + PermanentSyncError, + ServerHistoryChangedError +} from "../errors/errors"; export class SyncService { private readonly client: typeof fetch; @@ -21,7 +26,11 @@ export class SyncService { fetchController: FetchController, private readonly settings: Settings, logger: Logger, - fetchImplementation: typeof fetch = globalThis.fetch + fetchImplementation: typeof fetch = globalThis.fetch, + private readonly history?: { + get: () => string | undefined; + save: (checkpoint: string | undefined) => Promise; + } ) { this.rawFetch = async (...args) => fetchImplementation(...args); this.client = fetchController.getControlledFetchImplementation( @@ -30,6 +39,42 @@ export class SyncService { ); } + public get verifiesHistory(): boolean { + return this.history !== undefined; + } + public get hasHistoryCheckpoint(): boolean { + return this.history?.get() !== undefined; + } + + public async resetHistory(): Promise { + await this.history?.save(undefined); + } + + private historyHeaders(): Record { + const checkpoint = this.history?.get(); + return checkpoint ? { "X-Vault-Link-History": checkpoint } : {}; + } + + private async recordHistory(response: Response): Promise { + const checkpoint = response.headers.get("x-vault-link-history"); + if (checkpoint && this.history) { + const previous = this.history.get(); + if ( + !previous || + Number(checkpoint.split(":")[0]) >= + Number(previous.split(":")[0]) + ) + await this.history.save(checkpoint); + } + } + + private checkHistory(response: Response): void { + if (response.headers.get("x-vault-link-history-mismatch") === "1") + throw new ServerHistoryChangedError( + "Server history changed; recovering local work" + ); + } + public getUrl(path: string): string { const { remoteUri, vaultName } = this.settings.getSettings(); return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`; @@ -49,6 +94,7 @@ export class SyncService { this.settings.getSettings().requestTimeoutMs ), headers: { + ...(raw ? {} : this.historyHeaders()), Authorization: `Bearer ${this.settings.getSettings().token}`, "Device-Id": this.deviceId, "Content-Type": "application/json" @@ -56,6 +102,7 @@ export class SyncService { } ); + this.checkHistory(response); if (!response.ok) { this.throwForErrorResponse( response, @@ -63,13 +110,11 @@ export class SyncService { ); } + if (!raw) await this.recordHistory(response); return (await response.json()) as T; } - private throwForErrorResponse( - response: Response, - message: string - ): never { + private throwForErrorResponse(response: Response, message: string): never { if (response.status === 401 || response.status === 403) { throw new AuthenticationError(message); } @@ -117,6 +162,10 @@ export class SyncService { return this.request(`/documents/${id}`); } + public async metadata(id: string): Promise { + return this.request(`/documents/${id}/metadata`); + } + public async getDocumentVersionContent({ documentId, vaultUpdateId @@ -133,11 +182,13 @@ export class SyncService { this.settings.getSettings().requestTimeoutMs ), headers: { + ...this.historyHeaders(), Authorization: `Bearer ${this.settings.getSettings().token}` } } ); + this.checkHistory(response); if (!response.ok) { this.throwForErrorResponse( response, @@ -145,6 +196,7 @@ export class SyncService { ); } + await this.recordHistory(response); return new Uint8Array(await response.arrayBuffer()); } } diff --git a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts index 572e2d9d..85dc56ab 100644 --- a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts @@ -1,8 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { DocumentVersion } from "./DocumentVersion"; import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; /** * Response to an update document request. */ -export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersion; +export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersionWithoutContent; diff --git a/frontend/sync-client/src/services/types/EventBatch.ts b/frontend/sync-client/src/services/types/EventBatch.ts index f17f17ed..a3bfb573 100644 --- a/frontend/sync-client/src/services/types/EventBatch.ts +++ b/frontend/sync-client/src/services/types/EventBatch.ts @@ -1,4 +1,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { EventRecord } from "./EventRecord"; -export type EventBatch = { headEventId: number, events: Array, }; +export type EventBatch = { headEventId: number, +/** + * Last event in this page; `head_event_id` is the vault's current head. + */ +endEventId?: number, events: Array, }; diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts index 969a02eb..d5223b6a 100644 --- a/frontend/sync-client/src/services/websocket-manager.test.ts +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -1,9 +1,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ import { describe, it, beforeEach } from "node:test"; import assert from "node:assert"; +import { createServer } from "node:net"; import { WebSocketManager } from "./websocket-manager"; -import type { Logger } from "../tracing/logger"; -import type { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { Settings } from "../persistence/settings"; // eslint-disable-next-line @typescript-eslint/no-require-imports const WebSocket = require("ws") as typeof globalThis.WebSocket; @@ -337,3 +338,179 @@ describe("WebSocketManager", () => { await manager.stop(); }); }); + +it( + "refused WebSocket connections and replacing a connecting socket do not emit unhandled errors", + { timeout: 5_000 }, + async () => { + const server = createServer(); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve) + ); + const address = server.address(); + assert(address && typeof address !== "string"); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ); + const logger = new Logger(); + const settings = new Settings( + logger, + { + remoteUri: `http://127.0.0.1:${address.port}`, + webSocketRetryIntervalMs: 10 + }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + WebSocket + ); + let closes = 0; + const disconnected = Promise.withResolvers(); + manager.onWebSocketStatusChanged.add((connected) => { + if (!connected && ++closes >= 2) disconnected.resolve(); + }); + try { + manager.start(); + manager.start(); + await disconnected.promise; + assert.equal(manager.isWebSocketConnected, false); + } finally { + await manager.stop(); + } + } +); + +for (const failedAttempt of [1, 2]) { + it(`recovers from a WebSocket constructor failure on attempt ${failedAttempt}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + let attempts = 0; + class FailingSocket extends MockWebSocket { + static OPEN = 1; + constructor(url: string) { + if (++attempts === failedAttempt) + throw new Error("socket allocation failed"); + super(url); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test", webSocketRetryIntervalMs: 1000 }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + FailingSocket as unknown as typeof WebSocket + ); + try { + assert.doesNotThrow(() => manager.start()); + context.mock.timers.tick(0); + if (failedAttempt === 2) { + const socket = ( + manager as unknown as { webSocket: MockWebSocket } + ).webSocket; + socket.close(); + } + assert.doesNotThrow(() => context.mock.timers.tick(1000)); + context.mock.timers.tick(1000); + assert.equal(attempts, failedAttempt + 1); + assert.equal(manager.isWebSocketConnected, true); + } finally { + await manager.stop(); + } + }); +} + +for (const closeBehavior of ["throws", "stays open"]) { + it(`stop retires a socket even when close ${closeBehavior}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + class BrokenCloseSocket extends MockWebSocket { + static OPEN = 1; + close() { + if (closeBehavior === "throws") throw new Error("close failed"); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test" }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + BrokenCloseSocket as unknown as typeof WebSocket + ); + let connected = false, + delivered = 0; + manager.onWebSocketStatusChanged.add((status) => { + connected = status; + }); + manager.onRemoteVaultUpdateReceived.add(async () => { + delivered++; + }); + manager.start(); + context.mock.timers.tick(0); + const socket = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + const stopping = manager.stop(); + context.mock.timers.tick(10_000); + await stopping; + assert.equal(manager.isWebSocketConnected, false); + assert.equal(connected, false); + socket.simulateMessage({ + type: "vaultEvents", + headEventId: 0, + events: [] + }); + await manager.waitUntilFinished(); + assert.equal(delivered, 0); + }); +} + +it("a handshake send failure retires the connection without escaping the open callback", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + class BrokenSendSocket extends MockWebSocket { + static OPEN = 1; + send() { + throw new Error("transport closed during open"); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test" }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + BrokenSendSocket as unknown as typeof WebSocket + ); + manager.onWebSocketStatusChanged.add((connected) => { + if (connected) + manager.sendHandshakeMessage({ + type: "handshake", + token: "test", + deviceId: "test", + lastSeenVaultUpdateId: 0 + }); + }); + const statuses: boolean[] = []; + manager.onWebSocketStatusChanged.add((status) => statuses.push(status)); + try { + manager.start(); + assert.doesNotThrow(() => context.mock.timers.tick(0)); + assert.equal(manager.isWebSocketConnected, false); + assert.equal(statuses.at(-1), false); + } finally { + await manager.stop(); + } +}); diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index c2d323f7..3fb80947 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -26,6 +26,7 @@ export class WebSocketManager { private isStopped = true; private resolveDisconnectingPromise: null | (() => unknown) = null; + private receiveTimeoutId: ReturnType | undefined; private reconnectTimeoutId: ReturnType | undefined; private readonly outstandingPromises: Promise[] = []; @@ -56,12 +57,14 @@ export class WebSocketManager { public get isWebSocketConnected(): boolean { return ( - this.webSocket?.readyState === - this.webSocketFactoryImplementation.OPEN + this.webSocket !== undefined && + this.webSocket.readyState === + this.webSocketFactoryImplementation.OPEN ); } public start(): void { + if (!this.isStopped) return; this.isStopped = false; this.initializeWebSocket(); } @@ -71,13 +74,20 @@ export class WebSocketManager { this.resolveDisconnectingPromise = resolve; this.isStopped = true; + clearTimeout(this.receiveTimeoutId); if (this.reconnectTimeoutId !== undefined) { clearTimeout(this.reconnectTimeoutId); this.reconnectTimeoutId = undefined; } - this.webSocket?.close(1000, "WebSocketManager has been stopped"); + const socket = this.webSocket; + try { + socket?.close(1000, "WebSocketManager has been stopped"); + } catch (error) { + this.logger.warn(`Failed to close WebSocket: ${String(error)}`); + if (socket) this.reconnect(socket); + } // eslint-disable-next-line @typescript-eslint/init-declarations let timeoutId: ReturnType | undefined; @@ -107,6 +117,9 @@ export class WebSocketManager { if (timeoutId !== undefined) { clearTimeout(timeoutId); } + // A throwing, silent or timed-out close must still fence callbacks + // before a stopped client can release its state or restart. + if (socket) this.reconnect(socket); } await this.waitUntilFinished(); @@ -132,6 +145,7 @@ export class WebSocketManager { this.logger.error( `Failed to send handshake message: ${String(error)}` ); + this.reconnect(webSocket); throw error; } } @@ -162,7 +176,66 @@ export class WebSocketManager { } } + private expectResponse(socket: WebSocket): void { + clearTimeout(this.receiveTimeoutId); + // Cursor heartbeats are acknowledged every 15 seconds, including when + // there are no open editors. An OPEN TCP socket alone proves nothing. + this.receiveTimeoutId = setTimeout(() => { + if (this.webSocket !== socket || this.isStopped) return; + this.logger.warn("WebSocket receive deadline expired"); + this.reconnect(socket); + }, 45_000); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Browser timers have no unref method. + this.receiveTimeoutId.unref?.(); + } + + private reconnect(socket: WebSocket): void { + if (this.webSocket !== socket) return; + clearTimeout(this.receiveTimeoutId); + socket.onopen = null; + socket.onclose = null; + socket.onmessage = null; + socket.onerror = (): void => { + /* Retired transports may still emit errors. */ + }; + this.webSocket = undefined; + this.onWebSocketStatusChanged.trigger(false); + try { + socket.close(); + } catch { + /* The transport may already be gone. */ + } + if (this.isStopped) { + this.resolveDisconnectingPromise?.(); + this.resolveDisconnectingPromise = null; + } else { + this.scheduleReconnect(); + } + } + + private scheduleReconnect(): void { + if (this.isStopped) return; + clearTimeout(this.reconnectTimeoutId); + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = undefined; + this.initializeWebSocket(); + }, this.settings.getSettings().webSocketRetryIntervalMs); + } + private initializeWebSocket(): void { + if (this.isStopped) return; + try { + this.openWebSocket(); + } catch (error) { + this.logger.warn(`Cannot initialize WebSocket: ${String(error)}`); + this.webSocket = undefined; + clearTimeout(this.receiveTimeoutId); + this.onWebSocketStatusChanged.trigger(false); + this.scheduleReconnect(); + } + } + + private openWebSocket(): void { // Clean up old WebSocket handlers to prevent race conditions if (this.webSocket) { try { @@ -170,7 +243,9 @@ export class WebSocketManager { this.webSocket.onopen = null; this.webSocket.onclose = null; this.webSocket.onmessage = null; - this.webSocket.onerror = null; + this.webSocket.onerror = (): void => { + /* Retired transports may still emit errors. */ + }; this.webSocket.close(); } catch (e) { this.logger.error( @@ -188,7 +263,14 @@ export class WebSocketManager { this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); - this.webSocket = new this.webSocketFactoryImplementation(wsUri); + const socket = new this.webSocketFactoryImplementation(wsUri); + this.webSocket = socket; + this.expectResponse(socket); + + this.webSocket.onerror = (): void => { + this.logger.warn("WebSocket transport error"); + this.reconnect(socket); + }; this.webSocket.onopen = (): void => { // Check if we've been stopped while connecting @@ -200,7 +282,18 @@ export class WebSocketManager { return; } this.logger.info("WebSocket connection opened"); - this.onWebSocketStatusChanged.trigger(true); + try { + this.onWebSocketStatusChanged.trigger(true); + // A handshake failure can retire this socket from inside an + // observer. Finish delivery with its actual connection state. + if (this.webSocket !== socket) + this.onWebSocketStatusChanged.trigger(false); + } catch (error) { + this.logger.warn( + `WebSocket handshake failed: ${String(error)}` + ); + this.reconnect(socket); + } }; this.webSocket.onmessage = (event): void => { @@ -210,6 +303,8 @@ export class WebSocketManager { event.data ) as WebSocketServerMessage; + if (["vaultEvents", "cursorPositions"].includes(message.type)) + this.expectResponse(socket); // Track the message handling promise const messageHandlingPromise = this.handleWebSocketMessage( message @@ -238,17 +333,7 @@ export class WebSocketManager { this.logger.warn( `WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})` ); - this.onWebSocketStatusChanged.trigger(false); - - if (this.isStopped) { - this.resolveDisconnectingPromise?.(); - this.resolveDisconnectingPromise = null; - } else { - this.reconnectTimeoutId = setTimeout(() => { - this.reconnectTimeoutId = undefined; - this.initializeWebSocket(); - }, this.settings.getSettings().webSocketRetryIntervalMs); - } + this.reconnect(socket); }; } diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 2a272c86..71dabeaa 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -1,62 +1,246 @@ -import type { PersistenceProvider } from "./persistence/persistence"; -import type { HistoryEntry, HistoryStats } from "./tracing/sync-history"; -import { SyncHistory } from "./tracing/sync-history"; -import { Logger, LogLevel, LogLine } from "./tracing/logger"; -import type { RelativePath, StoredDatabase } from "./persistence/database"; -import { Database } from "./persistence/database"; import * as Sentry from "@sentry/browser"; -import type { SyncSettings } from "./persistence/settings"; -import { DEFAULT_SETTINGS, Settings } from "./persistence/settings"; -import { SyncService } from "./services/sync-service"; -import { Syncer } from "./sync-operations/syncer"; +import { setUpTelemetry } from "./utils/set-up-telemetry"; +import type { PersistenceProvider } from "./persistence/persistence"; +import { Database, type StoredDatabase } from "./persistence/database"; +import { + Settings, + DEFAULT_SETTINGS, + type SyncSettings +} from "./persistence/settings"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; +import { Logger, LogLevel } from "./tracing/logger"; +import { SyncHistory } from "./tracing/sync-history"; import { FetchController } from "./services/fetch-controller"; -import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer"; -import { rateLimit } from "./utils/rate-limit"; -import type { NetworkConnectionStatus } from "./types/network-connection-status"; -import { DocumentSyncStatus } from "./types/document-sync-status"; -import { WebSocketManager } from "./services/websocket-manager"; -import { createClientId } from "./utils/create-client-id"; -import { CursorTracker } from "./sync-operations/cursor-tracker"; -import type { CursorSpan } from "./services/types/CursorSpan"; -import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; -import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; -import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; -import { setUpTelemetry } from "./utils/set-up-telemetry"; -import { DIFF_CACHE_SIZE_MB } from "./consts"; +import { SyncService } from "./services/sync-service"; import { ServerConfig } from "./services/server-config"; -import type { EventListeners } from "./utils/data-structures/event-listeners"; +import { WebSocketManager } from "./services/websocket-manager"; +import { Syncer } from "./sync-operations/syncer"; +import { CursorTracker } from "./sync-operations/cursor-tracker"; +import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; +import { createClientId } from "./utils/create-client-id"; +import { DocumentSyncStatus } from "./types/document-sync-status"; +import type { CursorSpan } from "./services/types/CursorSpan"; +import { isInternalPath } from "./utils/portable-path"; +import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; +import { globsToRegexes } from "./utils/globs-to-regexes"; +import { Lock } from "./utils/data-structures/locks"; +import type { LocalChange } from "./sync-operations/local-changes"; + +type StoredClient = Partial<{ + settings: Partial; + database: Partial; + localChanges: LocalChange[]; + localChangesVaultKey: string; + historyCheckpoint: { vaultKey: string; checkpoint?: string }; +}>; + +const vaultKey = (settings: SyncSettings): string => + JSON.stringify([ + settings.remoteUri.replace(/\/$/u, ""), + settings.vaultName + ]); export class SyncClient { - private hasStartedOfflineSync = false; - private hasFinishedOfflineSync = false; - private hasStarted = false; - private hasBeenDestroyed = false; + private readonly lifecycle = new Lock(); + private destroying?: Promise; + private started = false; + private destroyed = false; private unloadTelemetry?: () => void; - private isDestroying = false; - private readonly eventUnsubscribers: (() => void)[] = []; - + private readonly unsubscribeLog: () => void; private constructor( + public readonly logger: Logger, private readonly history: SyncHistory, private readonly settings: Settings, private readonly database: Database, private readonly syncer: Syncer, private readonly webSocketManager: WebSocketManager, - public readonly logger: Logger, private readonly fetchController: FetchController, + private readonly serverConfig: ServerConfig, private readonly cursorTracker: CursorTracker, private readonly fileChangeNotifier: FileChangeNotifier, private readonly contentCache: FixedSizeDocumentCache, - private readonly fileOperations: FileOperations, - private readonly serverConfig: ServerConfig, - private readonly persistence: PersistenceProvider< - Partial<{ - settings: Partial; - database: Partial; - }> - > - ) {} + private readonly persistence: PersistenceProvider + ) { + syncer.onServerHistoryChanged.add(() => { + cursorTracker.reset(); + }); + syncer.onReadyForEvents.add(() => { + if ( + this.started && + !this.destroyed && + settings.getSettings().isSyncEnabled + ) + webSocketManager.start(); + }); + this.unsubscribeLog = logger.onLogEmitted.add((log) => { + if (log.level === LogLevel.ERROR && Sentry.isInitialized()) + Sentry.captureMessage(log.message); + }); + } + + public static async create({ + fs, + persistence, + fetch, + webSocket + }: { + fs: FileSystemOperations; + persistence: PersistenceProvider; + fetch?: typeof globalThis.fetch; + webSocket?: typeof globalThis.WebSocket; + /** V4 preserves bytes and line endings as stored, on every platform. */ + nativeLineEndings?: string; + }): Promise { + const logger = new Logger(); + let stored = (await persistence.load()) ?? {}; + let saving: Promise = Promise.resolve(); + let reloadBeforeSave = false; + // Settings and engine state share a store. Serialize complete durable + // replacements so one cannot overwrite a newer save from the other. + const save = async (update: StoredClient): Promise => { + const operation = saving.then(async () => { + if (reloadBeforeSave) { + stored = (await persistence.load()) ?? {}; + reloadBeforeSave = false; + } + const next = { ...stored, ...structuredClone(update) }; + try { + await persistence.save(next); + stored = next; + } catch (error) { + // A failed save may have committed. Do not let a queued + // settings/notification save roll back that engine state. + reloadBeforeSave = true; + throw error; + } + }); + saving = operation.catch(() => {}); + return operation; + }; + const settings = new Settings(logger, stored.settings, async (data) => + save({ settings: data }) + ); + const database = new Database( + logger, + stored.database, + async (data) => save({ database: data }), + vaultKey(settings.getSettings()), + async () => { + await saving; + stored = (await persistence.load()) ?? {}; + return stored.database as StoredDatabase | undefined; + } + ); + if ( + stored.localChanges?.length && + stored.localChangesVaultKey !== vaultKey(settings.getSettings()) + ) { + throw new Error( + "Offline notifications belong to another vault; use a separate state store" + ); + } + const fetchController = new FetchController( + settings.getSettings().isSyncEnabled, + logger + ); + const deviceId = createClientId(); + const service = new SyncService( + deviceId, + fetchController, + settings, + logger, + fetch, + { + get: () => + stored.historyCheckpoint?.vaultKey === + vaultKey(settings.getSettings()) + ? stored.historyCheckpoint.checkpoint + : undefined, + save: async (checkpoint) => + save({ + historyCheckpoint: { + vaultKey: vaultKey(settings.getSettings()), + checkpoint + } + }) + } + ); + const serverConfig = new ServerConfig(service); + const notifier = new FileChangeNotifier(); + const localChanges = stored.localChanges ?? []; + let syncer: Syncer; + const files = new FileOperations( + fs, + database, + serverConfig, + (paths) => { + for (const path of paths) notifier.notifyOfFileChange(path); + }, + { + entries: () => localChanges, + flush: async () => syncer?.flushLocalChanges() + }, + (path, size) => { + const current = settings.getSettings(); + return ( + size > current.maxFileSizeMB * 1024 * 1024 || + globsToRegexes(current.ignorePatterns, logger).some( + (pattern) => pattern.test(path) + ) + ); + } + ); + const websocket = new WebSocketManager( + deviceId, + logger, + settings, + webSocket + ); + const history = new SyncHistory(logger); + const contentCache = new FixedSizeDocumentCache( + settings.getSettings().diffCacheSizeMB * 1024 * 1024 + ); + syncer = new Syncer( + deviceId, + logger, + database, + settings, + service, + websocket, + files, + serverConfig, + history, + contentCache, + { + entries: localChanges, + save: async (entries) => + save({ + localChanges: entries, + localChangesVaultKey: vaultKey(settings.getSettings()) + }) + } + ); + const cursors = new CursorTracker(database, websocket, files, notifier); + return new SyncClient( + logger, + history, + settings, + database, + syncer, + websocket, + fetchController, + serverConfig, + cursors, + notifier, + contentCache, + persistence + ); + } + + private check(): void { + if (this.destroyed) throw new Error("SyncClient has been destroyed"); + } public get documentCount(): number { return this.database.length; @@ -66,478 +250,229 @@ export class SyncClient { return this.webSocketManager.isWebSocketConnected; } - public get onSyncHistoryUpdated(): EventListeners< - (stats: HistoryStats) => unknown - > { - this.checkIfDestroyed("onSyncHistoryUpdated getter"); + public get onSyncHistoryUpdated() { return this.history.onHistoryUpdated; } - public get onSettingsChanged(): EventListeners< - (newSettings: SyncSettings, oldSettings: SyncSettings) => unknown - > { - this.checkIfDestroyed("onSettingsChanged getter"); + public get onSettingsChanged() { return this.settings.onSettingsChanged; } - public get onRemainingOperationsCountChanged(): EventListeners< - (remainingOperationsCount: number) => unknown - > { - this.checkIfDestroyed("onRemainingOperationsCountChanged getter"); + public get onRemainingOperationsCountChanged() { return this.syncer.onRemainingOperationsCountChanged; } - public get onWebSocketStatusChanged(): EventListeners< - (isConnected: boolean) => unknown - > { - this.checkIfDestroyed("onWebSocketStatusChanged getter"); + public get onWebSocketStatusChanged() { return this.webSocketManager.onWebSocketStatusChanged; } - public get onRemoteCursorsUpdated(): EventListeners< - (cursors: MaybeOutdatedClientCursors[]) => unknown - > { - this.checkIfDestroyed("onRemoteCursorsUpdated getter"); + public get onRemoteCursorsUpdated() { return this.cursorTracker.onRemoteCursorsUpdated; } - public static async create({ - fs, - persistence, - fetch, - webSocket, - nativeLineEndings = "\n" - }: { - fs: FileSystemOperations; - persistence: PersistenceProvider< - Partial<{ - settings: Partial; - database: Partial; - }> - >; - fetch?: typeof globalThis.fetch; - webSocket?: typeof globalThis.WebSocket; - nativeLineEndings?: string; - }): Promise { - const logger = new Logger(); - - const deviceId = createClientId(); - - logger.info(`Creating SyncClient with client id ${deviceId}`); - - const history = new SyncHistory(logger); - - let state = (await persistence.load()) ?? { - settings: undefined, - database: undefined - }; - - const settings = new Settings( - logger, - state.settings, - async (data): Promise => { - state = { ...state, settings: data }; - // we're not rate-limiting settings saves as (1) we need to initialise the settings to know the rate limit - // and (2) settings changes are infrequent enough that rate-limiting is not necessary - await persistence.save(state); - } - ); - - const rateLimitedSave = rateLimit( - persistence.save, - () => settings.getSettings().minimumSaveIntervalMs - ); - - const database = new Database( - logger, - state.database, - async (data): Promise => { - state = { ...state, database: data }; - await rateLimitedSave(state); - } - ); - - const fetchController = new FetchController( - settings.getSettings().isSyncEnabled, - logger - ); - - const syncService = new SyncService( - deviceId, - fetchController, - settings, - logger, - fetch - ); - - const serverConfig = new ServerConfig(syncService); - - const fileOperations = new FileOperations( - logger, - database, - fs, - serverConfig, - nativeLineEndings - ); - - const contentCache = new FixedSizeDocumentCache( - 1024 * 1024 * DIFF_CACHE_SIZE_MB - ); - const unrestrictedSyncer = new UnrestrictedSyncer( - logger, - database, - settings, - syncService, - fileOperations, - history, - contentCache, - serverConfig - ); - - const webSocketManager = new WebSocketManager( - deviceId, - logger, - settings, - webSocket - ); - - const syncer = new Syncer( - deviceId, - logger, - database, - settings, - syncService, - webSocketManager, - fileOperations, - unrestrictedSyncer - ); - - const fileChangeNotifier = new FileChangeNotifier(); - const cursorTracker = new CursorTracker( - database, - webSocketManager, - fileOperations, - fileChangeNotifier - ); - const client = new SyncClient( - history, - settings, - database, - syncer, - webSocketManager, - logger, - fetchController, - cursorTracker, - fileChangeNotifier, - contentCache, - fileOperations, - serverConfig, - persistence - ); - - logger.info("SyncClient created successfully"); - - return client; - } - - public async start(): Promise { - this.checkIfDestroyed("start"); - - if (this.hasStarted) { - throw new Error("SyncClient has already been started"); - } - this.hasStarted = true; - - if ( - !this.unloadTelemetry && - this.settings.getSettings().enableTelemetry - ) { - this.unloadTelemetry = setUpTelemetry(); - } - - this.eventUnsubscribers.push( - this.settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) { - this.fetchController.canFetch = newSettings.isSyncEnabled; - } - }) - ); - - this.eventUnsubscribers.push( - this.logger.onLogEmitted.add((log): void => { - if (log.level === LogLevel.ERROR && Sentry.isInitialized()) { - Sentry.captureMessage(log.message); - } - }) - ); - - this.eventUnsubscribers.push( - this.settings.onSettingsChanged.add( - this.onSettingsChange.bind(this) - ) - ); - - if (this.settings.getSettings().isSyncEnabled) { - this.logger.info("Starting SyncClient"); - await this.startSyncing(); - this.logger.info("SyncClient has successfully started"); - } - } - - /** - * Reload settings from disk overriding current in-memory settings. - * Missing values will be filled in from DEFAULT_SETTINGS rather than - * retaining current in-memory settings. - */ - public async reloadSettings(): Promise { - this.checkIfDestroyed("reloadSettings"); - - const state = (await this.persistence.load()) ?? { - settings: undefined - }; - - const settings = { - ...DEFAULT_SETTINGS, - ...(state.settings ?? {}) - }; - - await this.setSettings(settings); - } - - public async checkConnection(): Promise { - this.checkIfDestroyed("checkConnection"); - - const server = await this.serverConfig.checkConnection(true); - return { - isSuccessful: server.isSuccessful, - serverMessage: server.message, - isWebSocketConnected: this.webSocketManager.isWebSocketConnected - }; - } - - public getHistoryEntries(): readonly HistoryEntry[] { + public getHistoryEntries() { return this.history.entries; } - /** - * Wait for the in-flight operations to finish, reset all tracking, - * and the local database but retain the settings. - * The SyncClient can be used again after calling this method. - */ - public async reset(): Promise { - this.checkIfDestroyed("reset"); - - this.logger.info( - "Stopping SyncClient to apply changed connection settings" - ); - await this.pause(); - - // clear all local state - this.logger.info("Resetting SyncClient's local state"); - this.database.reset(); - await this.database.save(); // ensure the new database reads as empty - this.resetInMemoryState(); - this.hasStartedOfflineSync = false; - this.hasFinishedOfflineSync = false; - this.serverConfig.reset(); - - await this.startSyncing(); - } - public getSettings(): SyncSettings { return this.settings.getSettings(); } - public async setSetting( - key: T, - value: SyncSettings[T] - ): Promise { - this.checkIfDestroyed("setSetting"); - - await this.settings.setSetting(key, value); - } - - public async setSettings(value: Partial): Promise { - this.checkIfDestroyed("setSettings"); - - await this.settings.setSettings(value); - } - - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { - this.checkIfDestroyed("syncLocallyCreatedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyCreatedFile(relativePath); - } - - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - this.checkIfDestroyed("syncLocallyDeletedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyDeletedFile(relativePath); - } - - public async syncLocallyUpdatedFile({ - oldPath, - relativePath - }: { - oldPath?: RelativePath; - relativePath: RelativePath; - }): Promise { - this.checkIfDestroyed("syncLocallyUpdatedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyUpdatedFile({ - oldPath, - relativePath - }); - } - - public getDocumentSyncingStatus( - relativePath: RelativePath - ): DocumentSyncStatus { - this.checkIfDestroyed("getDocumentSyncingStatus"); - - if (!this.settings.getSettings().isSyncEnabled) { - return DocumentSyncStatus.SYNCING_IS_DISABLED; - } - - if (!this.syncer.isFirstSyncComplete || !this.hasFinishedOfflineSync) { - return DocumentSyncStatus.SYNCING; - } - - const document = - this.database.getLatestDocumentByRelativePath(relativePath); - if (document === undefined) { - return DocumentSyncStatus.SYNCING; - } - return document.updates.length > 0 - ? DocumentSyncStatus.SYNCING - : DocumentSyncStatus.UP_TO_DATE; - } - - public async updateLocalCursors( - documentToCursors: Record - ): Promise { - this.checkIfDestroyed("updateLocalCursors"); - - await this.cursorTracker.sendLocalCursorsToServer(documentToCursors); - } - - public async waitUntilFinished(): Promise { - this.checkIfDestroyed("waitUntilIdle"); - await this.syncer.waitUntilFinished(); - await this.webSocketManager.waitUntilFinished(); - await this.database.save(); // flush all changes to disk - } - - /** - * Completely destroy the SyncClient, cancelling all in-progress operations. - * After calling this method, the SyncClient cannot be used again. - */ - public async destroy(): Promise { - this.checkIfDestroyed("destroy"); - - // Prevent concurrent destroy calls - if (this.isDestroying) { - this.logger.warn( - "destroy() called while already destroying, ignoring" + public async start(): Promise { + await this.lifecycle.withLock(async () => { + this.check(); + if (this.started) + throw new Error("SyncClient has already been started"); + await this.syncer.recoverLocalState(); + await this.database.bindVault( + vaultKey(this.settings.getSettings()) ); - return; - } - this.isDestroying = true; - - // cancel everything that's in progress - await this.pause(); - - this.hasBeenDestroyed = true; - - this.resetInMemoryState(); - - // Clean up event listeners to prevent memory leaks - this.eventUnsubscribers.forEach((unsubscribe) => { - unsubscribe(); + this.started = true; + this.configureTelemetry(); + if (this.settings.getSettings().isSyncEnabled) await this.resume(); }); - this.eventUnsubscribers.length = 0; - - this.logger.info("SyncClient has been successfully disposed"); - - this.unloadTelemetry?.(); } - private async startSyncing(): Promise { - this.checkIfDestroyed("startSyncing"); + private async resume(): Promise { + if (this.destroyed) return; + if (!this.settings.getSettings().isSyncEnabled) return; + await this.database.bindVault(vaultKey(this.settings.getSettings())); this.fetchController.finishReset(); - - await this.serverConfig.initialize(); - this.webSocketManager.start(); - - if (!this.hasStartedOfflineSync) { - this.hasStartedOfflineSync = true; - await this.syncer.scheduleSyncForOfflineChanges(); - } - - this.hasFinishedOfflineSync = true; + this.fetchController.canFetch = true; + this.syncer.start(); + await this.syncer.waitUntilFinished(); } private async pause(): Promise { + const stopped = this.syncer.stop(); this.fetchController.startReset(); await this.webSocketManager.stop(); - await this.waitUntilFinished(); + await stopped; } - private resetInMemoryState(): void { - this.history.reset(); - this.contentCache.reset(); - // don't reset the logger - this.cursorTracker.reset(); - this.syncer.reset(); - this.fileOperations.reset(); + /** Restart transports; never discard pending requests or filesystem journals. */ + public async reset(): Promise { + await this.lifecycle.withLock(async () => { + this.check(); + await this.pause(); + this.serverConfig.reset(); + this.cursorTracker.reset(); + await this.syncer.retryRejectedRequests(); + if (this.started && this.settings.getSettings().isSyncEnabled) + await this.resume(); + }); } - private async onSettingsChange( - newSettings: SyncSettings, - oldSettings: SyncSettings - ): Promise { - this.checkIfDestroyed("onSettingsChange"); + public async setSettings(change: Partial): Promise { + const update = structuredClone(change); + const notification = await this.lifecycle.withLock(async () => { + this.check(); + const old = this.settings.getSettings(); + const next = { ...old, ...update }; + if ( + vaultKey(next) !== vaultKey(old) && + (this.started || + this.database.state.initialized || + this.database.state.bootstrap || + this.database.state.pending || + this.database.state.application || + this.syncer.hasLocalChanges || + Object.keys(this.database.state.local).length) + ) + throw new Error( + "Changing vaults requires a separate state store and client; pending recovery data must not be reset" + ); - if ( - newSettings.vaultName !== oldSettings.vaultName || - newSettings.remoteUri !== oldSettings.remoteUri - ) { - await this.reset(); - } - - if (newSettings.isSyncEnabled !== oldSettings.isSyncEnabled) { - if (newSettings.isSyncEnabled) { - await this.startSyncing(); - } else { - await this.pause(); + try { + if (this.started) await this.pause(); + await this.database.recoverPersistence(); + await this.settings.setSettings(update, false); + await this.syncer.retryRejectedRequests(); + this.contentCache.resize(next.diffCacheSizeMB * 1024 * 1024); + await this.database.bindVault(vaultKey(next)); + } catch (error) { + this.serverConfig.reset(); + this.contentCache.resize( + this.settings.getSettings().diffCacheSizeMB * 1024 * 1024 + ); + this.configureTelemetry(); + if (this.started && this.settings.getSettings().isSyncEnabled) + await this.resume().catch((resumeError: unknown) => { + this.logger.error( + `Settings recovery could not resume sync: ${String(resumeError)}` + ); + }); + throw error; } - } + this.serverConfig.reset(); + this.configureTelemetry(); + if (this.started && next.isSyncEnabled) await this.resume(); + return [this.settings.getSettings(), old] as const; + }); + await this.settings.onSettingsChanged.triggerAsync(...notification); + } - if (newSettings.diffCacheSizeMB !== oldSettings.diffCacheSizeMB) { - this.contentCache.resize(newSettings.diffCacheSizeMB * 1024 * 1024); - } + public async setSetting( + key: K, + value: SyncSettings[K] + ): Promise { + await this.setSettings({ [key]: value }); + } - if (newSettings.enableTelemetry !== oldSettings.enableTelemetry) { - if (newSettings.enableTelemetry) { - this.unloadTelemetry = setUpTelemetry(); - } else { + public async reloadSettings(): Promise { + const stored = await this.persistence.load(); + await this.setSettings({ ...DEFAULT_SETTINGS, ...stored?.settings }); + } + + public async checkConnection() { + const result = await this.serverConfig.checkConnection(true); + return { + isSuccessful: result.isSuccessful, + serverMessage: result.message, + isWebSocketConnected: this.isWebSocketConnected + }; + } + + /** Notifications enqueue work. Use waitUntilFinished() to await convergence. */ + public async syncLocallyCreatedFile(path: string): Promise { + this.check(); + if (isInternalPath(path)) return; + this.fileChangeNotifier.notifyOfFileChange(path); + await this.syncer.syncLocallyCreatedFile(path); + } + + public async syncLocallyDeletedFile(path: string): Promise { + this.check(); + if (isInternalPath(path)) return; + this.fileChangeNotifier.notifyOfFileChange(path); + await this.syncer.syncLocallyDeletedFile(path); + } + + public async syncLocallyUpdatedFile(change: { + oldPath?: string; + relativePath: string; + }): Promise { + this.check(); + if ( + isInternalPath(change.relativePath) || + (change.oldPath && isInternalPath(change.oldPath)) + ) + return; + this.fileChangeNotifier.notifyOfFileChange(change.relativePath); + await this.syncer.syncLocallyUpdatedFile(change); + } + + public async updateLocalCursors( + cursors: Record + ): Promise { + this.check(); + await this.cursorTracker.sendLocalCursorsToServer(cursors); + } + + public getDocumentSyncingStatus(path: string): DocumentSyncStatus { + this.check(); + if (!this.settings.getSettings().isSyncEnabled) + return DocumentSyncStatus.SYNCING_IS_DISABLED; + return this.syncer.isDocumentUpToDate(path) + ? DocumentSyncStatus.UP_TO_DATE + : DocumentSyncStatus.SYNCING; + } + + public async waitUntilFinished(): Promise { + this.check(); + await this.syncer.waitUntilFinished(); + } + + private configureTelemetry(): void { + if ( + this.settings.getSettings().enableTelemetry && + typeof window !== "undefined" + ) { + this.unloadTelemetry ??= setUpTelemetry(); + } else { + this.unloadTelemetry?.(); + this.unloadTelemetry = undefined; + } + } + + public async destroy(): Promise { + if (this.destroying) return this.destroying; + this.destroyed = true; + // Interrupt a start/reset awaiting network progress before waiting for + // its lifecycle operation to finish. + this.fetchController.startReset(); + const stopped = this.syncer.stop(); + this.destroying = this.lifecycle.withLock(async () => { + try { + await this.webSocketManager.stop(); + await stopped; + await this.syncer.flushLocalChanges(); + } finally { + this.cursorTracker.reset(); + this.unsubscribeLog(); this.unloadTelemetry?.(); } - } - } - - private checkIfDestroyed(origin: string): void { - if (this.hasBeenDestroyed) { - throw new Error( - `SyncClient has been destroyed and can no longer be used; called from ${origin}` - ); - } + }); + return this.destroying; } } diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts index 01eac85a..8828d887 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts @@ -23,6 +23,9 @@ it("retains future cursors and publishes them when content catches up", async () }) } as unknown as Database; const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), onRemoteCursorsUpdateReceived: new EventListeners< (cursors: ClientCursors[]) => Promise >(), @@ -65,3 +68,209 @@ it("retains future cursors and publishes them when content catches up", async () "note.md" ); }); + +it("refreshes unchanged cursors on reconnect and heartbeat, and follows the document through renames", async (context) => { + context.mock.timers.enable({ apis: ["setInterval"] }); + const content = new TextEncoder().encode("hello"); + let path = "note.md"; + const record = () => ({ + documentId: "document", + relativePath: path, + metadata: { parentVersionId: 1, hash: contentHash } + }); + const contentHash = await hash(content); + const database = { + getLatestDocumentByRelativePath: (candidate: string) => + candidate === path ? record() : undefined, + getDocumentByDocumentId: () => record() + } as unknown as Database; + const sent: { + documentsWithCursors: { + relative_path: string; + vault_update_id: number | null; + }[]; + }[] = []; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: (value: (typeof sent)[number]) => { + sent.push(value); + } + } as unknown as WebSocketManager; + const notifier = new FileChangeNotifier(); + const tracker = new CursorTracker( + database, + websocket, + { read: async () => content } as unknown as FileOperations, + notifier + ); + const drain = async () => { + for (let i = 0; i < 20; i++) + await new Promise((resolve) => setImmediate(resolve)); + }; + try { + await tracker.sendLocalCursorsToServer({ + "note.md": [{ start: 1, end: 2 }] + }); + assert.equal(sent.length, 1); + await tracker.sendLocalCursorsToServer({ + "note.md": [{ start: 1, end: 2 }] + }); + assert.equal(sent.length, 1); + websocket.onWebSocketStatusChanged.trigger(true); + await drain(); + assert.equal(sent.length, 2); + context.mock.timers.tick(15_000); + await drain(); + assert.equal(sent.length, 3); + path = "renamed.md"; + await notifier.onFileChanged.triggerAsync(path); + assert.equal(sent.at(-1)!.documentsWithCursors[0].relative_path, path); + websocket.onWebSocketStatusChanged.trigger(false); + const count = sent.length; + context.mock.timers.tick(30_000); + await drain(); + assert.equal(sent.length, count); + websocket.onWebSocketStatusChanged.trigger(true); + await drain(); + assert.equal(sent.length, count + 1); + } finally { + tracker.reset(); + context.mock.timers.reset(); + } +}); + +for (const interruption of ["disconnect", "reset"] as const) { + it(`an in-flight cursor read cannot resurrect presence after ${interruption}`, async () => { + const content = new TextEncoder().encode("current"); + const contentHash = await hash(content); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const database = { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata: { parentVersionId: 1, hash: contentHash } + }) + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const tracker = new CursorTracker( + database, + websocket, + { + read: async () => { + entered.resolve(); + await release.promise; + return content; + } + } as unknown as FileOperations, + new FileChangeNotifier() + ); + let received: ClientCursors[] = []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + received = cursors; + }); + const cursor: ClientCursors = { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 1 }] + } + ] + }; + const first = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + cursor + ]); + await entered.promise; + const queued = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + cursor + ]); + if (interruption === "reset") tracker.reset(); + else websocket.onWebSocketStatusChanged.trigger(false); + assert.deepEqual(received, []); + release.resolve(); + await first; + await queued; + assert.deepEqual( + received, + [], + "a retired connection must not restore its cursors" + ); + tracker.reset(); + }); +} + +it("a remote cursor becomes outdated if the document advances during its snapshot read", async () => { + const content = new TextEncoder().encode("old"); + let metadata = { parentVersionId: 1, hash: await hash(content) }; + const entered = Promise.withResolvers(), + release = Promise.withResolvers(); + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const tracker = new CursorTracker( + { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata + }) + } as unknown as Database, + websocket, + { + read: async () => { + entered.resolve(); + await release.promise; + return content; + } + } as unknown as FileOperations, + new FileChangeNotifier() + ); + let outdated: boolean | undefined; + tracker.onRemoteCursorsUpdated.add((cursors) => { + outdated = cursors[0]?.isOutdated; + }); + const update = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 1 }] + } + ] + } + ]); + await entered.promise; + metadata = { + parentVersionId: 2, + hash: await hash(new TextEncoder().encode("new content")) + }; + release.resolve(); + await update; + assert.equal(outdated, true); + tracker.reset(); +}); diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index e3c63cd4..a6170195 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -23,6 +23,7 @@ export class CursorTracker { >(); private readonly updateLock = new Lock(); + private sessionGeneration = 0; private knownRemoteCursors: (ClientCursors & { upToDateness: DocumentUpToDateness; @@ -31,6 +32,10 @@ export class CursorTracker { private lastLocalCursorState: DocumentWithCursors[] = []; private lastLocalCursorStateWithoutDirtyDocuments: DocumentWithCursors[] = []; + private localCursors: Record = {}; + private localCursorDocuments: Record = {}; + private cursorTimer?: ReturnType; + private readonly sendLock = new Lock(); public constructor( private readonly database: Database, @@ -38,9 +43,31 @@ export class CursorTracker { private readonly fileOperations: FileOperations, private readonly fileChangeNotifier: FileChangeNotifier ) { + this.webSocketManager.onWebSocketStatusChanged.add((connected) => { + this.sessionGeneration++; + clearInterval(this.cursorTimer); + this.lastLocalCursorState = []; + this.lastLocalCursorStateWithoutDirtyDocuments = []; + if (!connected) { + this.knownRemoteCursors = []; + this.onRemoteCursorsUpdated.trigger([]); + return; + } + const refresh = () => { + void this.publishLocalCursors(true).catch(() => { + // A concurrent move/deletion can invalidate an editor view; + // the next notification or heartbeat retries the snapshot. + }); + }; + refresh(); + this.cursorTimer = setInterval(refresh, 15_000); + this.cursorTimer.unref?.(); + }); this.webSocketManager.onRemoteCursorsUpdateReceived.add( async (clientCursors) => { + const generation = this.sessionGeneration; await this.updateLock.withLock(async () => { + if (generation !== this.sessionGeneration) return; // The latest message will contain all active clients, so we can delete the ones // from the local list which are no longer active. const allIds = new Set( @@ -51,11 +78,7 @@ export class CursorTracker { allIds.has(c.deviceId) ); - for (const cursor of clientCursors.filter((client) => - client.documentsWithCursors.every( - (doc) => doc.vault_update_id != null - ) - )) { + for (const cursor of clientCursors) { updatedKnownRemoteCursors.push({ ...cursor, upToDateness: @@ -63,17 +86,20 @@ export class CursorTracker { }); } + if (generation !== this.sessionGeneration) return; this.knownRemoteCursors = updatedKnownRemoteCursors; + this.onRemoteCursorsUpdated.trigger( + this.getRelevantAndPruneKnownClientCursors() + ); }); - - this.onRemoteCursorsUpdated.trigger( - this.getRelevantAndPruneKnownClientCursors() - ); } ); this.fileChangeNotifier.onFileChanged.add(async (relativePath) => { + const generation = this.sessionGeneration; + await this.publishLocalCursors(true); const cursors = await this.updateLock.withLock(async () => { + if (generation !== this.sessionGeneration) return []; for (const clientCursor of this.knownRemoteCursors) { if ( clientCursor.documentsWithCursors.some( @@ -88,10 +114,13 @@ export class CursorTracker { await this.getDocumentsUpToDateness(clientCursor); } } - return this.getRelevantAndPruneKnownClientCursors(); + return generation === this.sessionGeneration + ? this.getRelevantAndPruneKnownClientCursors() + : []; }); - this.onRemoteCursorsUpdated.trigger(cursors); + if (generation === this.sessionGeneration) + this.onRemoteCursorsUpdated.trigger(cursors); }); } @@ -101,71 +130,106 @@ export class CursorTracker { public async sendLocalCursorsToServer( documentToCursors: Record ): Promise { - const documentsWithCursors: DocumentWithCursors[] = []; + this.localCursors = structuredClone(documentToCursors); + this.localCursorDocuments = {}; + await this.publishLocalCursors(false); + } - for (const [relativePath, cursors] of Object.entries( - documentToCursors - )) { - const record = - this.database.getLatestDocumentByRelativePath(relativePath); + private async publishLocalCursors(force: boolean): Promise { + const generation = this.sessionGeneration; + await this.sendLock.withLock(async () => { + if (generation !== this.sessionGeneration) return; + const documentsWithCursors: DocumentWithCursors[] = []; - if (!record) { - continue; // Let's wait for the file to be created before sending cursors + for (const [editorPath, cursors] of Object.entries( + this.localCursors + )) { + const id = this.localCursorDocuments[editorPath]; + const record = id + ? this.database.getDocumentByDocumentId(id) + : this.database.getLatestDocumentByRelativePath(editorPath); + + if (!record) { + continue; // Let's wait for the file to be created before sending cursors + } + this.localCursorDocuments[editorPath] = record.documentId; + const { relativePath } = record; + + if (!record.metadata) { + continue; // this is a new document, no need to sync the cursors + } + + documentsWithCursors.push({ + relative_path: relativePath, + document_id: record.documentId, + vault_update_id: record.metadata.parentVersionId, + cursors: cursors.map(({ start, end }) => ({ + start: Math.min(start, end), + end: Math.max(start, end) + })) // the client might send directional selections + }); } - if (!record.metadata) { - continue; // this is a new document, no need to sync the cursors + if ( + !force && + JSON.stringify(this.lastLocalCursorState) === + JSON.stringify(documentsWithCursors) + ) { + // Caching step to avoid reading the edited files all the time + return; + } + this.lastLocalCursorState = structuredClone(documentsWithCursors); + + for (const doc of documentsWithCursors) { + let readContent: Uint8Array; + try { + readContent = await this.fileOperations.read( + doc.relative_path + ); + } catch { + doc.vault_update_id = null; + continue; + } + const digest = await hash(readContent); + const record = this.database.getLatestDocumentByRelativePath( + doc.relative_path + ); + if ( + record?.documentId !== doc.document_id || + record.relativePath !== doc.relative_path || + record.metadata?.parentVersionId !== doc.vault_update_id || + record.metadata.hash !== digest + ) { + doc.vault_update_id = null; + } } - documentsWithCursors.push({ - relative_path: relativePath, - document_id: record.documentId, - vault_update_id: record.metadata.parentVersionId, - cursors: cursors.map(({ start, end }) => ({ - start: Math.min(start, end), - end: Math.max(start, end) - })) // the client might send directional selections - }); - } - - if ( - JSON.stringify(this.lastLocalCursorState) === - JSON.stringify(documentsWithCursors) - ) { - // Caching step to avoid reading the edited files all the time - return; - } - this.lastLocalCursorState = documentsWithCursors; - - for (const doc of documentsWithCursors) { - const readContent = await this.fileOperations.read( - doc.relative_path - ); - const record = this.database.getLatestDocumentByRelativePath( - doc.relative_path - ); - if (record?.metadata?.hash !== (await hash(readContent))) { - doc.vault_update_id = null; + if (generation !== this.sessionGeneration) return; + if ( + !force && + JSON.stringify( + this.lastLocalCursorStateWithoutDirtyDocuments + ) === JSON.stringify(documentsWithCursors) + ) { + return; } - } - if ( - JSON.stringify(this.lastLocalCursorStateWithoutDirtyDocuments) === - JSON.stringify(documentsWithCursors) - ) { - return; - } + this.lastLocalCursorStateWithoutDirtyDocuments = + documentsWithCursors; - this.lastLocalCursorStateWithoutDirtyDocuments = documentsWithCursors; - - this.webSocketManager.updateLocalCursors({ documentsWithCursors }); + this.webSocketManager.updateLocalCursors({ documentsWithCursors }); + }); } public reset(): void { + this.sessionGeneration++; + clearInterval(this.cursorTimer); this.knownRemoteCursors = []; + this.onRemoteCursorsUpdated.trigger([]); this.lastLocalCursorState = []; this.lastLocalCursorStateWithoutDirtyDocuments = []; - this.updateLock.reset(); + // Keep serialization intact while old reads drain. A generation fence + // discards both active and queued work from the retired connection. } private getRelevantAndPruneKnownClientCursors(): MaybeOutdatedClientCursors[] { @@ -245,6 +309,8 @@ export class CursorTracker { private async getDocumentUpToDateness( document: DocumentWithCursors ): Promise { + if (document.vault_update_id === null) + return DocumentUpToDateness.Prior; const record = this.database.getDocumentByDocumentId( document.document_id ); @@ -255,13 +321,11 @@ export class CursorTracker { } if ( - (record.metadata?.parentVersionId ?? 0) < - (document.vault_update_id ?? 0) + (record.metadata?.parentVersionId ?? 0) < document.vault_update_id ) { return DocumentUpToDateness.Later; } else if ( - (document.vault_update_id ?? 0) < - (record.metadata?.parentVersionId ?? 0) + document.vault_update_id < (record.metadata?.parentVersionId ?? 0) ) { // the document of the cursor must be from the past return DocumentUpToDateness.Prior; @@ -276,7 +340,14 @@ export class CursorTracker { return DocumentUpToDateness.Prior; } - return record.metadata?.hash === (await hash(currentContent)) + const contentHash = await hash(currentContent); + const current = this.database.getDocumentByDocumentId( + document.document_id + ); + return current?.metadata?.parentVersionId === + document.vault_update_id && + current.relativePath === record.relativePath && + current.metadata.hash === contentHash ? DocumentUpToDateness.UpToDate : DocumentUpToDateness.Prior; } diff --git a/frontend/sync-client/src/sync-operations/file-manifest.ts b/frontend/sync-client/src/sync-operations/file-manifest.ts new file mode 100644 index 00000000..ad2d393a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/file-manifest.ts @@ -0,0 +1,69 @@ +import type { FileManifestEntries } from "../types/file-manifest-entries"; +import { + allocatePortablePath, + validatePortablePaths +} from "../utils/portable-path"; + +export function validateFileManifest(entries: FileManifestEntries): void { + validatePortablePaths(Object.values(entries)); +} + +export function sameFileManifest( + a: FileManifestEntries, + b: FileManifestEntries +): boolean { + return ( + Object.keys(a).length === Object.keys(b).length && + Object.keys(a).every((id) => a[id] === b[id]) + ); +} + +export function resolvePaths( + proposed: FileManifestEntries, + remote: FileManifestEntries, + protectedIds: readonly string[] = [], + reservedPaths: readonly string[] = [] +): FileManifestEntries { + const result: FileManifestEntries = {}; + for (const id of protectedIds) + if (proposed[id] !== undefined) result[id] = proposed[id]; + const occupied = { ...result }; + for (const [index, path] of reservedPaths.entries()) + if (!Object.values(result).includes(path)) + occupied[`reserved-${index}`] = path; + const ids = Object.keys(proposed).sort( + (a, b) => + Number(remote[b] === proposed[b]) - + Number(remote[a] === proposed[a]) || + (a < b ? -1 : a > b ? 1 : 0) + ); + for (const id of ids.filter( + (candidate) => result[candidate] === undefined + )) { + result[id] = allocatePortablePath(proposed[id], id, occupied); + occupied[id] = result[id]; + } + return result; +} + +export function mergeFileManifests( + base: FileManifestEntries, + local: FileManifestEntries, + remote: FileManifestEntries +): FileManifestEntries { + const proposed: FileManifestEntries = {}; + for (const id of new Set([ + ...Object.keys(base), + ...Object.keys(local), + ...Object.keys(remote) + ])) { + const path = + local[id] === base[id] + ? remote[id] + : remote[id] === base[id] + ? local[id] + : remote[id]; + if (path !== undefined) proposed[id] = path; + } + return resolvePaths(proposed, remote); +} diff --git a/frontend/sync-client/src/sync-operations/local-changes.ts b/frontend/sync-client/src/sync-operations/local-changes.ts new file mode 100644 index 00000000..6022f992 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/local-changes.ts @@ -0,0 +1,82 @@ +import { v4 as uuid } from "uuid"; +import type { EngineState } from "../persistence/database"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import { arePathAliases } from "../utils/portable-path"; + +export type LocalChange = { + changeId?: string; + /** Identities at notification time, before a remote application can reuse paths. */ + identities?: Record; +} & ( + | { type: "move"; oldPath: string; relativePath: string } + | { type: "create"; path: string; documentId?: string } + | { type: "delete"; path: string } + | { type: "update"; path: string } +); + +export function unappliedChanges( + state: EngineState, + changes: readonly LocalChange[] +): LocalChange[] { + for (const change of changes) change.changeId ??= uuid(); + const applied = changes.findIndex( + (change) => change.changeId === state.lastAppliedLocalChangeId + ); + return changes.slice(applied + 1); +} + +export function applyLocalChange( + next: EngineState, + change: LocalChange, + ignored: (path: string) => boolean, + initial?: VaultSnapshot +): void { + if (change.type === "create") { + if ( + ignored(change.path) || + Object.values(next.local).includes(change.path) + ) + return; + const id = (change.documentId ??= + (initial && + Object.keys(initial.fileManifest.entries).find( + (id) => + !next.local[id] && + arePathAliases( + initial.fileManifest.entries[id], + change.path + ) + )) || + uuid()); + next.local[id] = change.path; + next.documents[id] ??= { + materialized: true, + bootstrap: initial?.fileManifest.entries[id] !== undefined + }; + return; + } + const from = change.type === "move" ? change.oldPath : change.path; + change.identities ??= Object.fromEntries( + Object.entries(next.local).filter( + ([id, path]) => + next.documents[id]?.materialized && + (path === from || path.startsWith(from + "/")) + ) + ); + if (change.type === "update") return; + for (const [id, path] of Object.entries(change.identities)) { + if (change.type === "delete") { + if (ignored(path)) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = next.fileManifest.entries[id] ?? null; + } + delete next.local[id]; + } else { + const to = change.relativePath + path.slice(from.length); + for (const [other, otherPath] of Object.entries(next.local)) + if (other !== id && otherPath === to) delete next.local[other]; + next.local[id] = to; + } + } +} diff --git a/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts new file mode 100644 index 00000000..b50d2b8a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { CursorTracker } from "./cursor-tracker"; +import { FileChangeNotifier } from "./file-change-notifier"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { hash } from "../utils/hash"; +import type { Database, DocumentRecord } from "../persistence/database"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { DocumentWithCursors } from "../services/types/DocumentWithCursors"; + +for (const change of ["version", "identity", "path", "delete"] as const) + test(`outgoing cursor is dirty when ${change} changes during its snapshot read`, async () => { + // Equal bytes deliberately defeat a freshness check that compares only hashes. + const content = new TextEncoder().encode("same bytes"); + const digest = await hash(content); + let current: DocumentRecord | undefined = { + documentId: "a", + relativePath: "a.md", + isDeleted: false, + metadata: { parentVersionId: 1, hash: digest } + }; + const sent: { documentsWithCursors: DocumentWithCursors[] }[] = []; + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: (data: { + documentsWithCursors: DocumentWithCursors[]; + }) => sent.push(data) + }; + const files = { + read: async () => { + if (change === "delete") current = undefined; + else if (change === "version") + current!.metadata!.parentVersionId = 2; + else if (change === "identity") + current!.documentId = "replacement"; + else current!.relativePath = "moved.md"; + return content; + } + }; + const tracker = new CursorTracker( + { + getDocumentByDocumentId: (id: string) => + current?.documentId === id ? current : undefined, + getLatestDocumentByRelativePath: (path: string) => + current?.relativePath === path ? current : undefined + } as unknown as Database, + websocket as unknown as WebSocketManager, + files as unknown as FileOperations, + new FileChangeNotifier() + ); + await tracker.sendLocalCursorsToServer({ + "a.md": [{ start: 4, end: 4 }] + }); + assert.equal(sent.length, 1); + assert.equal(sent[0].documentsWithCursors[0].vault_update_id, null); + tracker.reset(); + }); diff --git a/frontend/sync-client/src/sync-operations/scan.ts b/frontend/sync-client/src/sync-operations/scan.ts new file mode 100644 index 00000000..3e743231 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/scan.ts @@ -0,0 +1,152 @@ +import { v4 as uuid } from "uuid"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { EngineState } from "../persistence/database"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import { EMPTY_HASH, hash } from "../utils/hash"; +import { + arePathAliases, + findPathWithSameSpelling +} from "../utils/portable-path"; + +import { + applyLocalChange, + unappliedChanges, + type LocalChange +} from "./local-changes"; +export type { LocalChange } from "./local-changes"; + +interface ScanOptions { + next: EngineState; + guard: () => void; + changes: LocalChange[]; + files: FileOperations; + ignored: (path: string) => boolean; + oversized: (size: number) => boolean; + commit: (next: EngineState) => Promise; +} + +/** Scans are complete or fail. Missing unmaterialized/ignored entries are not deletions. */ +export async function scanLocalFiles( + { + next, + guard, + changes: queuedChanges, + files, + ignored, + oversized, + commit + }: ScanOptions, + initial?: VaultSnapshot +): Promise> { + // Leave the prefix queued until its resulting state is durably committed. + // A persisted change ID makes replay safe after an uncertain save. + const changes = [...queuedChanges]; + const unsyncablePaths = new Set(); + for (const change of unappliedChanges(next, changes)) { + applyLocalChange(next, change, ignored, initial); + next.lastAppliedLocalChangeId = change.changeId; + } + const paths: string[] = []; + const reportedPaths = await files.listFilesRecursively(); + for (const path of reportedPaths) { + // Some filesystems may report a different Unicode spelling after a + // rename. Keep an already-accessible spelling when possible. + const alias = findPathWithSameSpelling(Object.values(next.local), path); + paths.push( + alias && + !reportedPaths.includes(alias) && + (await files.fs.exists(alias)) + ? alias + : path + ); + } + const present = new Set(paths); + const snapshots = new Map(); + for (const path of paths) { + if (ignored(path)) { + unsyncablePaths.add(path); + continue; + } + const info = await files.fs.stat(path); + if (!info || info.kind !== "file") + throw new Error(`File changed during scan: ${path}`); + if (oversized(info.size)) { + unsyncablePaths.add(path); + continue; + } + const snapshot = await files.fs.readSnapshot(path); + if (!snapshot) throw new Error(`File changed during scan: ${path}`); + if (!oversized(snapshot.content.length)) { + snapshots.set(path, { hash: await hash(snapshot.content) }); + } else { + unsyncablePaths.add(path); + } + } + const missing = new Set( + Object.keys(next.local).filter( + (id) => + next.documents[id]?.materialized && + !ignored(next.local[id]) && + !present.has(next.local[id]) + ) + ); + const knownPaths = new Set(Object.values(next.local)); + const unknown = [...snapshots.keys()].filter( + (path) => !knownPaths.has(path) + ); + for (const path of unknown) { + const snapshot = snapshots.get(path)!; + const matches = [...missing].filter( + (id) => next.documents[id]?.observedHash === snapshot.hash + ); + const sameHash = unknown.filter( + (other) => snapshots.get(other)?.hash === snapshot.hash + ); + let id: string | undefined; + if ( + snapshot.hash !== EMPTY_HASH && + matches.length === 1 && + sameHash.length === 1 + ) { + id = matches[0]; + missing.delete(id); + } else if (initial) { + id = Object.keys(initial.fileManifest.entries).find( + (candidate) => + !next.local[candidate] && + arePathAliases( + initial.fileManifest.entries[candidate], + path + ) + ); + } + id ??= uuid(); + next.local[id] = path; + next.documents[id] ??= { + materialized: true, + bootstrap: initial?.fileManifest.entries[id] !== undefined + }; + } + for (const id of missing) delete next.local[id]; + for (const [id, path] of Object.entries(next.local)) { + if (unsyncablePaths.has(path) || ignored(path)) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = next.fileManifest.entries[id] ?? null; + } + const snapshot = snapshots.get(path); + if (snapshot) { + next.documents[id] ??= { materialized: true }; + if (!next.documents[id].materialized && !next.documents[id].base) + next.documents[id].bootstrap = true; + next.documents[id].materialized = true; + next.documents[id].observedHash = snapshot.hash; + } + } + next.protectedPaths = [...unsyncablePaths]; + guard(); + await commit(next); + queuedChanges.splice(0, changes.length); + guard(); + return unsyncablePaths; +} diff --git a/frontend/sync-client/src/sync-operations/syncer.test.ts b/frontend/sync-client/src/sync-operations/syncer.test.ts new file mode 100644 index 00000000..e3c11360 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/syncer.test.ts @@ -0,0 +1,351 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import assert from "node:assert"; +import { afterEach, beforeEach, describe, it, mock } from "node:test"; +import type { Database } from "../persistence/database"; +import { + DEFAULT_SETTINGS, + type Settings, + type SyncSettings +} from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { SyncService } from "../services/sync-service"; +import type { ServerConfig } from "../services/server-config"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; +import type { EventBatch } from "../services/types/EventBatch"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { Syncer } from "./syncer"; +import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { toStoredSnapshot } from "./content"; +import { undiff } from "reconcile-text"; + +function createSyncer(overrides: Partial = {}): { + syncer: Syncer; + runCount: () => number; +} { + const settings = { + getSettings: () => ({ + ...DEFAULT_SETTINGS, + ...overrides + }) + } as unknown as Settings; + const database = { + state: {} + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners<() => unknown>() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + {} as Logger, + database, + settings, + {} as SyncService, + websocket, + {} as FileOperations, + {} as ServerConfig, + {} as SyncHistory, + new FixedSizeDocumentCache(1024) + ); + let runs = 0; + const internals = syncer as unknown as { + dirty: boolean; + run: () => Promise; + }; + internals.run = async (): Promise => { + runs++; + internals.dirty = false; + }; + + return { syncer, runCount: () => runs }; +} + +describe("Syncer interval", () => { + beforeEach(() => { + mock.timers.enable({ apis: ["setTimeout"] }); + }); + + afterEach(() => { + mock.timers.reset(); + }); + + it("does not schedule a timer when syncIntervalMs is unset", async () => { + const { syncer, runCount } = createSyncer({ + syncIntervalMs: undefined + }); + + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("keeps periodic reconciliation opt-in by default", async () => { + assert.strictEqual(DEFAULT_SETTINGS.syncIntervalMs, undefined); + const { syncer, runCount } = createSyncer(); + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("does not poll when explicitly disabled with zero", async () => { + const { syncer, runCount } = createSyncer({ syncIntervalMs: 0 }); + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("wakes after the configured sync interval", async () => { + const { syncer, runCount } = createSyncer({ syncIntervalMs: 25 }); + + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(24); + assert.strictEqual(runCount(), 1); + + mock.timers.tick(1); + assert.strictEqual(runCount(), 2); + await syncer.waitUntilFinished(); + await syncer.stop(); + }); +}); + +describe("Syncer per-file state and event delivery", () => { + const head = { + vaultUpdateId: 1, + documentId: "document", + updatedDate: "2026-01-01T00:00:00Z", + userId: "user", + deviceId: "device", + contentSize: 4 + }; + + function fixture() { + const database = { + state: { + initialized: true, + local: { document: "note.md" }, + documents: { + document: { + materialized: true, + observedHash: "hash", + base: { ...head, hash: "hash" } + } + }, + fileManifest: { + fileManifestId: 1, + entries: { document: "note.md" } + }, + remoteHeads: { document: head }, + lastSeenUpdateId: 0 + }, + getLatestDocumentByRelativePath: (path: string) => + path === "note.md" + ? { + documentId: "document", + relativePath: path, + isDeleted: false + } + : undefined + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => Promise + >() + } as unknown as WebSocketManager; + let httpEventsCalls = 0; + const service = { + events: async () => { + httpEventsCalls++; + return { headEventId: 0, events: [] }; + } + } as unknown as SyncService; + const syncer = new Syncer( + "device", + {} as Logger, + database, + { + getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) + } as Settings, + service, + websocket, + {} as FileOperations, + {} as ServerConfig, + {} as SyncHistory, + new FixedSizeDocumentCache(1024) + ); + (syncer as unknown as { hasScanned: boolean }).hasScanned = true; + return { + database, + websocket, + syncer, + httpEventsCalls: () => httpEventsCalls + }; + } + + it("reports only a genuinely current path as up to date", () => { + const { syncer } = fixture(); + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true); + assert.strictEqual(syncer.isDocumentUpToDate("missing.md"), false); + + const internals = syncer as unknown as { + dirty: boolean; + unsyncablePaths: Set; + }; + internals.dirty = true; + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true); + internals.unsyncablePaths.add("note.md"); + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), false); + }); + + it("consumes a complete WebSocket event batch without fetching it again", async () => { + const { websocket, syncer, httpEventsCalls } = fixture(); + const batch: EventBatch = { + headEventId: 1, + events: [ + { + eventId: 1, + requestId: "00000000-0000-0000-0000-000000000001", + type: "fileManifest", + fileManifest: { fileManifestId: 1, entries: {} } + } + ] + }; + await websocket.onRemoteVaultUpdateReceived.triggerAsync(batch); + + const selected = await ( + syncer as unknown as { nextEventBatch: () => Promise } + ).nextEventBatch(); + assert.deepEqual(selected, batch); + assert.strictEqual(httpEventsCalls(), 0); + }); + + it("projects content responses to metadata before persistence", () => { + const { syncer } = fixture(); + const projected = ( + syncer as unknown as { + documentHead: (value: typeof head) => Record; + } + ).documentHead({ ...head, contentBase64: "secret" } as typeof head); + assert.deepEqual(projected, head); + assert.strictEqual("contentBase64" in projected, false); + }); +}); + +describe("Syncer diff uploads", () => { + const encoder = new TextEncoder(); + + function fixture(cacheSize = 1024): { + syncer: Syncer; + parent: Uint8Array; + } { + const cache = new FixedSizeDocumentCache(cacheSize); + const parent = encoder.encode("one two three\n"); + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners<() => unknown>() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + {} as Logger, + { state: {} } as Database, + { + getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) + } as Settings, + { + getDocumentVersionContent: async () => parent + } as unknown as SyncService, + websocket, + {} as FileOperations, + { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig, + {} as SyncHistory, + cache + ); + return { syncer, parent }; + } + + it("sends a diff when the fetched parent is still cached", async () => { + const { syncer, parent } = fixture(); + const internals = syncer as unknown as { + remoteSnapshot: ( + head: { + vaultUpdateId: number; + documentId: string; + }, + path: string + ) => Promise; + pushContent: ( + path: string, + parentVersionId: number, + snapshot: Awaited> + ) => Promise< + | { type: "Diff"; value: (number | string)[] } + | { type: "Snapshot"; value: string } + >; + }; + await internals.remoteSnapshot( + { vaultUpdateId: 7, documentId: "document" }, + "note.md" + ); + const changed = "one changed three\n"; + const payload = await internals.pushContent( + "note.md", + 7, + await toStoredSnapshot({ content: encoder.encode(changed) }) + ); + + assert.equal(payload.type, "Diff"); + if (payload.type === "Diff") { + assert.equal( + undiff(new TextDecoder().decode(parent), payload.value), + changed + ); + } + }); + + it("falls back to a snapshot when the parent cannot fit in the cache", async () => { + const { syncer } = fixture(0); + const internals = syncer as unknown as { + remoteSnapshot: ( + head: { + vaultUpdateId: number; + documentId: string; + }, + path: string + ) => Promise; + pushContent: ( + path: string, + parentVersionId: number, + snapshot: Awaited> + ) => Promise<{ type: string; value: unknown }>; + }; + await internals.remoteSnapshot( + { vaultUpdateId: 7, documentId: "document" }, + "note.md" + ); + const payload = await internals.pushContent( + "note.md", + 7, + await toStoredSnapshot({ content: encoder.encode("changed") }) + ); + + assert.equal(payload.type, "Snapshot"); + }); +}); diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 71dedd85..3a9bc83a 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,522 +1,1396 @@ +import type { SyncHistory } from "../tracing/sync-history"; +import { SyncStatus, SyncType } from "../tracing/sync-history"; +import { v4 as uuid } from "uuid"; +import { base64ToBytes } from "byte-base64"; +import { diff } from "reconcile-text"; import type { Database, - DocumentId, - DocumentRecord, - RelativePath + EngineState, + StoredSnapshot } from "../persistence/database"; -import type { SyncService } from "../services/sync-service"; -import type { Logger } from "../tracing/logger"; -import PQueue from "p-queue"; -import { hash } from "../utils/hash"; -import { v4 as uuidv4 } from "uuid"; +import { emptyState } from "../persistence/database"; +import type { + FileOperations, + FileWrite +} from "../file-operations/file-operations"; import type { Settings } from "../persistence/settings"; -import type { FileOperations } from "../file-operations/file-operations"; -import { findMatchingFile } from "../utils/find-matching-file"; -import type { UnrestrictedSyncer } from "./unrestricted-syncer"; -import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "../services/sync-reset-error"; -import { Locks } from "../utils/data-structures/locks"; -import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate"; +import type { SyncService } from "../services/sync-service"; +import type { ServerConfig } from "../services/server-config"; import type { WebSocketManager } from "../services/websocket-manager"; -import type { WebSocketClientMessage } from "../services/types/WebSocketClientMessage"; -import { awaitAll } from "../utils/await-all"; +import type { DocumentVersion } from "../services/types/DocumentVersion"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import type { EventBatch } from "../services/types/EventBatch"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import type { PushContent } from "../services/types/PushContent"; +import type { Logger } from "../tracing/logger"; import { EventListeners } from "../utils/data-structures/event-listeners"; +import { globsToRegexes } from "../utils/globs-to-regexes"; +import { mergeContent, toStoredSnapshot } from "./content"; +import { scanLocalFiles, type LocalChange } from "./scan"; +import { + mergeFileManifests, + resolvePaths, + sameFileManifest, + validateFileManifest +} from "./file-manifest"; +import { removeFromArray } from "../utils/remove-from-array"; +import { isInternalPath } from "../utils/portable-path"; +import { isBinary } from "../utils/is-binary"; +import { isFileTypeMergable } from "../utils/is-file-type-mergable"; +import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { + AuthenticationError, + LocalChangesDuringReconciliation, + PermanentSyncError, + ServerVersionMismatchError, + SyncResetError, + ServerHistoryChangedError +} from "../errors/errors"; export class Syncer { + public readonly onReadyForEvents = new EventListeners<() => void>(); + public readonly onServerHistoryChanged = new EventListeners<() => void>(); public readonly onRemainingOperationsCountChanged = new EventListeners< - (remainingOperations: number) => unknown + (count: number) => unknown >(); - private readonly remoteDocumentsLock: Locks; - - // FIFO to limit the number of concurrent sync operations - private readonly syncQueue: PQueue; - - private _isFirstSyncComplete = false; - private runningScheduleSyncForOfflineChanges: Promise | undefined; - private previousRemainingOperationsCount = 0; + private running?: Promise; + private timer?: ReturnType; + private stopped = true; + private dirty = false; + private localGeneration = 0; + private failure?: unknown; + private readonly changes: LocalChange[]; + private notificationSaving: Promise = Promise.resolve(); + private notificationSaveFailed = false; + private readonly pendingPaths = new Set(); + private unsyncablePaths = new Set(); + private eventBatches: EventBatch[] = []; + private hasScanned = false; public constructor( - private readonly deviceId: string, + deviceId: string, private readonly logger: Logger, private readonly database: Database, private readonly settings: Settings, - private readonly syncService: SyncService, - private readonly webSocketManager: WebSocketManager, - private readonly operations: FileOperations, - private readonly internalSyncer: UnrestrictedSyncer + private readonly service: SyncService, + websocket: WebSocketManager, + private readonly files: FileOperations, + private readonly serverConfig: ServerConfig, + private readonly history: SyncHistory, + private readonly contentCache: FixedSizeDocumentCache, + private readonly localChangeStorage: { + entries: LocalChange[]; + save: (entries: LocalChange[]) => Promise; + } = { entries: [], save: async () => {} } ) { - this.syncQueue = new PQueue({ - concurrency: settings.getSettings().syncConcurrency - }); - - this.remoteDocumentsLock = new Locks(this.logger); - - settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) { - this.syncQueue.concurrency = newSettings.syncConcurrency; + this.changes = localChangeStorage.entries; + websocket.onWebSocketStatusChanged.add((connected) => { + if (connected) { + websocket.sendHandshakeMessage({ + type: "handshake", + token: settings.getSettings().token, + deviceId, + lastSeenVaultUpdateId: database.state.lastSeenUpdateId + }); } + // Failed upgrades and lost transports still need HTTP catchup, + // including when the optional polling interval is disabled. + this.wake(); }); - - this.syncQueue.on("active", () => { - if (this.previousRemainingOperationsCount !== this.syncQueue.size) { - this.previousRemainingOperationsCount = this.syncQueue.size; - this.onRemainingOperationsCountChanged.trigger( - this.syncQueue.size - ); - } + websocket.onRemoteVaultUpdateReceived.add(async (batch) => { + this.eventBatches.push(batch); + // Notifications are only hints; HTTP replay recovers discarded pages. + if (this.eventBatches.length > 4) this.eventBatches.shift(); + this.wake(); }); - - this.webSocketManager.onWebSocketStatusChanged.add((isConnected) => { - if (isConnected) { - // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message - this.sendHandshakeMessage(); - } - }); - this.webSocketManager.onRemoteVaultUpdateReceived.add( - this.syncRemotelyUpdatedFile.bind(this) - ); } public get isFirstSyncComplete(): boolean { - return this._isFirstSyncComplete; + return this.database.state.initialized; } - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { + public get hasLocalChanges(): boolean { + return this.changes.length > 0; + } + + public get isBusy(): boolean { + return ( + !!this.running || + this.dirty || + !!this.database.state.pending || + !!this.database.state.application || + !!this.failure || + !!this.rejectionMessage + ); + } + + public isDocumentUpToDate(path: string): boolean { if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === false + !this.database.state.initialized || + !this.hasScanned || + this.pendingPaths.has(path) || + this.unsyncablePaths.has(path) || + this.ignored(path) ) { - this.logger.debug( - `Document ${relativePath} already exists in the database, skipping` - ); + return false; + } + + const record = this.database.getLatestDocumentByRelativePath(path); + if (!record) return false; + const document = this.database.state.documents[record.documentId]; + if ( + !document?.materialized || + !document.base || + document.bootstrap || + document.observedHash !== document.base.hash || + this.database.state.fileManifest.entries[record.documentId] !== path + ) { + return false; + } + + const remote = this.database.state.remoteHeads[record.documentId]; + if (remote && remote.vaultUpdateId > document.base.vaultUpdateId) + return false; + + const { pending } = this.database.state; + if ( + pending?.type === "content" && + pending.documentId === record.documentId + ) { + return false; + } + if ( + pending?.type === "fileManifest" && + pending.request.entries[record.documentId] !== + this.database.state.fileManifest.entries[record.documentId] + ) { + return false; + } + + return !this.database.state.application?.steps.some( + (step) => + step.documentId === record.documentId && + step.phase !== "installed" + ); + } + + public start(): void { + this.stopped = false; + this.failure = undefined; + this.hasScanned = false; + this.wake(); + } + + public async stop(): Promise { + this.stopped = true; + clearTimeout(this.timer); + await this.running; + } + + public wake(): void { + if (this.stopped) { return; } - const [promise, resolve, reject] = createPromise(); + this.dirty = true; - const id = uuidv4(); - const document = this.database.createNewPendingDocument( - id, - relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - } - - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { - // This is must be a consequence of us deleting a file because of a remote update - // which triggered a local delete, so we don't need to do anything here. - this.logger.debug( - `Document ${relativePath} has already been markes as deleted, skipping` - ); + if (this.running) { return; } - // We have to have a record of the delete in case there's an in-flight update for the same - // document which finishes after the delete has succeeded and would introduce a phantom metadata record. - this.database.delete(relativePath); + clearTimeout(this.timer); - const [promise, resolve, reject] = createPromise(); - - const document = await this.database.getResolvedDocumentByRelativePath( - relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document) + this.running = this.run().finally(() => { + this.running = undefined; + this.onRemainingOperationsCountChanged.trigger( + this.failure || + this.database.state.pending || + this.rejectionMessage + ? 1 + : 0 ); - resolve(); + if (this.stopped) { + return; + } - this.database.removeDocument(document); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); + if (this.dirty && !this.failure) { + this.wake(); + return; + } + + if ( + this.failure instanceof PermanentSyncError || + this.failure instanceof AuthenticationError || + this.failure instanceof ServerVersionMismatchError + ) { + return; + } + + const intervalMs = + this.failure === undefined + ? this.settings.getSettings().syncIntervalMs + : this.settings.getSettings().networkRetryIntervalMs; + if (intervalMs === undefined || intervalMs === 0) { + return; + } + + this.timer = setTimeout(() => { + this.wake(); + }, intervalMs); + }); + } + + public async waitUntilFinished(): Promise { + while (this.running) { + await this.running; } + if (this.failure) { + throw this.failure; + } + if (this.rejectionMessage) + throw new PermanentSyncError(this.rejectionMessage); + } + + public async syncLocallyCreatedFile(_path: string): Promise { + this.localGeneration++; + this.pendingPaths.add(_path); + await this.recordLocalChange({ type: "create", path: _path }); + } + + public async syncLocallyDeletedFile(_path: string): Promise { + this.localGeneration++; + this.pendingPaths.add(_path); + await this.recordLocalChange({ type: "delete", path: _path }); } public async syncLocallyUpdatedFile({ oldPath, relativePath }: { - oldPath?: RelativePath; - relativePath: RelativePath; + oldPath?: string; + relativePath: string; }): Promise { - if (oldPath !== undefined) { - // We might have moved the document in the database before calling this method, - // in that case, we mustn't move it again. - if ( - this.database.getLatestDocumentByRelativePath(relativePath) === - undefined || - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { - if (oldPath === relativePath) { - throw new Error( - `Old path and new path are the same: ${oldPath}` - ); - } - - this.database.move(oldPath, relativePath); - } - } - - let document = - this.database.getLatestDocumentByRelativePath(relativePath); - + // Content snapshots are re-read during journal staging. Only namespace + // notifications invalidate an identity plan; keystrokes must not starve + // a complete scan of unrelated documents. + if (oldPath !== undefined && oldPath !== "") this.localGeneration++; + this.pendingPaths.add(relativePath); + if (oldPath) this.pendingPaths.add(oldPath); if ( - oldPath !== undefined && - document?.metadata?.remoteRelativePath === relativePath + oldPath && + !isInternalPath(oldPath) && + !isInternalPath(relativePath) ) { - this.logger.debug( - `Document ${relativePath} has been moved as a result of a remote update, skipping sync` - ); - return; + await this.recordLocalChange({ + type: "move", + oldPath, + relativePath + }); + } else if (!isInternalPath(relativePath)) { + await this.recordLocalChange({ + type: "update", + path: relativePath + }); } + this.wake(); + } - if (document === undefined) { - this.logger.debug( - `Cannot find document ${relativePath} in the database, skipping` - ); - return; - } - - if (document.isDeleted) { - this.logger.debug( - `Document ${relativePath} has been deleted locally, skipping` - ); - return; - } - - const [promise, resolve, reject] = createPromise(); - - document = await this.database.getResolvedDocumentByRelativePath( - relativePath, - promise - ); + /** Local recovery must run even when network sync is disabled. Caller stops + * the engine first and holds the client's lifecycle lock. */ + public async recoverLocalState(): Promise { + await this.flushLocalChanges(); + await this.database.recoverPersistence(); + const { application } = this.database.state; + await this.files.recover(); + if (application) this.recordRecoveredApplication(application.steps); + } + private async recordLocalChange(change: LocalChange): Promise { + change.changeId = uuid(); + const previous = [...this.changes]; + this.changes.push(change); + const saving = this.notificationSaving.then(async () => { + await this.files.captureChange(change, previous); + await this.localChangeStorage.save(structuredClone(this.changes)); + this.notificationSaveFailed = false; + }); + this.notificationSaving = saving.catch(() => { + this.notificationSaveFailed = true; + }); try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ - oldPath, - document - }) - ); - - resolve(); - } catch (e) { - reject(e); + await saving; } finally { - this.database.removeDocumentPromise(promise); + this.wake(); } } - public async scheduleSyncForOfflineChanges(): Promise { - if (this.runningScheduleSyncForOfflineChanges !== undefined) { - this.logger.debug("Uploading local changes is already in progress"); - return this.runningScheduleSyncForOfflineChanges; + public async flushLocalChanges(): Promise { + await this.notificationSaving; + if (this.notificationSaveFailed) { + await this.localChangeStorage.save(structuredClone(this.changes)); + this.notificationSaveFailed = false; } + } + private async run(): Promise { + this.onRemainingOperationsCountChanged.trigger(1); + this.failure = undefined; try { - this.runningScheduleSyncForOfflineChanges = - this.internalScheduleSyncForOfflineChanges(); - await this.runningScheduleSyncForOfflineChanges; - this.logger.info(`All local changes have been applied remotely`); - } catch (e) { - if (e instanceof SyncResetError) { - this.logger.info( - "Failed to apply local changes remotely due to a reset" - ); + await this.recoverLocalState(); + if ( + this.database.state.historyRecovery || + (this.database.state.initialized && + this.service.verifiesHistory && + !this.service.hasHistoryCheckpoint) + ) + await this.recoverServerHistory(); + await this.serverConfig.initialize(); + if (!this.database.state.initialized) { + await this.bootstrap(); + } + if (!this.stopped) this.onReadyForEvents.trigger(); + do { + this.dirty = false; + // An unknown accepted request must be resolved before observing a + // newer base, otherwise we can merge our own edit a second time. + if (this.database.state.pending) { + await this.finishPending(); + } + await this.scan(); + const batch = await this.nextEventBatch(); + await this.incorporateEventBatch(batch); + if (this.database.state.eventReplay) { + this.dirty = true; + continue; + } + await this.scan(); + // Previously excluded content becomes eligible after a setting + // change. Its durable obligation survives advancing the cursor. + if (Object.keys(this.database.state.excluded ?? {}).length) + await this.incorporateFileManifest( + this.database.state.fileManifest + ); + for (const head of Object.values( + this.database.state.remoteHeads + )) { + const doc = this.database.state.documents[head.documentId]; + if ( + this.database.state.local[head.documentId] !== + undefined && + (!doc?.materialized || + !doc.base || + doc.bootstrap || + doc.base.vaultUpdateId < head.vaultUpdateId) + ) { + await this.incorporateContent(head); + } + } + if (await this.preparePush()) { + await this.finishPending(); + this.dirty = true; + } + if ( + this.eventBatches.some( + (batch) => + batch.headEventId > + this.database.state.lastSeenUpdateId + ) + ) + this.dirty = true; + } while (this.dirty && !this.stopped); + if (!this.dirty && !this.database.state.pending) + this.pendingPaths.clear(); + } catch (error) { + if (error instanceof ServerHistoryChangedError && !this.stopped) { + try { + await this.recoverServerHistory(); + this.dirty = true; + return; + } catch (recoveryError) { + error = recoveryError; + } + } + if (error instanceof LocalChangesDuringReconciliation) { + this.dirty = true; return; } - this.logger.error( - `Not all local changes have been applied remotely: ${e}` - ); - throw e; - } finally { - this.runningScheduleSyncForOfflineChanges = undefined; - } - } - - public async waitUntilFinished(): Promise { - await this.runningScheduleSyncForOfflineChanges; - await this.syncQueue.onIdle(); // Wait for queue to be empty and running tasks to finish - } - - public async syncRemotelyUpdatedFile( - message: WebSocketVaultUpdate - ): Promise { - try { - const handlerPromise = awaitAll( - message.documents.map(async (document) => - this.internalSyncRemotelyUpdatedFile(document) - ) - ); - - await handlerPromise; - - if (message.isInitialSync && message.documents.length > 0) { - this.database.setLastSeenUpdateId( - message.documents - .map((document) => document.vaultUpdateId) - .reduce((a, b) => Math.max(a, b)) - ); + if (!(error instanceof SyncResetError && this.stopped)) { + this.failure = error; + this.serverConfig.reset(); + this.logger.error(`Sync paused for retry: ${String(error)}`); } - - this._isFirstSyncComplete = true; - } catch (e) { - this.logger.error(`Failed to sync remotely updated file: ${e}`); } } - public reset(): void { - this._isFirstSyncComplete = false; - this.syncQueue.clear(); - this.remoteDocumentsLock.reset(); - this.runningScheduleSyncForOfflineChanges = undefined; + private async recoverServerHistory(): Promise { + this.onServerHistoryChanged.trigger(); + this.contentCache.reset(); + this.eventBatches = []; + this.hasScanned = false; + if (!this.database.state.historyRecovery) { + await this.scan(); + const guard = this.localGuard(); + const previous = this.database.state; + for (const pending of [ + previous.pending, + ...(previous.unconfirmed ?? []) + ]) { + if (pending?.type === "content") + await this.files.retainSnapshot( + pending.request.requestId, + pending.documentId, + previous.local[pending.documentId], + pending.snapshot + ); + } + const next = emptyState(previous.vaultKey); + next.historyRecovery = true; + next.lastAppliedLocalChangeId = previous.lastAppliedLocalChangeId; + next.protectedPaths = previous.protectedPaths; + for (const [id, path] of Object.entries(previous.local)) { + const info = await this.files.fs.stat(path); + if (info?.kind !== "file") continue; + const doc = previous.documents[id]; + if (this.ignored(path) || this.oversized(info.size)) { + next.local[id] = path; + next.documents[id] = { + materialized: true, + bootstrap: true + }; + (next.excluded ??= {})[id] = + previous.fileManifest.entries[id] ?? null; + continue; + } + const snapshot = await this.files.snapshot(path); + if (!snapshot) throw new LocalChangesDuringReconciliation(); + // An unsent edit from the abandoned history is a separate local + // document. It must never be diffed against a reused version ID. + const clean = doc?.base?.hash === snapshot.hash; + const localId = doc?.base && !clean ? uuid() : id; + next.local[localId] = path; + next.documents[localId] = { + materialized: true, + observedHash: snapshot.hash, + recoveryBase: clean ? snapshot : doc?.recoveryBase + }; + } + guard(); + // Persist the reset BEFORE releasing the old checkpoint. After a + // crash no old request can be retried against the replacement log. + await this.database.commit(next); + } + await this.service.resetHistory(); + const next = this.next(); + delete next.historyRecovery; + await this.database.commit(next); } - private sendHandshakeMessage(): void { - const message: WebSocketClientMessage = { - type: "handshake", - deviceId: this.deviceId, - token: this.settings.getSettings().token, - lastSeenVaultUpdateId: this.database.getLastSeenUpdateId() + private localGuard(): () => void { + const generation = this.localGeneration; + return () => { + if (generation !== this.localGeneration) { + throw new LocalChangesDuringReconciliation(); + } }; - this.webSocketManager.sendHandshakeMessage(message); } - private async internalSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent + private ignored(path: string): boolean { + return ( + isInternalPath(path) || + globsToRegexes( + this.settings.getSettings().ignorePatterns, + this.logger + ).some((pattern) => pattern.test(path)) + ); + } + + private oversized(size: number): boolean { + return size > this.settings.getSettings().maxFileSizeMB * 1024 * 1024; + } + + private async cacheContent( + updateId: number, + path: string | undefined, + content: Uint8Array ): Promise { - let document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - if (document === undefined) { - // Let's avoid the same documents getting created in parallel multiple times. - // There might be multiple tasks waiting for the lock - return this.remoteDocumentsLock.withLock( - remoteVersion.documentId, - async () => { - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile` - if (document === undefined) { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion - ) - ); - } else { - const [promise, resolve, reject] = createPromise(); - - document = - await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - } - - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); - } - ); + if ( + path !== undefined && + !isBinary(content) && + isFileTypeMergable( + path, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) + ) { + this.contentCache.put(updateId, content); } - - // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile` - const [promise, resolve, reject] = createPromise(); - - document = await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); } - private async internalScheduleSyncForOfflineChanges(): Promise { - await this.createFakeDocumentsFromRemoteState(); + private async pushContent( + path: string, + parentVersionId: number | undefined, + snapshot: StoredSnapshot + ): Promise { + const content = base64ToBytes(snapshot.contentBase64); + const parent = + parentVersionId === undefined + ? undefined + : this.contentCache.get(parentVersionId); + if ( + parent !== undefined && + !isBinary(content) && + !isBinary(parent) && + isFileTypeMergable( + path, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) + ) { + const decode = (bytes: Uint8Array): string => + new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true + }).decode(bytes); + const value = diff(decode(parent), decode(content)); + // Match the server's work bound; large edit sets remain syncable. + if (value.length <= 10_000) return { type: "Diff", value }; + } - const allLocalFiles = await this.operations.listFilesRecursively(); - this.logger.info( - `Scheduling sync for ${allLocalFiles.length} local files` + return { type: "Snapshot", value: snapshot.contentBase64 }; + } + + private next(): EngineState { + const { application: _, ...next } = structuredClone( + this.database.state ); + return next; + } - let locallyPossiblyDeletedFiles: DocumentRecord[] = []; + private async bootstrap(): Promise { + let vaultSnapshot = this.database.state.bootstrap; + if (!vaultSnapshot) { + vaultSnapshot = await this.service.vaultSnapshot(); + validateFileManifest(vaultSnapshot.fileManifest.entries); + await this.database.commit({ + ...this.next(), + bootstrap: vaultSnapshot + }); + } + await this.scan(vaultSnapshot); + await this.incorporateFileManifest( + vaultSnapshot.fileManifest, + vaultSnapshot.headEventId, + vaultSnapshot + ); + } - for (const document of this.database.resolvedDocuments) { + private async scan(initial?: VaultSnapshot): Promise { + await this.flushLocalChanges(); + this.unsyncablePaths = await scanLocalFiles( + { + next: this.next(), + guard: this.localGuard(), + changes: this.changes, + files: this.files, + ignored: (path) => this.ignored(path), + oversized: (size) => this.oversized(size), + commit: async (next) => this.database.commit(next) + }, + initial + ); + this.hasScanned = true; + await this.localChangeStorage.save(structuredClone(this.changes)); + } + + private async incorporateEventBatch(batch: EventBatch): Promise { + const replay = this.database.state.eventReplay; + const after = replay?.after ?? this.database.state.lastSeenUpdateId; + let cursor = after; + let manifest = replay?.manifest ?? this.database.state.fileManifest; + const heads = { ...(replay?.heads ?? this.database.state.remoteHeads) }; + const receipts = [...(replay?.receipts ?? [])]; + for (const event of batch.events) { + if (event.eventId <= cursor) continue; + if (event.eventId !== cursor + 1) + throw new PermanentSyncError( + "Non-contiguous event history; refusing to skip changes" + ); + cursor = event.eventId; if ( - !document.isDeleted && - !(await this.operations.exists(document.relativePath)) + this.database.state.unconfirmed?.some( + (pending) => pending.request.requestId === event.requestId + ) === true + ) + receipts.push(event); + if (event.type === "fileManifest") { + if (event.fileManifest.fileManifestId > manifest.fileManifestId) + manifest = event.fileManifest; + } else if ( + !heads[event.document.documentId] || + heads[event.document.documentId].vaultUpdateId < + event.document.vaultUpdateId ) { - locallyPossiblyDeletedFiles.push(document); + heads[event.document.documentId] = this.documentHead( + event.document + ); } } - - await awaitAll( - allLocalFiles.map(async (relativePath) => { - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.metadata !== undefined - ) { - this.logger.debug( - `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` + const end = batch.endEventId ?? batch.headEventId; + if ( + end !== cursor || + end > batch.headEventId || + (end === after && end < batch.headEventId) + ) + throw new PermanentSyncError("Incomplete event batch"); + if (end < batch.headEventId) { + await this.database.commit({ + ...this.next(), + eventReplay: { + after: end, + manifest, + heads, + receipts + } + }); + return; + } + if (cursor === this.database.state.lastSeenUpdateId) return; + // Recover our submitted bases before folding later remote edits. A + // permanent retry failure cannot cancel an earlier in-flight attempt. + const next = this.next(); + let acknowledged = false; + for (const event of receipts) { + const pending = next.unconfirmed?.find( + (candidate) => candidate.request.requestId === event.requestId + ); + if (!pending) continue; + if (event.type === "content" && pending.type === "content") { + const doc = next.documents[pending.documentId]; + if (!doc.base || doc.base.vaultUpdateId < event.eventId) { + doc.base = { + ...event.document, + hash: pending.snapshot.hash + }; + doc.bootstrap = false; + delete doc.rejected; + await this.cacheContent( + event.eventId, + next.local[pending.documentId], + base64ToBytes(pending.snapshot.contentBase64) ); - - return this.syncLocallyUpdatedFile({ - relativePath - }); } - - // Perhaps the file has been moved; let's check by looking at the deleted files - const contentHash = await this.syncQueue.add(async () => { - const contentBytes = - await this.operations.read(relativePath); // this can throw FileNotFoundError - return hash(contentBytes); - }); - - if (contentHash == undefined) { - // The file was deleted before we had a chance to read it, no need to sync it here - return; + } else if ( + event.type === "fileManifest" && + pending.type === "fileManifest" + ) { + if (next.fileManifest.fileManifestId < event.eventId) { + next.fileManifest = event.fileManifest; + delete next.rejectedManifest; } - - const originalFile = findMatchingFile( - contentHash, - locallyPossiblyDeletedFiles + } else + throw new PermanentSyncError( + "Request receipt type does not match submission" ); - if (originalFile !== undefined) { - // `originalFile` hasn't been deleted but it got moved instead - /* eslint-disable no-restricted-syntax -- Comparing by property, not direct equality */ - locallyPossiblyDeletedFiles = - locallyPossiblyDeletedFiles.filter( - (item) => - item.relativePath !== originalFile.relativePath - ); - /* eslint-enable no-restricted-syntax */ - - this.logger.debug( - `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` - ); - - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyUpdatedFile({ - oldPath: originalFile.relativePath, - relativePath - }); - } - - this.logger.debug( - `Document ${relativePath} not found in database, scheduling sync to create it` - ); - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyCreatedFile(relativePath); - }) - ); - - // this has to happen strictly after the previous awaitAll, as that one - // might have removed some of the documents from the list - await awaitAll( - locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { - this.logger.debug( - `Document ${relativePath} has been deleted locally, scheduling sync to delete it` - ); - - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyDeletedFile(relativePath); - }) + if (next.unconfirmed) removeFromArray(next.unconfirmed, pending); + acknowledged = true; + } + if (acknowledged) await this.database.commit(next); + // Three-way merging compares snapshots. Replaying intermediate remote + // edits can discard a local edit even when the remote ends at its base. + await this.incorporateFileManifest( + manifest, + cursor, + undefined, + false, + heads ); } - /** - * Create fake documents in the database for all files that are present locally - * and also exist remotely. This will stop the subequent syncs from duplicating - * the documents by creating the same documents from multiple clients. - */ - private async createFakeDocumentsFromRemoteState(): Promise { - if (this.database.getHasInitialSyncCompleted()) { + private async nextEventBatch(): Promise { + const after = + this.database.state.eventReplay?.after ?? + this.database.state.lastSeenUpdateId; + if (this.service.verifiesHistory) { + this.eventBatches = []; + return this.service.events(after); + } + this.eventBatches = this.eventBatches.filter( + (batch) => (batch.endEventId ?? batch.headEventId) > after + ); + const queued = this.eventBatches.findIndex((batch) => + batch.events.some((event) => event.eventId === after + 1) + ); + if (queued !== -1) return this.eventBatches.splice(queued, 1)[0]; + return this.service.events(after); + } + + private documentHead( + version: DocumentVersionWithoutContent + ): DocumentVersionWithoutContent { + return { + vaultUpdateId: version.vaultUpdateId, + documentId: version.documentId, + updatedDate: version.updatedDate, + userId: version.userId, + deviceId: version.deviceId, + contentSize: version.contentSize + }; + } + + private recordRecoveredApplication( + steps: readonly { + documentId: string; + from?: string; + to?: string; + replacement?: unknown; + }[] + ): void { + for (const step of steps) { + const to = this.database.state.local[step.documentId]; + if (!step.from && !to) continue; + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Filesystem application recovered", + details: + step.from && to && step.from !== to + ? { + type: SyncType.MOVE, + relativePath: to, + movedFrom: step.from + } + : !step.from && to + ? { type: SyncType.CREATE, relativePath: to } + : step.from && !to + ? { + type: SyncType.DELETE, + relativePath: step.from + } + : { + type: SyncType.UPDATE, + relativePath: to ?? step.from! + } + }); + } + } + + private recordManifestApplication( + before: Readonly>, + writes: Readonly> + ): void { + const after = this.database.state.local; + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(after) + ])) { + const from = before[id]; + const to = after[id]; + if (from === to && !writes[id]) continue; + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Remote change applied", + details: + from && to && from !== to + ? { + type: SyncType.MOVE, + relativePath: to, + movedFrom: from + } + : !from && to + ? { type: SyncType.CREATE, relativePath: to } + : from && !to + ? { type: SyncType.DELETE, relativePath: from } + : { + type: SyncType.UPDATE, + relativePath: to ?? from + } + }); + } + } + + private async remoteSnapshot( + head: DocumentVersionWithoutContent, + path: string, + supplied?: DocumentVersion + ): Promise { + const content = + supplied !== undefined + ? base64ToBytes(supplied.contentBase64) + : await this.service.getDocumentVersionContent({ + documentId: head.documentId, + vaultUpdateId: head.vaultUpdateId + }); + await this.cacheContent(head.vaultUpdateId, path, content); + return toStoredSnapshot({ content }); + } + + private async contentWrite( + next: EngineState, + head: DocumentVersionWithoutContent, + path: string, + supplied?: DocumentVersion + ): Promise { + const document = (next.documents[head.documentId] ??= { + materialized: false + }); + if ( + next.excluded?.[head.documentId] !== undefined || + this.ignored(path) || + this.oversized(head.contentSize) + ) { + return undefined; + } + const currentPath = this.database.state.local[head.documentId]; + const local = + currentPath && + (await this.files.fs.stat(currentPath))?.kind === "file" + ? await this.files.snapshot(currentPath) + : undefined; + if ( + local !== undefined && + this.oversized(base64ToBytes(local.contentBase64).length) + ) { + return undefined; + } + const base = + document.recoveryBase ?? + (local !== undefined && + document.base !== undefined && + document.bootstrap !== true + ? await this.remoteSnapshot(document.base, path) + : undefined); + // Cache the new head after the older merge base so it remains the most + // recently used version and can back the follow-up local diff. + const remote = await this.remoteSnapshot(head, path, supplied); + let result = remote; + if (local !== undefined) { + result = await mergeContent( + path, + base, + local, + remote, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ); + } + document.base = { ...this.documentHead(head), hash: remote.hash }; + delete document.recoveryBase; + document.bootstrap = false; + document.materialized = true; + document.observedHash = result.hash; + return local?.contentBase64 === result.contentBase64 + ? undefined + : { expected: local, replacement: result }; + } + + private async incorporateContent( + head: DocumentVersionWithoutContent, + cursor?: number, + supplied?: DocumentVersion, + clearPending = false + ): Promise { + await this.scan(); + const guard = this.localGuard(); + const next = this.next(); + if ( + !next.remoteHeads[head.documentId] || + next.remoteHeads[head.documentId].vaultUpdateId < head.vaultUpdateId + ) { + next.remoteHeads[head.documentId] = this.documentHead(head); + } + if (cursor !== undefined) { + next.lastSeenUpdateId = cursor; + } + if (clearPending) { + delete next.pending; + } + const path = next.local[head.documentId]; + const document = next.documents[head.documentId]; + const wasMaterialized = document?.materialized; + const writes: Record = {}; + if ( + path !== undefined && + (!document?.base || + !document.materialized || + document.bootstrap || + document.base.vaultUpdateId < head.vaultUpdateId) + ) { + const write = await this.contentWrite(next, head, path, supplied); + if (write) { + writes[head.documentId] = write; + } + } + await this.files.apply(next, writes, guard); + if (path !== undefined && writes[head.documentId]) { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Remote content applied", + details: { + type: wasMaterialized ? SyncType.UPDATE : SyncType.CREATE, + relativePath: path + }, + author: head.userId, + timestamp: new Date(head.updatedDate) + }); + } + } + + private async incorporateFileManifest( + remote: FileManifest, + cursor?: number, + initial?: VaultSnapshot, + clearPending = false, + heads?: EngineState["remoteHeads"] + ): Promise { + validateFileManifest(remote.entries); + await this.scan(initial); + const before = { ...this.database.state.local }; + const guard = this.localGuard(); + const next = this.next(); + if (cursor !== undefined) { + next.lastSeenUpdateId = cursor; + delete next.eventReplay; + } + if (clearPending) { + delete next.pending; + } + if (remote.fileManifestId < next.fileManifest.fileManifestId) { + await this.database.commit(next); return; } - const [allLocalFiles, remote] = await awaitAll([ - this.operations.listFilesRecursively(), - this.syncQueue.add(async () => this.syncService.getAll()) - ]); - - if (remote !== undefined) { - remote.latestDocuments - .filter( - (remoteDocument) => - allLocalFiles.includes(remoteDocument.relativePath) && - !remoteDocument.isDeleted && - this.database.getDocumentByDocumentId( - remoteDocument.documentId - ) === undefined - ) - .forEach((remoteDocument) => { - this.database.createNewEmptyDocument( - remoteDocument.documentId, - remoteDocument.vaultUpdateId, - remoteDocument.relativePath + const base = { ...next.fileManifest.entries }; + for (const [id, path] of Object.entries(next.excluded ?? {})) { + if (path === null) delete base[id]; + else base[id] = path; + } + const previousManifest = next.fileManifest.entries; + next.local = mergeFileManifests(base, next.local, remote.entries); + next.fileManifest = remote; + if (heads) next.remoteHeads = heads; + for (const [id, oldPath] of Object.entries(before)) { + const path = next.local[id]; + if ( + this.ignored(oldPath) || + this.unsyncablePaths.has(oldPath) || + (path !== undefined && this.ignored(path)) + ) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = previousManifest[id] ?? null; + next.local[id] = oldPath; + } else if (next.excluded) delete next.excluded[id]; + } + for (const [id, path] of Object.entries(next.excluded ?? {})) { + if ( + before[id] === undefined && + (path === null || !this.ignored(path)) + ) + delete next.excluded![id]; + } + next.local = resolvePaths( + next.local, + remote.entries, + Object.keys(next.excluded ?? {}), + next.protectedPaths + ); + if (initial) { + next.initialized = true; + delete next.bootstrap; + for (const head of initial.documents) { + next.remoteHeads[head.documentId] = this.documentHead(head); + } + } + const writes: Record = {}; + for (const [id, path] of Object.entries(next.local)) { + next.documents[id] ??= { materialized: false }; + if (before[id] === undefined) + next.documents[id].materialized = false; + if ( + next.excluded?.[id] === undefined && + !this.ignored(path) && + remote.entries[id] !== undefined && + (!this.database.state.local[id] || + !next.documents[id].base || + next.documents[id].bootstrap || + !next.documents[id].materialized || + (next.remoteHeads[id]?.vaultUpdateId ?? 0) > + (next.documents[id].base?.vaultUpdateId ?? 0)) + ) { + const knownHead = + next.remoteHeads[id] ?? + initial?.documents.find((doc) => doc.documentId === id); + const head = knownHead ?? (await this.service.metadata(id)); + if (!head) { + throw new PermanentSyncError( + `File manifest references missing content: ${id}` ); - }); + } + next.remoteHeads[id] = this.documentHead(head); + const write = await this.contentWrite(next, head, path); + if (write) { + writes[id] = write; + } + } + } + await this.files.apply(next, writes, guard); + this.recordManifestApplication(before, writes); + } + + private async preparePush(): Promise { + if (this.stopped || this.database.state.eventReplay) { + return false; } - this.database.setHasInitialSyncCompleted(true); + let guard = this.localGuard(); + const canonical = resolvePaths( + this.database.state.local, + this.database.state.fileManifest.entries, + Object.keys(this.database.state.excluded ?? {}), + this.database.state.protectedPaths + ); + if (!sameFileManifest(canonical, this.database.state.local)) { + await this.files.apply( + { ...this.next(), local: canonical }, + {}, + guard + ); + await this.scan(); + guard = this.localGuard(); + } + + // Resolve the map after restoring excluded server paths, then make + // those allocations visible locally before submitting a manifest. + const submitted = this.manifestToSubmit(); + const local = { ...this.database.state.local }; + for (const [id, path] of Object.entries(submitted)) + if (this.database.state.excluded?.[id] === undefined) + local[id] = path; + if (!sameFileManifest(local, this.database.state.local)) { + await this.files.apply({ ...this.next(), local }, {}, guard); + await this.scan(); + guard = this.localGuard(); + } + for (const [id, path] of Object.entries(this.database.state.local)) { + if ( + this.ignored(path) || + this.database.state.excluded?.[id] !== undefined || + !this.database.state.documents[id]?.materialized || + this.unsyncablePaths.has(path) || + this.oversized( + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime. + this.database.state.remoteHeads[id]?.contentSize ?? 0 + ) + ) { + continue; + } + const snapshot = await this.files.snapshot(path); + if ( + !snapshot || + this.oversized(base64ToBytes(snapshot.contentBase64).length) + ) { + continue; + } + const { base, rejected } = this.database.state.documents[id]; + if ( + base?.hash === snapshot.hash || + rejected?.hash === snapshot.hash + ) { + continue; + } + guard(); + await this.database.commit({ + ...this.next(), + pending: { + type: "content", + documentId: id, + snapshot, + request: { + requestId: uuid(), + parentVersionId: base?.vaultUpdateId ?? null, + content: await this.pushContent( + path, + base?.vaultUpdateId, + snapshot + ) + } + } + }); + return true; + } + + const entries = this.manifestToSubmit(); + validateFileManifest(entries); + if ( + !sameFileManifest( + entries, + this.database.state.fileManifest.entries + ) && + (!this.database.state.rejectedManifest || + !sameFileManifest( + entries, + this.database.state.rejectedManifest.entries + )) + ) { + guard(); + await this.database.commit({ + ...this.next(), + pending: { + type: "fileManifest", + request: { + requestId: uuid(), + parentFileManifestId: + this.database.state.fileManifest.fileManifestId, + entries + } + } + }); + return true; + } + return false; + } + + private manifestToSubmit(): Record { + // Do not publish a new document until its initial content is acknowledged. + const entries = { ...this.database.state.local }; + for (const id of Object.keys(this.database.state.excluded ?? {})) { + const remotePath = this.database.state.fileManifest.entries[id]; + if (remotePath === undefined) delete entries[id]; + else entries[id] = remotePath; + } + for (const id of Object.keys(entries)) { + if ( + !this.database.state.documents[id]?.base && + this.database.state.fileManifest.entries[id] === undefined + ) { + delete entries[id]; + } + } + return resolvePaths( + entries, + this.database.state.fileManifest.entries, + Object.keys(this.database.state.excluded ?? {}), + [ + ...(this.database.state.protectedPaths ?? []), + ...Object.keys(this.database.state.excluded ?? {}).flatMap( + (id) => + this.database.state.local[id] + ? [this.database.state.local[id]] + : [] + ) + ] + ); + } + + private get rejectionMessage(): string | undefined { + const { state } = this.database; + for (const [id, path] of Object.entries(state.local ?? {})) { + const doc = state.documents[id]; + if ( + doc?.rejected && + doc.observedHash === doc.rejected.hash && + doc.base?.hash !== doc.rejected.hash && + !this.ignored(path) && + state.excluded?.[id] === undefined + ) + return doc.rejected.message; + } + if ( + state.rejectedManifest && + sameFileManifest( + this.manifestToSubmit(), + state.rejectedManifest.entries + ) && + !sameFileManifest( + state.fileManifest.entries, + state.rejectedManifest.entries + ) + ) + return state.rejectedManifest.message; + return undefined; + } + + /** An explicit reset or settings change permits fresh attempts after rejection. */ + public async retryRejectedRequests(): Promise { + await this.database.recoverPersistence(); + const next = structuredClone(this.database.state); + let changed = false; + for (const state of [next, next.application?.next]) { + if (!state) continue; + if (state.rejectedManifest) { + delete state.rejectedManifest; + changed = true; + } + for (const doc of Object.values(state.documents)) { + if (doc.rejected) { + delete doc.rejected; + changed = true; + } + } + } + if (changed) await this.database.commit(next); + } + + private async finishPending(): Promise { + let { pending } = this.database.state; + if (!pending) { + return; + } + + if (!pending.response && !pending.rejection) { + const copy = structuredClone(pending); + try { + if (copy.type === "content") { + copy.response = await this.service.putFileContent( + copy.documentId, + copy.request + ); + } else { + copy.response = await this.service.pushFileManifest( + copy.request + ); + } + } catch (error) { + if (!(error instanceof PermanentSyncError)) throw error; + copy.rejection = error.message; + } + await this.database.commit({ ...this.next(), pending: copy }); + pending = copy; + } + + if (pending.rejection && !pending.response) { + const next = this.next(); + if (pending.type === "content") { + await this.files.retainSnapshot( + pending.request.requestId, + pending.documentId, + next.local[pending.documentId], + pending.snapshot + ); + next.documents[pending.documentId].rejected = { + hash: pending.snapshot.hash, + message: pending.rejection + }; + } else + next.rejectedManifest = { + entries: pending.request.entries, + message: pending.rejection + }; + (next.unconfirmed ??= []).push(pending); + delete next.pending; + await this.database.commit(next); + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + message: pending.rejection, + details: { + type: SyncType.SKIPPED, + relativePath: + pending.type === "content" + ? (next.local[pending.documentId] ?? + pending.documentId) + : "File manifest" + } + }); + return; + } + + if (pending.type === "fileManifest") { + const response = pending.response!; + if (response.type === "StaleBase") { + await this.incorporateFileManifest( + response, + undefined, + undefined, + true + ); + } else { + const next = this.next(); + delete next.rejectedManifest; + next.fileManifest = { + fileManifestId: response.fileManifestId, + entries: pending.request.entries + }; + delete next.pending; + const previous = this.database.state.fileManifest.entries; + await this.database.commit(next); + for (const id of new Set([ + ...Object.keys(previous), + ...Object.keys(next.fileManifest.entries) + ])) { + const before = previous[id], + after = next.fileManifest.entries[id]; + if (before === after) { + continue; + } + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "File manifest change committed", + details: + after === undefined + ? { + type: SyncType.DELETE, + relativePath: before + } + : before === undefined + ? { + type: SyncType.CREATE, + relativePath: after + } + : { + type: SyncType.MOVE, + relativePath: after, + movedFrom: before + } + }); + } + } + } else { + const response = pending.response!; + if (response.type === "StaleBase") { + await this.files.retainSnapshot( + pending.request.requestId, + pending.documentId, + this.database.state.local[pending.documentId], + pending.snapshot + ); + await this.incorporateContent( + response, + undefined, + undefined, + true + ); + } else { + const next = this.next(); + const doc = (next.documents[pending.documentId] ??= { + materialized: false + }); + doc.base = { + ...this.documentHead(response), + hash: pending.snapshot.hash + }; + doc.bootstrap = false; + delete doc.rejected; + next.remoteHeads[pending.documentId] = + this.documentHead(response); + delete next.pending; + await this.database.commit(next); + const path = next.local[pending.documentId]; + await this.cacheContent( + response.vaultUpdateId, + path, + base64ToBytes(pending.snapshot.contentBase64) + ); + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Content committed", + details: { + type: SyncType.UPDATE, + relativePath: + next.local[pending.documentId] ?? pending.documentId + }, + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + } + } } } diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts index a5f0cc7c..47c850f9 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts @@ -3,6 +3,30 @@ import assert from "node:assert"; import { EventListeners } from "./event-listeners"; describe("EventListeners", () => { + it("isolates throwing and rejecting observers and still notifies later listeners", async () => { + const failures: unknown[] = []; + const listeners = new EventListeners<() => unknown>((error) => + failures.push(error) + ); + const syncError = new Error("sync observer"); + const asyncError = new Error("async observer"); + let called = 0; + listeners.add(() => { + throw syncError; + }); + listeners.add(async () => { + throw asyncError; + }); + listeners.add(() => { + called++; + }); + assert.doesNotThrow(() => { + listeners.trigger(); + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(called, 1); + assert.deepEqual(failures, [syncError, asyncError]); + }); it("should add & remove listeners", () => { const listeners = new EventListeners<() => void>(); // eslint-disable-next-line @typescript-eslint/no-empty-function diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.ts index e08ca65e..0c7b0e0b 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.ts @@ -8,50 +8,65 @@ import { awaitAll } from "../await-all"; export class EventListeners any> { private readonly listeners: TListener[] = []; + public constructor( + private readonly onError: (error: unknown) => void = (error) => { + console.error("Event observer failed", error); + } + ) {} + public get count(): number { return this.listeners.length; } /** - * Adds a new listener to the collection. - * - * @param listener The listener callback to add - * @returns An unsubscribe function that removes this listener when called - */ + * Adds a new listener to the collection. + * + * @param listener The listener callback to add + * @returns An unsubscribe function that removes this listener when called + */ public add(listener: TListener): () => void { this.listeners.push(listener); return () => this.remove(listener); } /** - * Removes a listener from the collection. - * - * @param listener The listener callback to remove - * @returns true if the listener was found and removed, false otherwise - */ + * Removes a listener from the collection. + * + * @param listener The listener callback to remove + * @returns true if the listener was found and removed, false otherwise + */ public remove(listener: TListener): boolean { return removeFromArray(this.listeners, listener); } /** - * Triggers all listeners synchronously with the provided arguments. - * Any returned promises are ignored. Use triggerAsync() to await them. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners synchronously with the provided arguments. + * Observer failures are reported without interrupting delivery or background + * work. Returned promises are observed; use triggerAsync() to await them. + * + * @param args The arguments to pass to each listener + */ public trigger(...args: Parameters): void { - this.listeners.forEach((listener) => { - listener(...args); - }); + for (const listener of [...this.listeners]) { + try { + void Promise.resolve(listener(...args)).catch( + (error: unknown) => { + this.reportFailure(error); + } + ); + } catch (error) { + this.reportFailure(error); + } + } } /** - * Triggers all listeners and awaits any promises they return. - * Synchronous listeners are called immediately, and any async listeners - * are awaited in parallel. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners and awaits any promises they return. + * Synchronous listeners are called immediately, and any async listeners + * are awaited in parallel. + * + * @param args The arguments to pass to each listener + */ public async triggerAsync(...args: Parameters): Promise { await awaitAll( this.listeners @@ -68,4 +83,11 @@ export class EventListeners any> { public clear(): void { this.listeners.length = 0; } + private reportFailure(error: unknown): void { + try { + this.onError(error); + } catch { + // A diagnostic callback must not break background work either. + } + } } diff --git a/frontend/sync-client/src/utils/portable-path.test.ts b/frontend/sync-client/src/utils/portable-path.test.ts new file mode 100644 index 00000000..1ada24d5 --- /dev/null +++ b/frontend/sync-client/src/utils/portable-path.test.ts @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { test } from "node:test"; +import { validatePortablePaths } from "./portable-path"; + +test("a small deeply nested manifest fits within a bounded validation heap", () => { + const source = pathToFileURL(resolve(__dirname, "portable-path.ts")).href; + const result = spawnSync( + process.execPath, + [ + "--max-old-space-size=64", + "--import", + require.resolve("tsx"), + "--input-type=module", + "--eval", + ` + import { validatePortablePaths } from ${JSON.stringify(source)}; + const directory = Array(20000).fill("a").join("/"); + validatePortablePaths([directory + "/one.md", directory + "/two.md"]); + ` + ], + { encoding: "utf8", timeout: 10000 } + ); + assert.equal(result.status, 0, result.stderr || String(result.error)); +}); + +test("shared directories remain distinct from aliases and file ancestors", () => { + validatePortablePaths(["Notes/one.md", "Notes/two.md", "Other/one.md"]); + for (const paths of [ + ["Notes/one.md", "notes/two.md"], + ["Straße/one.md", "STRASSE/two.md"], + ["Notes", "Notes/one.md"], + ["Notes/one.md", "Notes"], + ["Notes/one.md", "Notes/one.md"] + ]) + assert.throws(() => validatePortablePaths(paths), /Conflicting path/); +}); diff --git a/frontend/sync-client/src/utils/portable-path.ts b/frontend/sync-client/src/utils/portable-path.ts new file mode 100644 index 00000000..f794cae7 --- /dev/null +++ b/frontend/sync-client/src/utils/portable-path.ts @@ -0,0 +1,190 @@ +export const INTERNAL_DIRECTORY = ".vault-link-sync"; +const MAX_COMPONENT_BYTES = 255; +const byteLength = (value: string): number => + new TextEncoder().encode(value).length; +function truncate(value: string, limit: number): string { + let result = ""; + for (const character of value) { + if (byteLength(result + character) > limit) break; + result += character; + } + return result; +} + +const reservedDeviceName = + /^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$/u; + +// NFC -> uppercase -> NFC matches the server's locale-independent alias rules. +const pathKey = (path: string): string => + path.normalize("NFC").toUpperCase().normalize("NFC"); + +export const arePathAliases = (left: string, right: string): boolean => + pathKey(left) === pathKey(right); + +export function findPathWithSameSpelling( + paths: Iterable, + wanted: string +): string | undefined { + const normalized = wanted.normalize("NFC"); + for (const path of paths) { + if (path !== wanted && path.normalize("NFC") === normalized) + return path; + } + return undefined; +} + +export const isInternalPath = (path: string): boolean => + arePathAliases(path.split("/")[0], INTERNAL_DIRECTORY); + +function validatePath(path: string): void { + if (!path || path !== path.normalize("NFC") || isInternalPath(path)) + throw new Error(`Invalid portable path: ${path}`); + + for (const part of path.split("/")) { + if ( + !part || + part === "." || + part === ".." || + /[<>:"\\|?*\p{Cc}]|[. ]$/u.test(part) || + reservedDeviceName.test( + part.split(".")[0].replace(/ +$/u, "").toUpperCase() + ) + ) + throw new Error(`Invalid portable path: ${path}`); + } +} + +export function validatePortablePaths(paths: Iterable): void { + // Component keys keep storage proportional to the input, even when a + // manifest contains thousands of directory levels. + const nodes = new Map< + string, + { id: number; spelling: string; file: boolean } + >(); + for (const path of paths) { + validatePath(path); + const parts = path.split("/"); + let parent = 0; + parts.forEach((spelling, index) => { + const file = index === parts.length - 1; + const key = `${parent}/${pathKey(spelling)}`; + const previous = nodes.get(key); + if ( + previous && + (previous.spelling !== spelling || previous.file || file) + ) + throw new Error(`Conflicting path: ${path}`); + const node = previous ?? { id: nodes.size + 1, spelling, file }; + nodes.set(key, node); + parent = node.id; + }); + } +} + +function portableName(part: string): string { + let result = part + .normalize("NFC") + .replace(/[<>:"\\|?*\p{Cc}]/gu, "_") + .replace(/[. ]+$/u, "_"); + if (!result || result === "." || result === "..") result = "_"; + if ( + reservedDeviceName.test( + result.split(".")[0].replace(/ +$/u, "").toUpperCase() + ) + ) + result = "_" + result; + return result; +} + +function fitName(name: string, suffix: string, file: boolean): string { + const dot = file ? name.lastIndexOf(".") : -1; + const extension = dot > 0 ? truncate(name.slice(dot), 64) : ""; + const stem = dot > 0 ? name.slice(0, dot) : name; + return portableName( + truncate(stem, MAX_COMPONENT_BYTES - byteLength(suffix + extension)) + + suffix + + extension + ); +} + +function pathsConflict(left: string, right: string): boolean { + const a = left.split("/"), + b = right.split("/"); + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (!arePathAliases(a[i], b[i])) return false; + if (a[i] !== b[i]) return true; + } + return true; // Equal names, or a file occupies an ancestor directory. +} + +function isAvailablePath( + candidate: string, + id: string, + occupied: Readonly> +): boolean { + validatePath(candidate); + return Object.entries(occupied).every( + ([otherId, path]) => otherId === id || !pathsConflict(candidate, path) + ); +} + +/** Deterministic allocation also handles a file blocking an ancestor directory. */ +export function allocatePortablePath( + wanted: string, + id: string, + occupied: Readonly> +): string { + id = portableName(truncate(id, 64)); + const originals = wanted.split("/"); + const parts = originals.map((part, index) => { + const portable = portableName(part); + return byteLength(portable) > MAX_COMPONENT_BYTES + ? fitName( + portable, + ` (conflict ${id})`, + index === originals.length - 1 + ) + : portable; + }); + if (isInternalPath(parts.join("/"))) parts[0] = "_" + parts[0]; + let candidate = parts.join("/"); + const suffix = ` (conflict ${id})`; + const limit = (Object.keys(occupied).length + 1) * (parts.length + 1); + for (let attempt = 0; attempt <= limit; attempt++) { + try { + if (isAvailablePath(candidate, id, occupied)) return candidate; + } catch { + /* allocate below */ + } + const current = candidate.split("/"); + let conflictAt = current.length - 1; + for (let i = 0; i < current.length; i++) { + const prefix = current.slice(0, i + 1).join("/"); + if ( + Object.values(occupied).some((path) => { + const other = path + .split("/") + .slice(0, i + 1) + .join("/"); + return ( + arePathAliases(prefix, other) && + (prefix !== other || + path.split("/").length === i + 1 || + i === current.length - 1) + ); + }) + ) { + conflictAt = i; + break; + } + } + const original = parts[conflictAt] ?? current[conflictAt]; + current[conflictAt] = fitName( + original, + suffix + (attempt ? ` (${attempt})` : ""), + conflictAt === current.length - 1 + ); + candidate = current.join("/"); + } + throw new Error(`Unable to allocate a portable path: ${wanted}`); +} diff --git a/frontend/test-client/README.md b/frontend/test-client/README.md new file mode 100644 index 00000000..230f42d7 --- /dev/null +++ b/frontend/test-client/README.md @@ -0,0 +1,36 @@ +# Seeded sync workloads + +The old random mock filesystem/agent have been replaced by the same v4 adapter, +client wrapper, and canonical-state assertions as the scripted suite. +Periodic polling is explicitly disabled so it cannot repair missed notifications +and hide failures in these workloads. Polling has separate fake-clock tests. + +From `frontend`: + +```sh +npm run build --workspace sync-client --workspace test-client +node test-client/dist/cli.js --seed 42 --iterations 25 +node test-client/dist/cli.js --replay /path/to/trace-42.json +``` + +The server binary must already be built. Each invocation starts and cleans up its +own server/database, and prints the replay command before executing any actions. +`--artifacts` chooses the trace/result directory. Failures include database, +visible/durable disk images, and CAS request records. The default 25 iterations +are scenarios selected in shuffled cycles of nine operations, using two or three +clients. Existing documents are repeatedly edited, moved, swapped, deleted and +recreated. Concurrent edits share an established base; delayed editor events stay +queued through reconnection and a subsequent remote edit. Create, content-update +and manifest requests receive before/after network faults. + +Each checkpoint serializes a complete expected file set with exact bytes, paths +and persistent identity keys. Independent creates must remain distinct, and a +recreated file must obtain a new UUID. Later checkpoints reassert earlier edits +and moves, rather than retaining only their markers. Old traces without a client +count still replay with two clients. + +All preservation assertions run even with delayed editor events. Explicit deletes +and overwrites exempt only their specific markers; unrelated content still must +appear exactly once. UUID continuity and canonical server state are checked too. +Seeds reproduce choices, not OS scheduling. See the +[full harness documentation](../deterministic-tests/README.md). diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts deleted file mode 100644 index 1640c2ec..00000000 --- a/frontend/test-client/src/agent/mock-agent.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { choose } from "../utils/choose"; -import { v4 as uuidv4 } from "uuid"; -import { assert } from "../utils/assert"; -import type { RelativePath, SyncSettings } from "sync-client"; -import { debugging, Logger, LogLevel, utils } from "sync-client"; -import { MockClient } from "./mock-client"; -import { sleep } from "../utils/sleep"; -import type { LogLine } from "sync-client"; -import { withTimeout } from "../utils/with-timeout"; - -const TIMEOUT_MS = 10 * 60 * 1000; - -export class MockAgent extends MockClient { - private readonly writtenContents: string[] = []; - private readonly pendingActions: Promise[] = []; - - // The renamed file finding algorithm isn't too smart so we can't both update and rename the same file - private readonly doNotTouchWhileOffline: string[] = []; - - public constructor( - initialSettings: Partial, - public readonly name: string, - private readonly doDeletes: boolean, - private readonly doResets: boolean, - useSlowFileEvents: boolean, - private readonly jitterScaleInSeconds: number - ) { - super(initialSettings, useSlowFileEvents); - } - - public async init(): Promise { - await super.init( - debugging.slowFetchFactory(this.jitterScaleInSeconds), - debugging.slowWebSocketFactory( - this.jitterScaleInSeconds, - new Logger() // this logger isn't wired anywhere, so messages to it will be ignored - ) - ); - - assert( - (await this.client.checkConnection()).isSuccessful, - "Connection check failed" - ); - - this.client.logger.onLogEmitted.add((logLine: LogLine) => { - const state = this.client.getSettings().isSyncEnabled - ? "(online) " - : "(offline)"; - const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; - - // HACK: we have to ensure the file has been synced if we want to change it offline without data loss - const historyEntry = /.*History entry: (.*.md).*/.exec( - logLine.message - ); - - if (historyEntry) { - utils.removeFromArray( - this.doNotTouchWhileOffline, - historyEntry[1] - ); - } - switch (logLine.level) { - case LogLevel.ERROR: - console.error(formatted); - - if (!this.useSlowFileEvents) { - // Let's wait for the error to be caught if there was one - // eslint-disable-next-line @typescript-eslint/no-floating-promises - sleep(100).then(() => process.exit(1)); - } - - break; - case LogLevel.WARNING: - console.warn(formatted); - break; - case LogLevel.INFO: - console.info(formatted); - break; - case LogLevel.DEBUG: - console.debug(formatted); - break; - } - }); - - this.client.logger.info("Agent initialized"); - } - - public async act(): Promise { - const options: (() => Promise)[] = [ - this.createFileAction.bind(this) - ]; - - if ( - this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.length === 0 - ) { - options.push(this.disableSyncAction.bind(this)); - } else { - options.push(this.enableSyncAction.bind(this)); - } - - const files = await this.listFilesRecursively(); - - if (files.length > 0) { - options.push( - this.renameFileAction.bind(this, files), - this.updateFileAction.bind(this, files) - ); - - if (this.doDeletes) { - options.push(this.deleteFileAction.bind(this, files)); - } - } - - if (Math.random() < 0.015 && this.doResets) { - // we can't just queue this up as once it's destroyed, no more method calls can go to SyncClient - await this.resetClient(); - } else { - this.pendingActions.push( - (async (): Promise => { - try { - return await choose(options)(); - } catch (error) { - this.client.logger.error( - `Failed to perform an action: ${error}` - ); - this.client.logger.info( - JSON.stringify(this.data, null, 2) - ); - this.client.logger.info( - JSON.stringify(this.localFiles, null, 2) - ); - throw error; - } - })() - ); - } - } - - public async finish(): Promise { - await withTimeout( - (async (): Promise => { - await this.client.setSetting("isSyncEnabled", true); - await utils.awaitAll(this.pendingActions); - await this.client.waitUntilFinished(); - })(), - TIMEOUT_MS, - "finish()" - ); - } - - public async destroy(): Promise { - await withTimeout( - (async (): Promise => { - await this.client.waitUntilFinished(); - await this.client.destroy(); - })(), - TIMEOUT_MS, - "destroy()" - ); - } - - public assertFileSystemsAreConsistent(otherAgent: MockAgent): void { - const globalFiles = Array.from(otherAgent.localFiles.keys()); - const localFiles = Array.from(this.localFiles.keys()); - - const missingInOther = localFiles.filter( - (file) => !otherAgent.localFiles.has(file) - ); - const missingInLocal = globalFiles.filter( - (file) => !this.localFiles.has(file) - ); - - try { - assert( - missingInOther.length === 0, - `Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}` - ); - assert( - missingInLocal.length === 0, - `Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}` - ); - - for (const file of globalFiles) { - const localContent = new TextDecoder().decode( - this.localFiles.get(file) - ); - const otherContent = new TextDecoder().decode( - otherAgent.localFiles.get(file) - ); - assert( - localContent === otherContent, - `Content mismatch for file ${file}:\n${localContent}\n${otherContent}` - ); - } - } catch (e) { - this.client.logger.info( - "Local data: " + JSON.stringify(this.data, null, 2) - ); - this.client.logger.info( - "Local files: " + - Array.from(otherAgent.localFiles.keys()).join(", ") - ); - otherAgent.client.logger.info( - "Local data: " + JSON.stringify(otherAgent.data, null, 2) - ); - otherAgent.client.logger.info( - "Local files: " + - Array.from(otherAgent.localFiles.keys()).join(", ") - ); - - throw e; - } - } - - public assertAllContentIsPresentOnce(): void { - if (this.useSlowFileEvents) { - this.client.logger.info( - // We can't ensure that we have seen every single update - `Skipping content check for ${this.name} because slow file events are enabled` - ); - return; - } - - for (const content of this.writtenContents) { - const found = Array.from(this.localFiles.keys()).filter((key) => { - return new TextDecoder() - .decode(this.localFiles.get(key)) - .includes(content); - }); - - if (this.doDeletes) { - assert( - found.length <= 1, - `[${this.name}] Content ${content} found in ${found.join(", ")}` - ); - } else { - assert( - found.length >= 1, - `[${this.name}] Content ${content} not found in any files` - ); - - assert( - found.length <= 1, - `[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}` - ); - - const [file] = found; - const fileContent = new TextDecoder().decode( - this.localFiles.get(file) - ); - assert( - fileContent.split(content).length == 2, - `Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}` - ); - } - } - } - - private async resetClient(): Promise { - this.client.logger.info(`Resetting client ${this.name}`); - await this.client.destroy(); - await this.init(); - } - - private async createFileAction(): Promise { - const file = this.getFileName(); - - if ( - (!this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file)) || - (await this.exists(file)) - ) { - return; - } - - const content = this.getContent(); - this.client.logger.info( - `Decided to create file ${file} with content ${content}` - ); - - return this.create(file, new TextEncoder().encode(` ${content} `)); - } - - private async disableSyncAction(): Promise { - this.client.logger.info(`Decided to disable sync`); - await this.client.setSetting("isSyncEnabled", false); - } - - private async enableSyncAction(): Promise { - this.client.logger.info(`Decided to enable sync`); - await this.client.setSetting("isSyncEnabled", true); - } - - private async renameFileAction(files: RelativePath[]): Promise { - const file = choose(files); - - // We can't edit files offline that have been updated while offline. - // Otherwise, the resolution logic couldn't handle it. - if ( - !this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file) - ) { - this.client.logger.info( - `Skipping file ${file} because it has been updated while offline` - ); - return; - } - - const newName = this.getFileName(); - - if ( - (!this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(newName)) || - (await this.exists(newName)) - ) { - return; - } - - this.client.logger.info(`Decided to rename file ${file} to ${newName}`); - this.doNotTouchWhileOffline.push(file, newName); - - return this.rename(file, newName); - } - - private async updateFileAction(files: RelativePath[]): Promise { - const file = choose(files); - - // We can't edit files offline that have been updated while offline. - // Otherwise, the resolution logic couldn't handle it. - if ( - !this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file) - ) { - this.client.logger.info( - `Skipping file ${file} because it has been updated while offline` - ); - return; - } - - const content = this.getContent(); - this.client.logger.info( - `Decided to update file ${file} with ${content}` - ); - this.doNotTouchWhileOffline.push(file); - await this.atomicUpdateText(file, (old) => ({ - text: old.text + ` ${content} `, - cursors: [] - })); - } - - private async deleteFileAction(files: RelativePath[]): Promise { - const file = choose(files); - this.client.logger.info(`Decided to delete file ${file}`); - return this.delete(file); - } - - private getContent(): string { - const uuid = uuidv4(); - this.writtenContents.push(uuid); - return uuid; - } - - private getFileName(): string { - // Simulate name collisions between the clients - return `file-${Math.floor(Math.random() * 64)}.md`; - } -} diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts deleted file mode 100644 index c814879a..00000000 --- a/frontend/test-client/src/agent/mock-client.ts +++ /dev/null @@ -1,197 +0,0 @@ -import type { StoredDatabase, TextWithCursors } from "sync-client"; -import { assert } from "../utils/assert"; -import { - type RelativePath, - type FileSystemOperations, - type SyncSettings, - SyncClient -} from "sync-client"; - -export class MockClient implements FileSystemOperations { - protected readonly localFiles = new Map(); - protected client!: SyncClient; - - protected data: Partial<{ - settings: Partial; - database: Partial; - }> = { - database: { - // Assume all clients start at the same time so there's no need to fetch - // any shared state. - hasInitialSyncCompleted: true - } - }; - - public constructor( - initialSettings: Partial, - protected readonly useSlowFileEvents: boolean - ) { - this.data.settings = initialSettings; - } - - public async init( - fetchImplementation: typeof globalThis.fetch, - webSocketImplementation: typeof globalThis.WebSocket - ): Promise { - this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: fetchImplementation, - webSocket: webSocketImplementation - }); - - await this.client.start(); - } - - public async listFilesRecursively( - _root: RelativePath | undefined = undefined // we don't use multi-level paths during tests - ): Promise { - return Array.from(this.localFiles.keys()); - } - - public async read(path: RelativePath): Promise { - const file = this.localFiles.get(path); - if (!file) { - throw new Error(`File ${path} does not exist`); - } - return file; - } - - public async getFileSize(path: RelativePath): Promise { - return (await this.read(path)).length; - } - - public async exists(path: RelativePath): Promise { - return this.localFiles.has(path); - } - - public async create( - path: RelativePath, - newContent: Uint8Array - ): Promise { - if (this.localFiles.has(path)) { - throw new Error(`File ${path} already exists`); - } - this.client.logger.info( - `Creating file ${path} with content ${new TextDecoder().decode(newContent)}` - ); - this.localFiles.set(path, newContent); - - this.executeFileOperation(async () => - this.client.syncLocallyCreatedFile(path) - ); - } - - public async createDirectory(_path: RelativePath): Promise { - // This doesn't mean anything in our virtual FS representation - } - - public async atomicUpdateText( - path: RelativePath, - updater: (currentContent: TextWithCursors) => TextWithCursors - ): Promise { - const file = this.localFiles.get(path); - if (!file) { - throw new Error(`File ${path} does not exist`); - } - const currentContent = new TextDecoder().decode(file); - const newContent = updater({ text: currentContent, cursors: [] }).text; - const newContentUint8Array = new TextEncoder().encode(newContent); - this.localFiles.set(path, newContentUint8Array); - - if (!this.useSlowFileEvents) { - const existingParts = currentContent - .split(" ") - .map((part) => part.trim()); - const newParts = newContent.split(" ").map((part) => part.trim()); - existingParts.forEach((part) => - // all changes should be additive - { - assert( - newParts.includes(part), - `Part ${part} not found in new content: ${newContent}` - ); - } - ); - } - - this.client.logger.info( - `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` - ); - - this.executeFileOperation(async () => - this.client.syncLocallyUpdatedFile({ - relativePath: path - }) - ); - - return newContent; - } - - public async write(path: RelativePath, content: Uint8Array): Promise { - const hasExisted = this.localFiles.has(path); - this.localFiles.set(path, content); - - this.client.logger.info( - `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` - ); - - this.executeFileOperation(async () => { - if (hasExisted) { - return this.client.syncLocallyUpdatedFile({ - relativePath: path - }); - } else { - return this.client.syncLocallyCreatedFile(path); - } - }); - } - - public async delete(path: RelativePath): Promise { - this.client.logger.info( - `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` - ); - this.localFiles.delete(path); - - this.executeFileOperation(async () => - this.client.syncLocallyDeletedFile(path) - ); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - const file = this.localFiles.get(oldPath); - if (!file) { - throw new Error(`File ${oldPath} does not exist`); - } - this.localFiles.set(newPath, file); - if (oldPath !== newPath) { - this.localFiles.delete(oldPath); - } - - this.client.logger.info( - `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` - ); - - this.executeFileOperation(async () => - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }) - ); - } - - private executeFileOperation(callback: () => unknown): void { - if (this.useSlowFileEvents) { - // we aren't the best client and it takes some time to notice changes - setTimeout(callback, Math.random() * 100); - } else { - callback(); - } - } -} diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 3af547e7..eb407615 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -1,205 +1,90 @@ -import type { SyncSettings } from "sync-client"; -import { utils } from "sync-client"; -import { MockAgent } from "./agent/mock-agent"; -import { sleep } from "./utils/sleep"; -import { v4 as uuidv4 } from "uuid"; -import { randomCasing } from "./utils/random-casing"; +import { parseArgs } from "node:util"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { Logger } from "sync-client"; +import { ServerControl } from "../../deterministic-tests/src/server-control"; +import { ServerManager } from "../../deterministic-tests/src/server-manager"; +import { TestRunner } from "../../deterministic-tests/src/test-runner"; +import { generateTrace, traceTest, type Trace } from "./workload"; -const TEST_ITERATIONS = 5; - -// Simulate async file access by injecting waiting time before returning from file operations. -let slowFileEvents = false; - -// Whether to do resets in the test runs -let doResets = false; - -async function runTest({ - agentCount, - concurrency, - iterations, - doDeletes, - useResets, - useSlowFileEvents, - jitterScaleInSeconds -}: { - agentCount: number; - concurrency: number; - iterations: number; - doDeletes: boolean; - useResets: boolean; - useSlowFileEvents: boolean; - jitterScaleInSeconds: number; -}): Promise { - slowFileEvents = useSlowFileEvents; - doResets = useResets; - - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; - console.info(`Running test ${settings}`); - - const vaultName = uuidv4(); - console.info(`Using vault name: ${vaultName}`); - const initialSettings: Partial = { - isSyncEnabled: true, - token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces - vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter - syncConcurrency: concurrency, - remoteUri: "http://localhost:3000" - }; - - const clients: MockAgent[] = []; - for (let i = 0; i < agentCount; i++) { - clients.push( - new MockAgent( - initialSettings, - `agent-${i}`, - doDeletes, - useResets, - useSlowFileEvents, - jitterScaleInSeconds - ) - ); - } - - try { - await utils.awaitAll(clients.map(async (client) => client.init())); - - for (let i = 0; i < iterations; i++) { - console.info(`Iteration ${i + 1}/${iterations}`); - await utils.awaitAll(clients.map(async (client) => client.act())); - await sleep(Math.random() * 200); - } - - console.info("Stopping agents"); - - // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and - for (const client of clients) { - try { - console.info(`Finishing up ${client.name}`); - await client.finish(); - } catch (err) { - if (!slowFileEvents) { - throw err; - } - } - } - - // then we need a second pass to ensure that all agents pull the same state. - for (const client of clients) { - try { - console.info(`Destroying ${client.name}`); - await client.destroy(); - } catch (err) { - if (!slowFileEvents) { - throw err; - } - } - } - - console.info("Agents finished successfully"); - - clients.slice(0, -1).forEach((client, i) => { - console.info( - `Checking consistency between ${client.name} and ${clients[i + 1].name}` - ); - client.assertFileSystemsAreConsistent(clients[i]); - console.info(`Consistency check for ${client.name} passed`); - }); - - console.info("File systems found to be consistent"); - - clients.forEach((client) => { - console.info(`Checking content for ${client.name}`); - client.assertAllContentIsPresentOnce(); - console.info(`Content check for ${client.name} passed`); - }); - - console.info(`Test passed ${settings}`); - } catch (err) { - console.error(`Test failed ${settings}`); - throw err; - } +function integer(value: string, zero = false): number { + const number = Number(value); + if ( + !/^\d+$/.test(value) || + !Number.isSafeInteger(number) || + number < (zero ? 0 : 1) + ) + throw new Error(`Invalid integer: ${value}`); + return number; } -async function runTests(): Promise { - for (let i = 0; i < TEST_ITERATIONS; i++) { - await runTest({ - agentCount: 2, - concurrency: 16, - iterations: 100, - doDeletes: true, - useResets: true, - useSlowFileEvents: true, - jitterScaleInSeconds: 0.75 - }); - - for (const useSlowFileEvents of [true, false]) { - for (const concurrency of [ - 16, - 1 // test with concurrency 1 to check for deadlocks - ]) { - for (const doDeletes of [false, true]) { - await runTest({ - agentCount: 2, - concurrency, - iterations: 100, - doDeletes, - useResets: false, - useSlowFileEvents, - jitterScaleInSeconds: 0.75 - }); - } - } +async function main(): Promise { + const { values } = parseArgs({ + options: { + seed: { type: "string", default: "1" }, + iterations: { type: "string", default: "25" }, + replay: { type: "string" }, + artifacts: { type: "string" }, + "server-binary": { type: "string" }, + config: { type: "string" } } - } -} - -process.on("uncaughtException", (error) => { - if ( - error instanceof Error && - error.message.includes( - "WebSocket was closed before the connection was established" - ) - ) { - return; - } - - console.error("Uncaught exception:", error); - process.exit(1); -}); - -process.on("unhandledRejection", (error, _promise) => { - if (error instanceof Error && error.message === "Sync was reset") { - return; - } - - if ( - slowFileEvents && - error instanceof Error && - (error.message.includes("Document not found") || - error.message.includes("Document already exists at new location")) - ) { - return; - } - - if ( - doResets && - error instanceof Error && - error.message.includes( - "SyncClient has been destroyed and can no longer be used" - ) - ) { - return; - } - - console.error("Unhandled rejection:", error); - process.exit(1); -}); - -runTests() - .then(() => { - process.exit(0); - }) - .catch((err: unknown) => { - console.error(err); - process.exit(1); }); + const seed = integer(values.seed!, true), + iterations = integer(values.iterations!); + const trace: Trace = values.replay + ? (JSON.parse(fs.readFileSync(values.replay, "utf8")) as Trace) + : generateTrace(seed, iterations); + const artifacts = values.artifacts + ? path.resolve(values.artifacts) + : fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-fuzz-")); + fs.mkdirSync(artifacts, { recursive: true }); + const tracePath = path.join(artifacts, `trace-${trace.seed}.json`); + fs.writeFileSync(tracePath, JSON.stringify(trace, null, 2)); + console.log(`Replay: node ${__filename} --replay ${tracePath}`); + const logger = new Logger(); + logger.onLogEmitted.add((line) => console.log(line.message)); + const root = path.resolve(__dirname, "../../.."); + const server = new ServerControl( + values["server-binary"] ?? + path.join(root, "sync-server/target/release/sync_server"), + values.config ?? path.join(root, "sync-server/config-e2e.yml"), + logger + ); + const manager = new ServerManager(logger); + manager.track(server); + manager.installSignalHandlers(); + try { + await server.start(); + const result = await new TestRunner( + server, + logger, + "test-token-change-me", + server.remoteUri + ).runTest(`seed-${trace.seed}`, traceTest(trace)); + fs.writeFileSync( + path.join(artifacts, `result-${trace.seed}.json`), + JSON.stringify(result, null, 2) + ); + if (!result.success) throw new Error(result.error); + } finally { + await server.stop(); + manager.untrack(server); + } +} + +process.on("unhandledRejection", (error) => { + console.error(error); + process.exit(1); +}); +process.on("uncaughtException", (error) => { + console.error(error); + process.exit(1); +}); +main().then( + () => process.exit(0), + (error: unknown) => { + console.error(error); + process.exit(1); + } +); diff --git a/frontend/test-client/src/utils/random-casing.test.ts b/frontend/test-client/src/utils/random-casing.test.ts index 33217525..34d36a37 100644 --- a/frontend/test-client/src/utils/random-casing.test.ts +++ b/frontend/test-client/src/utils/random-casing.test.ts @@ -6,7 +6,7 @@ describe("randomCasing", () => { it("simple test", () => { const input = "hello, this is a really long string with a lot of characters"; - const result = randomCasing(input); + const result = randomCasing(input, () => 0); assert.strictEqual(result.toLowerCase(), input.toLowerCase()); assert.notStrictEqual(result, input); }); diff --git a/frontend/test-client/src/utils/random-casing.ts b/frontend/test-client/src/utils/random-casing.ts index ba09dace..96d7d25a 100644 --- a/frontend/test-client/src/utils/random-casing.ts +++ b/frontend/test-client/src/utils/random-casing.ts @@ -1,7 +1,7 @@ -export function randomCasing(str: string): string { +export function randomCasing(str: string, random: () => number = Math.random): string { const chars = str.split(""); const randomCasedChars = chars.map((char) => { - if (Math.random() < 0.5) { + if (random() < 0.5) { return char.toUpperCase(); } return char.toLowerCase(); diff --git a/frontend/test-client/src/workload.test.ts b/frontend/test-client/src/workload.test.ts new file mode 100644 index 00000000..8d9814f7 --- /dev/null +++ b/frontend/test-client/src/workload.test.ts @@ -0,0 +1,71 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { generateTrace, traceTest } from "./workload"; + +test("concrete traces replay all choices and retain preservation assertions with delayed events", () => { + const trace = generateTrace(42, 12); + assert.deepEqual(trace, generateTrace(42, 12)); + assert.notDeepEqual(trace, generateTrace(43, 12)); + assert.deepEqual( + traceTest(trace), + traceTest(JSON.parse(JSON.stringify(trace))) + ); + assert.equal( + trace.steps.filter((step) => step.type === "assert-markers").length, + 12 + ); + assert(trace.steps.some((step) => step.type === "delay-notifications")); + assert(trace.steps.some((step) => step.type === "delete")); + assert(trace.steps.some((step) => step.type === "assert-documents")); + const assertions = trace.steps.filter( + (step) => step.type === "assert-documents" + ); + assert( + assertions.at(-1)!.expected.some((doc) => doc.conflict), + "Independent creates need two expected documents" + ); + assert( + assertions + .at(-1)! + .expected.some((doc) => + assertions[0].expected.some( + (initial) => initial.key === doc.key + ) + ), + "Later assertions must retain earlier identities" + ); + for (const kind of ["create", "manifest", "content"]) { + assert( + Array.from({ length: 10 }, (_, seed) => + generateTrace(seed, 9) + ).some((generated) => + generated.steps.some( + (step) => + step.type === "drop-response" && step.kind === kind + ) + ) + ); + } + const delayed = trace.steps.findIndex( + (step) => step.type === "delay-notifications" + ); + const flushed = trace.steps.findIndex( + (step, i) => i > delayed && step.type === "flush-notifications" + ); + assert( + trace.steps + .slice(delayed, flushed) + .some((step) => step.type === "enable-sync") + ); + assert( + trace.steps + .slice(delayed, flushed) + .some((step) => step.type === "assert-documents") + ); + assert.equal(traceTest({ ...trace, clients: undefined }).clients, 2); + assert( + Array.from({ length: 10 }, (_, seed) => generateTrace(seed, 1)).some( + (generated) => generated.clients === 3 + ) + ); +}); diff --git a/frontend/test-client/src/workload.ts b/frontend/test-client/src/workload.ts new file mode 100644 index 00000000..4f199f46 --- /dev/null +++ b/frontend/test-client/src/workload.ts @@ -0,0 +1,310 @@ +import type { + ExpectedDocument, + TestDefinition, + TestStep +} from "../../deterministic-tests/src/test-definition"; +import { Random } from "../../test-support/random"; + +export interface Trace { + version: 1; + seed: number; + iterations: number; + /** Old traces omit this and retain their original two-client behavior. */ + clients?: number; + steps: TestStep[]; +} + +type Document = ExpectedDocument & { content: string }; + +/** Concrete actions and independent expected states are serialized together. + * Every round checks the whole history, including earlier bytes and UUIDs. + * A shuffled cycle covers nine actions; most revisit existing documents. + */ +export function generateTrace(seed: number, iterations: number): Trace { + const random = new Random(seed); + const clients = 2 + random.int(2); + const steps: TestStep[] = []; + const documents: Document[] = []; + const markers: string[] = []; + const removed: string[] = []; + let serial = 0; + const newDocument = (path: string): Document => { + const key = `seed-${seed}-document-${serial++}`; + const marker = `[${key}]`; + markers.push(marker); + return { + key, + path, + content: `${marker}\n\nfirst paragraph\n\nlast paragraph\n` + }; + }; + const all = (type: "enable-sync" | "disable-sync") => { + for (let client = 0; client < clients; client++) + steps.push({ type, client }); + }; + const expected = () => + steps.push({ + type: "assert-documents", + expected: structuredClone(documents) + }); + const barrier = () => { + steps.push({ type: "barrier" }); + expected(); + }; + const forget = (doc: Document) => { + for (const marker of markers) + if (doc.content.includes(marker) && !removed.includes(marker)) + removed.push(marker); + documents.splice(documents.indexOf(doc), 1); + }; + all("enable-sync"); + steps.push({ type: "barrier" }); + // A lasting pool makes later actions interact with earlier versions/moves. + for (const path of ["shared/a.md", "日本語/b.md", "café/c.md"]) { + const doc = newDocument(path); + documents.push(doc); + steps.push({ type: "create", client: 0, path, content: doc.content }); + } + barrier(); + let choices: number[] = []; + for (let round = 0; round < iterations; round++) { + if (!choices.length) { + choices = Array.from({ length: 9 }, (_, i) => i); + for (let i = choices.length - 1; i > 0; i--) { + const j = random.int(i + 1); + [choices[i], choices[j]] = [choices[j], choices[i]]; + } + } + const kind = choices.pop()!; + const writer = random.int(clients); + const other = (writer + 1 + random.int(clients - 1)) % clients; + const doc = random.pick(documents.filter((entry) => !entry.conflict)); + const directory = `round-${round}/${random.pick(["nested", "日本語", "café", "space folder"])}`; + if (kind === 0) { + all("disable-sync"); + const first = newDocument(`${directory}/same.md`); + const second = { ...newDocument(first.path), conflict: true }; + steps.push( + { + type: "create", + client: writer, + path: first.path, + content: first.content + }, + { + type: "create", + client: other, + path: second.path, + content: second.content + }, + { + type: "drop-response", + client: writer, + kind: random.pick(["create", "manifest"] as const), + point: random.pick(["before", "after"] as const) + }, + { type: "enable-sync", client: writer }, + { type: "sync", client: writer }, + { type: "wait-for-response-drop", client: writer } + ); + all("enable-sync"); + documents.push(first, second); + } else if (kind === 1) { + // Both clients edit the same established version, at opposite ends. + all("disable-sync"); + const prefix = `remote prefix ${round}\n\n`; + const suffix = `\n\nlocal suffix ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + suffix + }, + { + type: "update", + client: other, + path: doc.path, + content: prefix + doc.content + }, + { type: "enable-sync", client: writer } + ); + all("enable-sync"); + doc.content = prefix + doc.content + suffix; + } else if (kind === 2) { + all("disable-sync"); + const target = `${directory}/moved.md`; + doc.content += `\nremote addition ${round}\n`; + steps.push( + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + }, + { + type: "update", + client: other, + path: doc.path, + content: doc.content + }, + { type: "enable-sync", client: writer } + ); + all("enable-sync"); + doc.path = target; + } else if (kind === 3) { + const second = random.pick( + documents.filter((entry) => !entry.conflict && entry !== doc) + ); + steps.push( + { type: "disable-sync", client: writer }, + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: `${directory}/temporary.md` + }, + { + type: "rename", + client: writer, + oldPath: second.path, + newPath: doc.path + }, + { + type: "rename", + client: writer, + oldPath: `${directory}/temporary.md`, + newPath: second.path + }, + { type: "enable-sync", client: writer } + ); + [doc.path, second.path] = [second.path, doc.path]; + } else if (kind === 4) { + // Reuse the same path but require a new UUID, even in later rounds. + steps.push({ type: "delete", client: writer, path: doc.path }); + forget(doc); + barrier(); + const replacement = newDocument(doc.path); + steps.push({ + type: "create", + client: other, + path: replacement.path, + content: replacement.content + }); + documents.push(replacement); + } else if (kind === 5) { + const target = `${directory}/rapid.md`; + steps.push( + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + }, + { + type: "rename", + client: writer, + oldPath: target, + newPath: doc.path + }, + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + } + ); + doc.path = target; + doc.content += `\nrapid edit ${round}\n`; + steps.push({ + type: "update", + client: writer, + path: doc.path, + content: doc.content + }); + } else if (kind === 6) { + steps.push({ + type: "drop-response", + client: writer, + kind: "content", + point: random.pick(["before", "after"] as const) + }); + doc.content += `\nretry edit ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + }, + { type: "wait-for-response-drop", client: writer } + ); + } else if (kind === 7) { + // Keep an editor event delayed across reconnection and another edit. + steps.push( + { type: "disable-sync", client: writer }, + { type: "delay-notifications", client: writer } + ); + doc.content += `\ndelayed local edit ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + }, + { type: "enable-sync", client: writer } + ); + barrier(); + doc.content += `\nlater remote edit ${round}\n`; + steps.push({ + type: "update", + client: other, + path: doc.path, + content: doc.content + }); + barrier(); + steps.push({ type: "flush-notifications", client: writer }); + } else { + // Explicit replacement retains the UUID. Only markers in the + // overwritten bytes are exempted from preservation. + for (const marker of markers) + if (doc.content.includes(marker) && !removed.includes(marker)) + removed.push(marker); + doc.content = newDocument(doc.path).content; + steps.push({ + type: "update", + client: writer, + path: doc.path, + content: doc.content + }); + } + barrier(); + steps.push({ + type: "assert-markers", + markers: [...markers], + removed: [...removed] + }); + if (random.int(4) === 0) { + steps.push({ type: "reset", client: random.int(clients) }); + barrier(); + } + } + return { version: 1, seed, iterations, clients, steps }; +} + +export function traceTest(trace: Trace): TestDefinition { + if ( + trace.version !== 1 || + !Array.isArray(trace.steps) || + !trace.steps.length || + (trace.clients !== undefined && + (!Number.isInteger(trace.clients) || trace.clients < 2)) + ) + throw new Error("Invalid/empty replay trace"); + return { + clients: trace.clients ?? 2, + description: `Seed ${trace.seed}; ${trace.iterations} generated conflict scenarios`, + steps: trace.steps + }; +} diff --git a/frontend/test-client/tsconfig.json b/frontend/test-client/tsconfig.json index e86df89d..7c59fb92 100644 --- a/frontend/test-client/tsconfig.json +++ b/frontend/test-client/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "baseUrl": ".", "strict": true, + "skipLibCheck": true, "target": "ES2022", "module": "CommonJS", "esModuleInterop": true, diff --git a/frontend/test-client/webpack.config.js b/frontend/test-client/webpack.config.js index 6aee1547..11c3e6dd 100644 --- a/frontend/test-client/webpack.config.js +++ b/frontend/test-client/webpack.config.js @@ -4,6 +4,7 @@ const webpack = require("webpack"); module.exports = { entry: "./src/cli.ts", target: "node", + externals: { "sync-client": "commonjs sync-client" }, mode: "production", optimization: { minimize: false diff --git a/frontend/test-support/README.md b/frontend/test-support/README.md new file mode 100644 index 00000000..e3601ba5 --- /dev/null +++ b/frontend/test-support/README.md @@ -0,0 +1,163 @@ +# Shared sync harness support + +This is test-only code, shared by `deterministic-tests` and `test-client` rather +than exported as debugging machinery by the production sync client. + +The journal suites also cover repeated saves replacing earlier unsynced text, +tracked file/directory moves into ignored paths, and saves crossing the size +limit. They check exact bytes and identities through recovery, including real +server reads. Client and Rust unit suites bound deep-path validation resources. + +| Layer | What it establishes | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `harness.test.ts` | Divergence/self-comparison detection, binary-safe comparisons, marker loss/duplication, cleanup failure propagation, deep persistence ownership, exclusive operations, exercised faults, seed reproducibility | +| `oracle-regressions.test.ts` | Negative checks for regenerated retries, wrong fault kinds, duplicate clients, corrupt hashes, BOMs, path byte limits, idempotent unlink, directory durability, and complete identity/content expectations | +| `recovery.test.ts` | Real engine journal recovery at every enumerated apply and replay mutation/save boundary; two interruptions followed by another power loss; identity, retained bytes and merge-policy regressions | +| `engine-regressions.test.ts` | Recovery archives, offline notifications, rejection/retry, exclusions, reverted remote updates, lifecycle races and editor changes during staging/install | +| `event-delivery.test.ts` | Real sync loop with checkpoint-controlled WS bursts before an iteration and during event application; no polling or extra wakeups to hide queued events | +| `polling.test.ts` | Real sync loop with fake clocks: missed local/remote notifications, disabled controls, repeated polls, shutdown and network retries independent of polling | +| `protocol.test.ts` | Terminated worker runtimes after accepted creates, existing-document Snapshot/Diff updates, and manifest requests, exact durable retries after newer remote heads and real server SIGKILL; concurrent CAS and ordered WS catchup | +| `real-disk.test.ts` | Host filesystem exclusive create/rename, symlink/hardlink rejection and a real engine rename-cycle application | +| Scripted suite | Existing multi-client regression scenarios plus v4 binary, identity, path-conflict, retry, and server-restart cases | +| Seeded workloads | Concrete replay traces, offline and rapid online moves/edits, concurrent creates, exact deletion/overwrite exemptions, nested Unicode paths, delayed notifications and before/after network failures | + +`scripts/e2e-runner.test.mjs` also guards clean-install workspace/lockfile +consistency, CI dependency installation and artifact upload wiring, subprocess +exit codes, log failures, spawn failures, and timeouts. + +`MemoryDisk` distinguishes visible namespace/data from durable entries. Each +mutation has `before:`, `visible:`, and `durable:` boundaries. `crash(false)` +models process death with dirty kernel state retained; `crash(true)` discards +unflushed state. Operations that return successfully have flushed their affected +entries. Idempotent unlink flushes absent entries too, and durable directory +removal makes all former descendants unreachable. This is a deliberately small atomic-operation model, not an emulation +of APFS/ext4 writeback. A crashed session is fenced against subsequent writes. + +`MemoryPersistence` atomically saves complete, cloned objects. Failure before a +save leaves the old object; failure after durable replacement leaves the new one. +The journal sweep recreates the production database/file-operations objects after +each fault. From each interrupted state it traces recovery and injects a second +process death or power loss at every replay mutation/save boundary. Final recovery +is replayed again and followed by another power loss. Cases cover swaps with new +output, cross-directory moves with a surviving sibling, concurrent text merges, +binary replacement, deletion and recovery archive cleanup. Exact visible bytes, +UUID/path mappings, retained bytes and absence of transaction files are checked. An intent is resubmitted only if nothing was durably saved; +resubmitting a partially recovered operation would hide journal defects. + +`crash-worker.ts` runs a complete SyncClient in a separate worker. The parent +terminates it at an explicit accepted-response checkpoint without `destroy()`; +no old timers, websocket handlers, or promises survive. Update cases bootstrap an existing UUID and assert the non-null parent and exact +Snapshot/Diff request type before termination. Its saved memory-disk image and persisted database seed a fresh runtime. The separate real-disk fixture +uses a test-only native exclusive-rename primitive. It is **not** a +production adapter or an adversarial `openat` sandbox: concurrent hostile changes +to ancestor directories and actual machine power cuts are outside its coverage. + +## Regression coverage + +The previously exposed WS queue, notified-rename UUID, uncertain-save replay, +and cross-directory durability failures are fixed and remain asserted by the +default harness. Scripted, seeded, event-delivery and engine regression clients +explicitly disable polling so it cannot hide a missing wakeup. Crash workers also +assert that this setting survives restart. Polling is enabled only in separate +tests that intentionally omit notifications and advance a fake clock. Identity +assertions distinguish documents even when bytes +are equal. The engine regressions additionally exercise retained recovery bytes, +durable offline notifications, permanently rejected requests, ignored namespace +conflicts, reverted remote updates, and concurrent lifecycle changes. + +Tests assert intended invariants, including bytes and UUID mappings, rather than +accepting the observed output as an oracle. The memory fault model and real-disk +checks complement one another; neither proves behavior under every real power cut. + +## September 2026 audit regressions + +`scripts/e2e.sh` now runs the Rust and client unit suites as well as the harness, +real-server protocol checks, scripted scenarios, and seeded workloads. A failing +unit suite makes the complete run fail. The audit tests are separate, individually +quoted arguments in the default scripts, with a runner regression guarding that wiring. + +The audit cases were first run with assertions for the required behavior and +observed failing before the corresponding fixes. Coverage is organized by the +original finding numbers: + +| Findings | Permanent coverage and resulting guarantee | +| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Engine and real-server lost-reply/rejected-retry tests preserve later content and rename undos; interrupted receipt saves and paged replay are tested after process replacement. | +| 2, 6 | Ignored/oversized unmanaged occupants keep their paths, including arrivals after scanning; submitted manifests reserve excluded server paths and remain valid. | +| 3, 12 | Oversized remote heads defer uploads; oversized local files are rejected before snapshot reads; scans retain hashes rather than encoded vault contents; content notifications cannot starve namespace scans. | +| 4 | A subprocess deadline detects synchronous conflict-allocation hangs caused by long extensions. | +| 5 | Persistent destination errors select a durable root fallback. The recovery sweep keeps that error active through every first and second crash boundary, checking exact bytes, identities and retained originals. | +| 7–9 | Bootstrap notifications adopt remote identities; settings updates resume enabled engines, disabling sync persists across reset, and initial vault bindings survive restart; disabled startup recovers local journals. | +| 10–11 | Fake-clock silent-socket recovery and real-server connection admission, slot release and handshake expiry. | +| 13 | Rust tests bound event count and aggregate bytes per page; client tests fold remote reversions before application and preserve the replay position across restart. | +| 14–15 | Metadata-only manifest reference validation and active-pool cleanup tests ensure unrelated payload decoding and checked-out transactions cannot block these operations. | +| 16 | Dirty cursor updates invalidate precise positions; disconnect/reset clear presented cursors. | +| 17 | Read-only configuration preservation and concurrent initialization verify that existing credentials are never rewritten or replaced. | + +The follow-up audit added these independently reproduced failures. Each behavior +was asserted before its fix; the default test commands include every regression. + +| Additional failure | Regression coverage | +| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Delayed content receipts crash after local deletion | `sync-audit.test.ts` verifies receipt consumption, preserved deletion, and cursor advancement without a local path. | +| Cursor reads restore presence after disconnect/reset | Cursor unit tests hold an active read and queue another update, retire the connection, then release both. | +| Failed WebSocket upgrades strand clients with polling disabled | `polling.test.ts` requires exact remote content and rename catchup over HTTP after a disconnect signal. | +| WebSocket construction errors escape startup/retry callbacks | Transport unit tests fail the first or second construction and require a successful subsequent connection. | +| Opening a locked vault blocks unrelated vaults | Rust tests hold a real SQLite writer lock during lazy opening; another vault must remain readable. Concurrent openings must share the configured pool, including after cancellation. | +| Failure clearing rejected requests leaves an enabled client stopped | Settings tests fail the second save, before and after commit, and require later notifications to wake synchronization. | +| Caller mutations bypass the checked settings snapshot | A settings test changes the caller's vault name and exclusion array while transports pause; persisted settings must retain the original input. | +| Cursor freshness uses metadata captured before an asynchronous read | A cursor test advances the document while the read is suspended and requires the old cursor to be marked outdated. | +| Throwing/silent socket closes leave live handlers after shutdown | Transport tests require disconnected status and no message delivery after a failed or timed-out close. | +| Handshake send errors escape the socket-open callback | A transport test drops the send during opening and requires safe retirement and retry. | +| Old connection cleanup deletes its replacement's cursors | A real-server overlapping reconnect test preserves the replacement; Rust tests also reject delayed updates from the old session. | + +These cases exercise transitions the earlier tests did not combine: a deletion +between submission and acknowledgement, disconnects during reads, failures after +the settings save, and two connections sharing a device identity. Happy-path +reconnect and single-save fault tests do not establish those guarantees. + +Replay pages contain at most 64 events and approximately 1 MiB of aggregate event +JSON. One atomic manifest can exceed that budget; it is sent alone so replay still +advances. `endEventId` identifies the page boundary while `headEventId` identifies +the vault head. Deploy the updated client and server together: older clients +require each batch to reach the vault head. Local partial replay folds retain only +latest document heads, the latest manifest, and matching outstanding receipts; +local files are reconciled only when catchup reaches the head. + +Permanent destination failures use `Recovered ` in the +vault root, resolving any occupied name. The journal retains the original bytes +when content is replaced. Permission or storage failures that also prevent that +fallback remain explicit recovery failures with the journal intact. + +A rejected retry retains an unconfirmed receipt until its request appears in the +event log. Absence from one catchup is not proof that an earlier in-flight attempt +cannot still commit. Receipt/artifact pruning remains an explicit operational concern; these tests do not claim +formal completeness or simulate arbitrary physical-media corruption. + +Reproduce the complete validation run from the repository root: + +```sh +E2E_ITERATIONS=100 E2E_SEED=8201 E2E_WORKERS=2 scripts/e2e.sh +``` + +## Sync regression coverage added after the September audit + +The normal test commands include these regressions: + +| Failure | Maintained coverage | +| ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Ignored occupant renamed into scope; incoming download discarded by unrelated notifications | `journal-protocol.test.ts` checks visible files, original IDs and actual server content. | +| Source path reused or moved during staging; editor save becomes a duplicate | `journal-interleavings.test.ts` injects moves, delete/recreate, move/recreate and content saves at every enumerated read, mutation and state-save boundary. Assertions specify bytes and document identities independently. | +| Journal/editor interaction followed by a crash | `journal-protocol.test.ts` crosses three editor operations with failures before/after state persistence, process replacement and simulated power loss, then checks another reset. | +| Restored server history falsely appears current | `server-restore.test.ts` restores actual SQLite backups and restarts Rust, covering equal/lower counters, dirty files, lost acknowledgements, interrupted recovery saves and old client state without checkpoints. | +| Oversized remote fallback and stale responses | `content-limits.test.ts` measures payload downloads, checks metadata-only responses and verifies highly fragmented local edits fall back to snapshots. | +| Adversarial diff work | Rust endpoint tests reject excessive operations without appending events and exercise Unicode reconstruction. | +| Observer exception kills sync | `observer.test.ts` runs throwing/rejecting subscribers in a child process and verifies a second sync; listener unit tests verify continued delivery. | +| Outgoing cursor version race | `outgoing-cursor.test.ts` changes version, identity, path and existence during a read, deliberately including identical bytes. | +| Unpersisted credentials | Rust config tests reject both missing users and missing user lists while preserving existing file bytes; initialization tests verify stable saved defaults. | +| Extensionless filenames enable merging | Extension unit tests include root/nested `md` and `txt` and dots in ancestor directories. | +| Corrupt vault blocks startup | Rust database regression opens a healthy vault alongside an invalid SQLite file and verifies the invalid file remains untouched. | + +The filesystem sweep reports the number of boundaries and injected interleavings. +A passing sweep covers this adapter model and these operations; arbitrary physical +media corruption and every possible editor/OS behavior remain outside its scope. diff --git a/frontend/test-support/canonical.ts b/frontend/test-support/canonical.ts new file mode 100644 index 00000000..0f30ffa4 --- /dev/null +++ b/frontend/test-support/canonical.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import type { StoredDatabase } from "sync-client"; +import { assertQuiescent, assertSameFiles } from "./oracles"; + +export interface CanonicalSnapshot { + headEventId: number; + fileManifest: { fileManifestId: number; entries: Record }; + documents: { documentId: string; vaultUpdateId: number }[]; +} +export interface InspectableClient { + files(): Map; + database(): Partial | undefined; +} + +export async function getJson(url: string, token: string): Promise { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + "Device-Id": "harness-observer" + }, + signal: AbortSignal.timeout(5_000) + }); + assert(response.ok, `Observer GET ${url}: HTTP ${response.status}`); + return (await response.json()) as T; +} + +/** Equality alone can pass when every replica loses the same bytes. Check the + * canonical UUIDs, content heads, receipts, journals and event cursor as well. + * Scenario-specific content/identity oracles still run separately. + */ +export async function assertCanonical( + clients: InspectableClient[], + vaultUrl: string, + token: string +): Promise { + assert.equal( + new Set(clients).size, + clients.length, + "Cannot compare a client to itself" + ); + const files = clients.map((client) => client.files()); + const states = clients.map((client) => client.database()); + assertSameFiles(files); + const snapshot = await getJson( + `${vaultUrl}/vault-snapshot`, + token + ); + const entries = snapshot.fileManifest.entries; + for (const id of Object.keys(entries)) + assert( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + id + ), + `Non-UUID document identity: ${id}` + ); + const heads = Object.fromEntries( + snapshot.documents.map((doc) => [doc.documentId, doc]) + ); + assert.equal( + Object.keys(heads).length, + snapshot.documents.length, + "Duplicate document heads" + ); + assert.deepEqual( + Object.keys(heads).sort(), + Object.keys(entries).sort(), + "Snapshot heads differ from membership" + ); + for (const [i, state] of states.entries()) { + assert(state?.initialized, `Client ${i} never initialized`); + assertQuiescent(state, entries, snapshot.headEventId); + assert.equal( + state.fileManifest?.fileManifestId, + snapshot.fileManifest.fileManifestId, + "Wrong manifest CAS parent" + ); + assert.deepEqual( + [...files[i].keys()].sort(), + Object.values(entries).sort(), + "Untracked or missing disk files" + ); + for (const id of Object.keys(entries)) { + assert( + state.documents?.[id]?.materialized, + `Unmaterialized document: ${id}` + ); + assert.equal( + state.documents?.[id]?.base?.vaultUpdateId, + heads[id]?.vaultUpdateId, + `Wrong content CAS parent: ${id}` + ); + } + } + for (const [id, path] of Object.entries(entries)) { + const doc = await getJson<{ contentBase64: string }>( + `${vaultUrl}/documents/${id}`, + token + ); + const serverBytes = new Uint8Array( + Buffer.from(doc.contentBase64, "base64") + ); + assert.deepEqual( + files[0].get(path), + serverBytes, + `Server bytes differ for ${id} at ${path}` + ); + const hash = createHash("sha256").update(serverBytes).digest("hex"); + for (const [i, state] of states.entries()) { + assert.equal( + state!.documents?.[id]?.base?.hash, + hash, + `Wrong base hash: client ${i}, ${id}` + ); + assert.equal( + state!.documents?.[id]?.observedHash, + hash, + `Wrong observed hash: client ${i}, ${id}` + ); + } + } + const events: { eventId: number; requestId: string }[] = []; + let after = 0; + do { + const batch = await getJson<{ + headEventId: number; + endEventId?: number; + events: { eventId: number; requestId: string }[]; + }>(`${vaultUrl}/events-since?after=${after}`, token); + assert.equal( + batch.headEventId, + snapshot.headEventId, + "Server changed while checking convergence" + ); + const end = batch.endEventId ?? batch.headEventId; + assert.equal( + end, + batch.events.at(-1)?.eventId ?? after, + "Event page cursor differs from its contents" + ); + if (after < snapshot.headEventId) + assert(end > after, "Event page made no progress"); + events.push(...batch.events); + after = end; + } while (after < snapshot.headEventId); + assert.deepEqual( + events.map((e) => e.eventId), + Array.from({ length: snapshot.headEventId }, (_, i) => i + 1), + "Server event log is not contiguous and totally ordered" + ); + assert.equal( + new Set(events.map((e) => e.requestId)).size, + events.length, + "An idempotent request emitted multiple events" + ); + return snapshot; +} diff --git a/frontend/test-support/content-limits.test.ts b/frontend/test-support/content-limits.test.ts new file mode 100644 index 00000000..baf510a4 --- /dev/null +++ b/frontend/test-support/content-limits.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +test("stale manifest reconciliation must not fetch excluded remote content", async () => { + const f = await fixture(); + try { + const id = randomUUID(), + big = randomUUID(); + await f.put("/documents/" + id, f.content("A")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.disk.userRename("a.md", "renamed.md"); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "renamed.md" + }); + let injected = false, + downloaded = 0; + f.setFetchHook(async (input, init) => { + const path = String(input); + if ( + !injected && + init?.method === "PUT" && + path.endsWith("/file-manifest") + ) { + injected = true; + await f.put("/documents/" + big, f.content("B".repeat(4096))); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "a.md", [big]: "big.bin" } + }); + } + const response = await fetch(input, init); + if ( + path.includes("/documents/" + big) && + !path.endsWith("/metadata") && + (!init?.method || init.method === "GET") + ) + downloaded += (await response.clone().arrayBuffer()).byteLength; + return response; + }); + await f.client.setSetting("isSyncEnabled", true); + + assert(injected); + assert(!f.disk.userFiles().has("big.bin")); + assert.equal(downloaded, 0); + } finally { + await f.dispose(); + } +}); + +test("stale content acknowledgements contain metadata without the excluded payload", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + await f.put("/documents/" + id, f.content("B".repeat(4096))); + const stale = await f.put("/documents/" + id, f.content("LOCAL")); + assert.equal(stale.type, "StaleBase"); + assert.equal(stale.contentSize, 4096); + assert.equal(stale.contentBase64, undefined); + const metadata = await f.get("/documents/" + id + "/metadata"); + assert.equal(metadata.contentSize, 4096); + assert.equal(metadata.contentBase64, undefined); + } finally { + await f.dispose(); + } +}); + +test("highly fragmented local edits use a snapshot within the server operation budget", async () => { + const { fixture } = await import("./sync-fixture"); + const { toStoredSnapshot } = await import( + "../sync-client/src/sync-operations/content" + ); + const f = await fixture({}); + f.contentCache.resize(1024 * 1024); + const original = Array.from({ length: 12_000 }, (_, i) => `word${i}`).join( + " " + ); + const edited = Array.from({ length: 12_000 }, (_, i) => + i % 2 ? `changed${i}` : `word${i}` + ).join(" "); + f.contentCache.put(1, bytes(original)); + const syncer = f.syncer as unknown as { + pushContent( + path: string, + version: number, + snapshot: unknown + ): Promise<{ type: string }>; + }; + const request = await syncer.pushContent( + "a.md", + 1, + await toStoredSnapshot({ content: bytes(edited) }) + ); + assert.equal(request.type, "Snapshot"); +}); diff --git a/frontend/test-support/engine-regressions.test.ts b/frontend/test-support/engine-regressions.test.ts new file mode 100644 index 00000000..517d1360 --- /dev/null +++ b/frontend/test-support/engine-regressions.test.ts @@ -0,0 +1,910 @@ +import assert from "node:assert/strict"; +import { PermanentSyncError } from "../sync-client/src/errors/errors"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { allocatePortablePath } from "../sync-client/src/utils/portable-path"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import type { LocalChange } from "../sync-client/src/sync-operations/local-changes"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const head = (id: string, version: number, content: string) => ({ + documentId: id, + vaultUpdateId: version, + contentSize: bytes(content).length, + updatedDate: "2026-09-19T00:00:00Z", + userId: "user", + deviceId: "device" +}); + +async function fixture( + contents: Record = { "a.md": "local" }, + service: Partial = {} +) { + const disk = new MemoryDisk(); + const initial = emptyState("test"); + initial.initialized = true; + for (const [path, content] of Object.entries(contents)) { + await disk.userWrite(path, bytes(content)); + const id = path.split(".")[0]; + initial.local[id] = path; + initial.documents[id] = { + materialized: true, + observedHash: (await toStoredSnapshot({ content: bytes(content) })) + .hash + }; + } + initial.fileManifest = { fileManifestId: 1, entries: { ...initial.local } }; + const persistence = new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + "test", + async () => (await persistence.load()).database as StoredDatabase + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + const changes: LocalChange[] = []; + const files = new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} } + ); + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteVaultUpdateReceived: new EventListeners() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket, + files, + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(1024), + { entries: changes, save: async () => {} } + ); + const internals = syncer as unknown as { + finishPending(): Promise; + preparePush(): Promise; + scan(): Promise; + incorporateEventBatch( + batch: import("../sync-client/src/services/types/EventBatch").EventBatch + ): Promise; + stopped: boolean; + incorporateFileManifest(remote: { + fileManifestId: number; + entries: Record; + }): Promise; + incorporateContent(remote: ReturnType): Promise; + }; + return { + disk, + database, + files, + settings, + syncer, + internals, + persistence, + changes + }; +} + +test("remote binary replacement retains displaced bytes outside the visible namespace", async () => { + const f = await fixture({ "a.bin": "private edit" }); + const next = structuredClone(f.database.state); + await f.files.apply(next, { + a: { + expected: await f.files.snapshot("a.bin"), + replacement: await toStoredSnapshot({ content: bytes("remote") }) + } + }); + assert.deepEqual(f.disk.userFiles(), new Map([["a.bin", bytes("remote")]])); + const retained = (await f.disk.listFilesRecursively()).filter((path) => + path.startsWith(".vault-link-sync/recovery/") + ); + assert( + ( + await Promise.all(retained.map((path) => f.disk.readSnapshot(path))) + ).some( + (value) => + value && + Buffer.from(value.content).toString() === "private edit" + ) + ); + await f.files.recover(); + assert.deepEqual( + await f.disk.listFilesRecursively(), + ["a.bin", ...retained].sort() + ); +}); + +test("remote deletion retains unsent bytes for recovery", async () => { + const f = await fixture(); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + assert.equal(f.disk.userFiles().size, 0); + const retained = await f.disk.listFilesRecursively(); + assert( + ( + await Promise.all(retained.map((path) => f.disk.readSnapshot(path))) + ).some( + (value) => + value && Buffer.from(value.content).toString() === "local" + ) + ); +}); + +test("rejected content recovery remains idempotent across a rename and network failure", async () => { + let offline = true; + const f = await fixture(undefined, { + getDocumentVersionContent: async () => { + if (offline) throw new Error("offline"); + return bytes("base"); + } + }); + const snapshot = (await f.files.snapshot("a.md"))!; + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.pending = { + type: "content", + documentId: "a", + snapshot, + request: { + requestId: "request", + parentVersionId: 1, + content: { type: "Snapshot", value: snapshot.contentBase64 } + }, + response: { + type: "StaleBase", + ...head("a", 2, "remote") + } + }; + await f.database.save(); + await f.disk.userRename("a.md", "renamed.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "renamed.md" + }); + await assert.rejects(f.internals.finishPending(), /offline/); + offline = false; + await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.equal(f.database.state.local.a, "renamed.md"); +}); + +test("conflict names respect component limits without imposing a total path limit", () => { + const wanted = "a".repeat(235) + ".md"; + const allocated = allocatePortablePath( + wanted, + "00000000-0000-4000-8000-000000000001", + { other: wanted } + ); + assert(Buffer.byteLength(allocated) <= 255); + assert(allocated.endsWith(".md")); + assert.notEqual(allocated, wanted); + const longPath = "folder/".repeat(100) + "note.md"; + assert.equal(allocatePortablePath(longPath, "id", {}), longPath); +}); + +test("an unmaterialized file with a directory at its path is deconflicted", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("remote") } + ); + await f.disk.userWrite("folder/note.md", bytes("local")); + f.database.state.local.a = "folder"; + f.database.state.documents.a = { materialized: false }; + f.database.state.fileManifest.entries.a = "folder"; + await f.database.save(); + await f.internals.incorporateContent(head("a", 2, "remote")); + assert( + [...f.disk.userFiles().values()].some( + (value) => Buffer.from(value).toString() === "remote" + ) + ); + assert( + [...f.disk.userFiles().values()].some( + (value) => Buffer.from(value).toString() === "local" + ) + ); +}); + +test("ignored materialized content survives remote deletion", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); + assert.deepEqual(f.database.state.fileManifest.entries, {}); +}); + +test("unfinished bootstrap cannot be rebound to another vault", async () => { + const database = emptyState(JSON.stringify(["http://example.test", "old"])); + database.bootstrap = { + headEventId: 1, + fileManifest: { fileManifestId: 1, entries: {} }, + documents: [] + }; + const persistence = new MemoryPersistence({ + settings: { remoteUri: "http://example.test", vaultName: "old" }, + database + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence + }); + try { + await assert.rejects( + client.setSettings({ vaultName: "new" }), + /separate state store/ + ); + assert.equal(persistence.snapshot().settings!.vaultName, "old"); + } finally { + await client.destroy(); + } +}); + +test("local swap arriving after a remote journal is saved preserves both identities and contents", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let fired = false; + f.persistence.boundary = async (label) => { + if ( + fired || + label !== "durable:save" || + !f.persistence.snapshot().database?.application + ) + return; + fired = true; + for (const [oldPath, relativePath] of [ + ["a.md", "tmp.md"], + ["b.md", "a.md"], + ["tmp.md", "b.md"] + ]) { + await f.disk.userRename(oldPath, relativePath); + await f.syncer.syncLocallyUpdatedFile({ oldPath, relativePath }); + } + }; + const next = structuredClone(f.database.state); + next.local = { a: "remote-a.md", b: "remote-b.md" }; + await f.files.apply(next); + assert(fired); + assert.equal( + Buffer.from( + f.disk.userFiles().get(f.database.state.local.a)! + ).toString(), + "A" + ); + assert.equal( + Buffer.from( + f.disk.userFiles().get(f.database.state.local.b)! + ).toString(), + "B" + ); + assert.deepEqual(f.database.state.local, { a: "b.md", b: "a.md" }); +}); + +test("a rejected upload is retained locally and does not block another file or a corrected snapshot", async () => { + let reject = true; + const sent: string[] = []; + const f = await fixture( + { "a.md": "too large", "b.md": "other" }, + { + putFileContent: async (id) => { + sent.push(id); + if (id === "a" && reject) + throw new PermanentSyncError("HTTP 413"); + return { + type: "Accepted", + ...head(id, sent.length + 2, "small") + }; + } + } + ); + f.internals.stopped = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.match(f.database.state.documents.a.rejected!.message, /413/); + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(sent, ["a", "b"]); + assert.equal(await f.internals.preparePush(), false); + await f.disk.userWrite("a.md", bytes("small")); + reject = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(sent, ["a", "b", "a"]); + assert.equal(f.database.state.documents.a.rejected, undefined); +}); + +test("a transient upload failure retries the exact request and recovers its acknowledgement", async () => { + const requests: unknown[] = []; + const f = await fixture(undefined, { + putFileContent: async (_id, request) => { + requests.push(structuredClone(request)); + if (requests.length === 1) throw new Error("lost upload reply"); + return { type: "Accepted", ...head("a", 2, "local") }; + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + await assert.rejects(f.internals.finishPending(), /lost upload reply/); + assert(f.database.state.pending); + await f.internals.finishPending(); + assert.equal(requests.length, 2); + assert.deepEqual(requests[1], requests[0]); + assert.equal(f.database.state.documents.a.base!.vaultUpdateId, 2); + assert.equal(f.database.state.documents.a.rejected, undefined); + assert.equal(f.database.state.pending, undefined); +}); + +test("a persisted upload rejection finishes locally after a recovery write fails", async () => { + let uploads = 0; + const f = await fixture(undefined, { + putFileContent: async () => { + uploads++; + throw new PermanentSyncError("HTTP 413"); + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert(pending); + const { requestId } = pending.request; + const retained = `.vault-link-sync/requests/${requestId}.content`; + const { boundary } = f.disk; + f.disk.boundary = (label): void => { + if (label === `before:write:${retained}`) + throw new Error("recovery write failed"); + }; + await assert.rejects(f.internals.finishPending(), /recovery write failed/); + assert.equal( + f.persistence.snapshot().database?.pending?.rejection, + "HTTP 413" + ); + f.disk.boundary = boundary; + await f.internals.finishPending(); + assert.equal(uploads, 1); + assert.equal(f.database.state.pending, undefined); + assert.equal(f.database.state.documents.a.rejected?.message, "HTTP 413"); + assert.deepEqual( + (await f.disk.readSnapshot(retained))?.content, + bytes("local") + ); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); +}); + +test("event replay incorporates an earlier accepted upload after a permanent retry rejection", async () => { + const f = await fixture(undefined, { + putFileContent: async () => { + throw new PermanentSyncError( + "HTTP 413 after proxy reconfiguration" + ); + }, + getDocumentVersionContent: async () => bytes("local") + }); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert(pending); + const { requestId } = pending.request; + await f.internals.finishPending(); + await assert.rejects(f.syncer.waitUntilFinished(), /413/); + f.database.state.lastSeenUpdateId = 1; + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "local") + } + ] + }); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + assert.equal(f.database.state.pending, undefined); + assert.equal(await f.internals.preparePush(), false); + await f.syncer.waitUntilFinished(); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); +}); + +test("rejected manifests can be replaced after editing the local namespace", async () => { + let reject = true; + const f = await fixture( + {}, + { + pushFileManifest: async () => { + if (reject) throw new PermanentSyncError("HTTP 413"); + return { type: "Accepted", fileManifestId: 3 }; + } + } + ); + f.database.state.fileManifest.entries = { a: "old.md" }; + f.database.state.pending = { + type: "fileManifest", + request: { + requestId: "request", + parentFileManifestId: 1, + entries: { a: "new.md" } + } + }; + f.database.state.local = { a: "new.md" }; + f.database.state.documents.a = { + materialized: false, + base: { ...head("a", 1, "x"), hash: "hash" } + }; + await f.database.save(); + await f.internals.finishPending(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + f.database.state.local = {}; + reject = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(f.database.state.fileManifest.entries, {}); +}); + +test("offline delete/recreate identities survive restarting before any network connection", async () => { + const key = JSON.stringify(["http://offline.test", "test"]); + const initial = emptyState(key); + initial.initialized = true; + initial.local.a = "a.md"; + initial.documents.a = { materialized: true }; + initial.fileManifest = { fileManifestId: 1, entries: { a: "a.md" } }; + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("same")); + const persistence = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + const first = await SyncClient.create({ fs: disk, persistence }); + await disk.userDelete("a.md"); + await first.syncLocallyDeletedFile("a.md"); + await disk.userWrite("a.md", bytes("same")); + await first.syncLocallyCreatedFile("a.md"); + await first.destroy(); + const second = await SyncClient.create({ fs: disk, persistence }); + try { + const internals = second as unknown as { + syncer: { scan(): Promise }; + database: Database; + }; + await internals.syncer.scan(); + assert.equal(internals.database.state.local.a, undefined); + assert.equal( + Object.values(internals.database.state.local).filter( + (path) => path === "a.md" + ).length, + 1 + ); + } finally { + await second.destroy(); + } +}); + +test("offline notifications bind an otherwise empty client to its vault", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("local")); + const persistence = new MemoryPersistence({ + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + const client = await SyncClient.create({ fs: disk, persistence }); + await client.syncLocallyCreatedFile("a.md"); + await assert.rejects( + client.setSettings({ vaultName: "other" }), + /separate state store/ + ); + await client.destroy(); + const stored = persistence.snapshot(); + stored.settings!.vaultName = "other"; + await persistence.save(stored); + await assert.rejects( + SyncClient.create({ fs: disk, persistence }), + /another vault/ + ); +}); + +test("unignoring a deleted file resumes the deferred remote deletion without resurrection", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert.equal(f.disk.userFiles().size, 0); + assert.deepEqual(f.database.state.local, {}); +}); + +test("an ignored file does not move when a remote rename targets its path", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "c.md", b: "a.md" } + }); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes("A")], + ["b.md", bytes("B")] + ]) + ); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes("A")], + ["b.md", bytes("B")] + ]) + ); +}); + +for (const kind of ["path", "content"] as const) { + test(`a remote ${kind} change that was reverted before observation preserves the local edit`, async () => { + const f = await fixture( + { "a.md": "local" }, + { + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes(vaultUpdateId === 2 ? "temporary" : "base") + } + ); + f.database.state.lastSeenUpdateId = 1; + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + if (kind === "path") { + await f.disk.userRename("a.md", "c.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "c.md" + }); + } + await f.internals.incorporateEventBatch({ + headEventId: 3, + events: [2, 3].map((id) => + kind === "path" + ? { + eventId: id, + requestId: String(id), + type: "fileManifest", + fileManifest: { + fileManifestId: id, + entries: { a: id === 2 ? "b.md" : "a.md" } + } + } + : { + eventId: id, + requestId: String(id), + type: "content", + document: head( + "a", + id, + id === 2 ? "temporary" : "base" + ) + } + ) + }); + assert.equal( + f.database.state.local.a, + kind === "path" ? "c.md" : "a.md" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))!.content + ).toString(), + "local" + ); + assert.equal(f.database.state.lastSeenUpdateId, 3); + }); +} + +test("concurrent settings updates cannot bind the engine to a different vault than its settings", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence(); + const client = await SyncClient.create({ fs: disk, persistence }); + try { + await Promise.all([ + client.setSettings({ vaultName: "changed" }), + client.setSettings({ maxFileSizeMB: 20 }) + ]); + const stored = persistence.snapshot(); + assert.equal( + stored.database!.vaultKey, + JSON.stringify([ + stored.settings!.remoteUri, + stored.settings!.vaultName + ]) + ); + assert.equal(stored.settings!.maxFileSizeMB, 20); + } finally { + await client.destroy(); + } +}); + +test("destroy waits for an in-progress offline notification to become durable", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence(); + const client = await SyncClient.create({ fs: disk, persistence }); + await disk.userWrite("a.md", bytes("local")); + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + persistence.boundary = async (label) => { + if (label === "before:save") { + reached.resolve(); + await release.promise; + } + }; + const notification = client.syncLocallyCreatedFile("a.md"); + await reached.promise; + let destroyed = false; + const destroying = client.destroy().then(() => { + destroyed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(destroyed, false); + release.resolve(); + await Promise.all([notification, destroying]); +}); + +test("ignoring a locally renamed file defers its move without forgetting the local rename", async () => { + const f = await fixture(); + f.database.state.documents.a.base = { + ...head("a", 1, "local"), + hash: f.database.state.documents.a.observedHash! + }; + await f.disk.userRename("a.md", "ignored.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "ignored.md" + }); + await f.settings.setSettings({ ignorePatterns: ["ignored.md"] }); + await f.internals.scan(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "a.md" } + }); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert.equal(f.database.state.local.a, "ignored.md"); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("ignored.md"))!.content + ).toString(), + "local" + ); +}); + +test("a notified deletion of an ignored file does not delete its server membership", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.disk.userDelete("a.md"); + await f.syncer.syncLocallyDeletedFile("a.md"); + await f.internals.scan(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + assert.equal(f.database.state.local.a, undefined); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert(await f.internals.preparePush()); + assert.equal(f.database.state.pending?.type, "fileManifest"); +}); + +test("an incoming ignored document cannot force an existing ignored file to move", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md", "*conflict*"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "b.md", other: "a.md" } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); + assert.equal( + new Set(Object.values(f.database.state.local)).size, + Object.keys(f.database.state.local).length + ); +}); + +for (const boundary of ["visible", "durable"] as const) { + for (const action of [ + "move-installed", + "recreate-staged-source" + ] as const) { + test(`${action} at a ${boundary} filesystem boundary keeps original identities and all bytes`, async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let changed = false; + f.disk.boundary = async (label) => { + if (changed || !label.startsWith(`${boundary}:rename:`)) return; + if (action === "move-installed" && label.endsWith("->b.md")) { + changed = true; + await f.disk.userRename("b.md", "later.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "later.md" + }); + } else if ( + action === "recreate-staged-source" && + label.includes("rename:a.md->") + ) { + changed = true; + await f.disk.userWrite("a.md", bytes("new")); + await f.syncer.syncLocallyCreatedFile("a.md"); + } + }; + const next = structuredClone(f.database.state); + next.local = { a: "b.md", b: "a.md" }; + next.fileManifest = { + fileManifestId: 2, + entries: { ...next.local } + }; + await f.files.apply(next); + assert(changed); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))! + .content + ).toString(), + "A" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.b))! + .content + ).toString(), + "B" + ); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((value) => Buffer.from(value).toString()) + .sort(), + action === "move-installed" ? ["A", "B"] : ["A", "B", "new"] + ); + f.disk.crash(true); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))! + .content + ).toString(), + "A" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.b))! + .content + ).toString(), + "B" + ); + }); + } +} + +test("rejected content is reported to callers and an explicit retry clears the rejection", async () => { + const f = await fixture(undefined, { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + await f.internals.finishPending(); + await assert.rejects(f.syncer.waitUntilFinished(), /413/); + assert(f.syncer.isBusy); + await f.syncer.retryRejectedRequests(); + assert(await f.internals.preparePush()); +}); + +test("notification saves cannot roll back an uncertain engine commit, even if the first reload fails", async () => { + const key = JSON.stringify(["http://offline.test", "test"]); + const initial = emptyState(key); + initial.initialized = true; + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + let failLoad = false; + const persistence = { + load: async () => { + if (failLoad) { + failLoad = false; + throw new Error("reload failed"); + } + return store.load(); + }, + save: async (state: Parameters[0]) => + store.save(state) + }; + const client = await SyncClient.create({ fs: disk, persistence }); + try { + const internals = client as unknown as { + database: Database; + syncer: Syncer; + }; + const next = structuredClone(initial); + next.lastSeenUpdateId = 2; + store.boundary = (label) => { + if (label === "durable:save") { + store.boundary = () => {}; + throw new Error("durable save failed"); + } + }; + await assert.rejects( + internals.database.commit(next), + /durable save failed/ + ); + await disk.userWrite("a.md", bytes("new")); + failLoad = true; + await assert.rejects( + client.syncLocallyCreatedFile("a.md"), + /reload failed/ + ); + await internals.syncer.flushLocalChanges(); + assert.equal(store.snapshot().database!.lastSeenUpdateId, 2); + await internals.database.recoverPersistence(); + assert.equal(internals.database.state.lastSeenUpdateId, 2); + } finally { + await client.destroy(); + } +}); + +test( + "a settings listener can await client shutdown without deadlocking the lifecycle lock", + { timeout: 1_000 }, + async () => { + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: new MemoryPersistence() + }); + let notified = false; + client.onSettingsChanged.add(async () => { + await client.destroy(); + notified = true; + }); + await client.setSettings({ maxFileSizeMB: 20 }); + assert(notified); + } +); diff --git a/frontend/test-support/event-delivery.test.ts b/frontend/test-support/event-delivery.test.ts new file mode 100644 index 00000000..a31d58e9 --- /dev/null +++ b/frontend/test-support/event-delivery.test.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import type { EventBatch } from "../sync-client/src/services/types/EventBatch"; +import type { EventRecord } from "../sync-client/src/services/types/EventRecord"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +for (const [count, delivery] of [ + [1, "before the loop"], + [3, "before the loop"], + [3, "during the first batch"] +] as const) { + test( + `${count} WS batches delivered ${delivery} drain without polling or another notification`, + { timeout: 5_000 }, + async () => { + const logger = new Logger(); + const initial = emptyState("event-delivery"); + initial.initialized = true; + const persistence = new MemoryPersistence({ database: initial }); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + initial.vaultKey, + async () => + (await persistence.load()).database as StoredDatabase + ); + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + assert.equal(settings.getSettings().syncIntervalMs, 0); + const disk = new MemoryDisk(); + const initializing = Promise.withResolvers(); + const initialized = Promise.withResolvers(); + const firstBatch = Promise.withResolvers(); + const continueBatch = Promise.withResolvers(); + let heldBatch = false; + const applied: number[] = []; + persistence.boundary = async (label) => { + if (label !== "durable:save") return; + const cursor = + persistence.snapshot().database!.lastSeenUpdateId; + if (cursor && applied.at(-1) !== cursor) applied.push(cursor); + if ( + delivery === "during the first batch" && + cursor === 1 && + !heldBatch + ) { + heldBatch = true; + firstBatch.resolve(); + await continueBatch.promise; + } + }; + const config = { + initialize: async () => { + initializing.resolve(); + await initialized.promise; + }, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const events: EventRecord[] = Array.from( + { length: count }, + (_, index) => ({ + eventId: index + 1, + requestId: `manifest-${index + 1}`, + type: "fileManifest", + fileManifest: { fileManifestId: index + 1, entries: {} } + }) + ); + const service = { + // A correct client may drain the queue or catch up over HTTP. Either + // must reach the same head without the test manually waking it. + events: async (after: number): Promise => ({ + headEventId: count, + events: events.filter((event) => event.eventId > after) + }) + } as unknown as SyncService; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => unknown + >() + }; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service, + websocket as unknown as WebSocketManager, + new FileOperations(disk.session(), database, config), + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(1024) + ); + const deliver = async (event: EventRecord) => { + await websocket.onRemoteVaultUpdateReceived.triggerAsync({ + headEventId: event.eventId, + events: [event] + }); + }; + try { + syncer.start(); + await initializing.promise; + if (delivery === "before the loop") { + for (const event of events) await deliver(event); + initialized.resolve(); + } else { + await deliver(events[0]); + initialized.resolve(); + await firstBatch.promise; + for (const event of events.slice(1)) await deliver(event); + continueBatch.resolve(); + } + await syncer.waitUntilFinished(); + assert.equal( + database.state.lastSeenUpdateId, + count, + "Client became idle with already-delivered events unapplied" + ); + assert.deepEqual( + applied, + events.map((event) => event.eventId), + "Every event must be committed in order" + ); + assert.equal(database.state.fileManifest.fileManifestId, count); + assert.equal(database.state.application, undefined); + assert.equal(database.state.pending, undefined); + assert.equal(syncer.isBusy, false); + } finally { + initialized.resolve(); + continueBatch.resolve(); + await syncer.stop(); + } + } + ); +} diff --git a/frontend/test-support/harness.test.ts b/frontend/test-support/harness.test.ts new file mode 100644 index 00000000..e4604b4b --- /dev/null +++ b/frontend/test-support/harness.test.ts @@ -0,0 +1,316 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { NetworkFaults, classifyRequest } from "./network"; +import { + ContentLedger, + assertManifest, + assertSameFiles, + pathDecision +} from "./oracles"; +import { Random } from "./random"; +import { TestRunner } from "../deterministic-tests/src/test-runner"; +import type { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { ManagedWebSocketFactory } from "../deterministic-tests/src/managed-websocket"; + +const bytes = (text: string) => new TextEncoder().encode(text); + +test("consistency oracle rejects self comparison, divergent paths and invalid UTF-8 bytes", () => { + const first = new Map([["a.bin", new Uint8Array([0xff])]]); + assert.throws(() => assertSameFiles([first, first]), /itself/); + assert.throws(() => + assertSameFiles([first, new Map([["a.bin", new Uint8Array([0xfe])]])]) + ); + assert.throws(() => assertSameFiles([first, new Map()])); + assertSameFiles([first, structuredClone(first)]); +}); + +test("ledger catches shared loss, duplicated bytes, and unrelated loss in delete workloads", () => { + const ledger = new ContentLedger(); + ledger.add("first-marker"); + ledger.add("second-marker"); + ledger.removedBy("first-marker", "explicit delete of doc A"); + assert.throws( + () => ledger.assertPreserved(new Map()), + /second-marker.*lost/ + ); + assert.throws( + () => + ledger.assertPreserved( + new Map([["a", bytes("second-marker second-marker")]]) + ), + /duplicated/ + ); + ledger.assertPreserved(new Map([["a", bytes("second-marker")]])); +}); + +test("memory adapter enforces v4 operations and snapshot ownership", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("folder/a", bytes("A")); + const snap = (await disk.readSnapshot("folder/a"))!; + snap.content[0] = 0; + assert.equal( + Buffer.from((await disk.readSnapshot("folder/a"))!.content).toString(), + "A" + ); + await assert.rejects( + disk.write("folder/a", { content: bytes("B") }), + /exists/ + ); + await disk.userWrite("folder/b", bytes("B")); + await assert.rejects(disk.rename("folder/a", "folder/b"), /exists/); + await assert.rejects(disk.delete("folder/a"), /regular file/); + await assert.rejects(disk.delete("folder"), /regular file/); + await assert.rejects(disk.readSnapshot("../outside"), /Unsafe/); + await assert.rejects(disk.stat("folder/a/child"), /ancestor/); + await assert.rejects(disk.readSnapshot("folder"), /directory/); +}); + +for (const phase of ["before", "visible", "durable"] as const) { + for (const powerLoss of [false, true]) { + test(`${powerLoss ? "power loss" : "process death"} at ${phase} rename boundary`, async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a", bytes("A")); + const session = disk.session(); + let fired = false; + disk.boundary = (label) => { + if (label === `${phase}:rename:a->b`) { + fired = true; + disk.crash(powerLoss); + throw new Error("crash"); + } + }; + await assert.rejects(session.rename("a", "b"), /crash/); + assert(fired); + disk.boundary = () => {}; + const moved = + phase === "durable" || (phase === "visible" && !powerLoss); + assert.deepEqual([...disk.userFiles().keys()], [moved ? "b" : "a"]); + assert.throws( + () => session.write("stale", { content: bytes("bad") }), + /crashed/ + ); + }); + } +} + +test("persistence has deep ownership and both old/new uncertain save outcomes", async () => { + for (const phase of ["before", "durable"]) { + const store = new MemoryPersistence({ settings: { vaultName: "old" } }); + const loaded = await store.load(); + loaded.settings!.vaultName = "mutated"; + assert.equal(store.snapshot().settings!.vaultName, "old"); + store.boundary = (label) => { + if (label === `${phase}:save`) throw new Error("save failed"); + }; + await assert.rejects( + store.save({ settings: { vaultName: "new" } }), + /save failed/ + ); + assert.equal( + store.snapshot().settings!.vaultName, + phase === "before" ? "old" : "new" + ); + } +}); + +test("CAS hooks match v4 create, update and manifest and preserve retry payload", async () => { + for (const kind of ["create", "content", "manifest"] as const) { + const net = new NetworkFaults(); + const url = `http://test/vaults/test/${kind === "manifest" ? "file-manifest" : "documents/uuid"}`; + const body = { + requestId: "same", + parentVersionId: kind === "create" ? null : 3 + }; + const init = { method: "PUT", body: JSON.stringify(body) }; + assert.equal(classifyRequest(url, init)?.kind, kind); + let committed = 0; + const wrapped = net.wrap(async () => { + committed++; + return new Response('{"type":"Accepted"}'); + }); + net.arm(kind); + await assert.rejects(wrapped(url, init), /Injected after-commit/); + await net.wait(); + assert.throws(() => net.assertConsumed(), /never retried/); + assert.equal(committed, 1); + await wrapped(url, init); + net.assertConsumed(); + await assert.rejects( + wrapped(url, { + ...init, + body: JSON.stringify({ ...body, changed: true }) + }), + /Retry changed/ + ); + } + assert.equal( + classifyRequest("http://test/documents", { method: "POST" }), + undefined + ); +}); + +test("unconsumed and before-send faults cannot silently pass", async () => { + const net = new NetworkFaults(); + net.arm("manifest", "before"); + assert.throws(() => net.assertConsumed(), /never fired/); + let called = false; + await assert.rejects( + net.wrap(async () => { + called = true; + return new Response(); + })("http://test/file-manifest", { + method: "PUT", + body: '{"requestId":"r"}' + }) + ); + assert(!called); + assert.throws(() => net.assertConsumed(), /never retried/); + await net.wrap(async () => new Response())("http://test/file-manifest", { + method: "PUT", + body: '{"requestId":"r"}' + }); + net.assertConsumed(); +}); + +test("HTTP observation checkpoint blocks catchup, resumes, and respects abort", async () => { + const network = new NetworkFaults(); + const wrapped = network.wrap(async () => new Response("{}")); + network.pauseObservation(); + let delivered = false; + const request = wrapped("http://test/events-since?after=0").then(() => { + delivered = true; + }); + await network.waitForObservation(); + assert(!delivered, "Catchup bypassed the observation checkpoint"); + network.resumeObservation(); + await request; + network.assertConsumed(); + network.pauseObservation(); + const controller = new AbortController(); + const interrupted = wrapped("http://test/vault-snapshot", { + signal: controller.signal + }); + const rejected = assert.rejects(interrupted, /abort/i); + await network.waitForObservation(); + controller.abort(); + await rejected; + network.resumeObservation(); + network.assertConsumed(); +}); + +test("independent path decision truth table covers create, delete and rename", () => { + for (const base of [undefined, "a", "b"]) + for (const local of [undefined, "a", "b"]) + for (const remote of [undefined, "a", "b"]) { + const result = pathDecision(base, local, remote); + assert.equal(result, remote !== base ? remote : local); + } + assertManifest({ a: "folder/é.md", b: "other.bin" }); + for (const entries of [ + { a: "A", b: "a" }, + { a: "é", b: "e\u0301" }, + { a: "x", b: "x/y" } + ]) + assert.throws(() => assertManifest(entries)); +}); + +test("seeded actions are repeatable, including seed zero", () => { + for (const seed of [0, 1, 12345, 0xffffffff]) { + const first = new Random(seed); + const second = new Random(seed); + assert.deepEqual( + Array.from({ length: 100 }, () => first.int(100)), + Array.from({ length: 100 }, () => second.int(100)) + ); + } +}); + +test("runner cleanup errors fail an otherwise successful test", async () => { + const server = { + isRunning: () => true, + resume: () => {} + } as unknown as ServerControl; + const runner = new TestRunner( + server, + new Logger(), + "unused", + "http://unused" + ); + let cleaned = 0; + Object.assign(runner, { + initializeAgents: async () => {}, + agents: [ + { + clientId: 0, + database: () => ({ evidence: "before cleanup" }), + disk: { image: () => ({ bytes: [0, 255] }) }, + network: { requests: [{ body: { requestId: "original" } }] }, + cleanup: async () => { + cleaned++; + throw new Error("late background error"); + } + } + ] + }); + const result = await runner.runTest("cleanup-probe", { + clients: 2, + steps: [] + }); + assert.equal(result.success, false); + assert.match(result.error!, /late background error/); + assert.equal(cleaned, 1, "Failed cleanup was run twice"); + assert.deepEqual(result.diagnostics, [ + { + client: 0, + database: { evidence: "before cleanup" }, + disk: { bytes: [0, 255] }, + requests: [{ body: { requestId: "original" } }] + } + ]); +}); + +test("disposed harness rejects late websocket reconnects", async () => { + const factory = new ManagedWebSocketFactory(); + const Socket = factory.constructorFn; + await factory.finish(); + assert.throws(() => new Socket("ws://unused"), /after harness cleanup/); +}); + +test("runner preserves primary failure and cleanup diagnostics", async () => { + const server = { + isRunning: () => true, + resume: () => {} + } as unknown as ServerControl; + const runner = new TestRunner( + server, + new Logger(), + "unused", + "http://unused" + ); + Object.assign(runner, { + initializeAgents: async () => {}, + executeStep: async () => { + throw new Error("primary assertion failed"); + }, + agents: [ + { + clientId: 0, + database: () => ({}), + disk: { image: () => ({}) }, + network: { requests: [] }, + cleanup: async () => { + throw new Error("cleanup failed too"); + } + } + ] + }); + const result = await runner.runTest("cleanup-probe", { + clients: 2, + steps: [{ type: "barrier" }] + }); + assert.equal(result.success, false); + assert.match(result.error!, /primary assertion failed/); + assert.match(result.error!, /cleanup failed too/); +}); diff --git a/frontend/test-support/journal-interleavings.test.ts b/frontend/test-support/journal-interleavings.test.ts new file mode 100644 index 00000000..93f0bbb0 --- /dev/null +++ b/frontend/test-support/journal-interleavings.test.ts @@ -0,0 +1,407 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, head } from "./sync-fixture"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; + +for (const origin of ["bootstrap", "history recovery"]) + test(`cancelling a replacement retains the ${origin} merge context`, async () => { + const f = await fixture( + { "a.md": "BASE\n" }, + { getDocumentVersionContent: async () => bytes("REMOTE\nBASE\n") } + ); + const recoveryBase = await toStoredSnapshot({ + content: bytes("BASE\n") + }); + if (origin === "bootstrap") + f.database.state.documents.a.bootstrap = true; + else f.database.state.documents.a.recoveryBase = recoveryBase; + await f.database.save(); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("BASE\nLOCAL\n")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + await f.internals.incorporateContent(head("a", 2, "REMOTE\nBASE\n")); + assert(injected); + const deferred = f.database.state.documents.a; + assert.equal(deferred.base, undefined); + if (origin === "bootstrap") assert.equal(deferred.bootstrap, true); + else assert.deepEqual(deferred.recoveryBase, recoveryBase); + f.disk.boundary = () => {}; + await f.settings.setSetting("ignorePatterns", []); + await f.internals.incorporateFileManifest( + f.database.state.fileManifest + ); + const content = Buffer.from( + f.disk.userFiles().get("private.md")! + ).toString(); + assert.equal(content.match(/REMOTE/g)?.length, 1); + assert.equal(content.match(/LOCAL/g)?.length, 1); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + assert.equal(f.database.state.documents.a.recoveryBase, undefined); + }); + +for (const timing of ["before", "durable"]) + test(`a save that exceeds the size limit ${timing} staging keeps its exact bytes`, async () => { + const f = await fixture({ "a.bin": "PUBLIC" }); + await f.settings.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + const large = "PRIVATE".repeat(100); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + `${timing}:rename:a.bin->.vault-link-sync/transactions/` + ) + ) + return; + injected = true; + await f.disk.userWrite("a.bin", bytes(large)); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.bin" }); + }; + await f.files.apply(structuredClone(f.database.state), { + a: { + expected: await toStoredSnapshot({ content: bytes("PUBLIC") }), + replacement: await toStoredSnapshot({ + content: bytes("REMOTE") + }) + } + }); + assert(injected); + assert.equal( + Buffer.from(f.disk.userFiles().get("a.bin")!).toString(), + large + ); + assert.equal(f.database.state.excluded?.a, "a.bin"); + assert.deepEqual(f.database.state.local, { a: "a.bin" }); + }); + +test("an excluded move after installation survives an uncertain journal acknowledgement", async () => { + const f = await fixture({ "a.md": "PUBLIC" }); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + let moved = false, + crashed = false; + f.disk.boundary = async (label) => { + if ( + moved || + !label.startsWith("durable:rename:") || + !label.endsWith(".output->a.md") + ) + return; + moved = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("SECRET")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + f.persistence.boundary = (label) => { + if (!moved || crashed || label !== "durable:save") return; + crashed = true; + throw new Error("lost journal acknowledgement"); + }; + await assert.rejects( + f.files.apply(structuredClone(f.database.state), { + a: { + expected: await toStoredSnapshot({ content: bytes("PUBLIC") }), + replacement: await toStoredSnapshot({ + content: bytes("PUBLIC REMOTE") + }) + } + }), + /lost journal acknowledgement/ + ); + assert(moved && crashed); + f.disk.boundary = () => {}; + f.persistence.boundary = () => {}; + await f.database.recoverPersistence(); + await f.files.recover(); + assert.deepEqual(f.database.state.local, { a: "private.md" }); + assert.equal( + Buffer.from(f.disk.userFiles().get("private.md")!).toString(), + "SECRET" + ); +}); + +test("moving a tracked file into an ignored path during a journal leaves it protected", async () => { + const f = await fixture({ "a.md": "PUBLIC", "b.md": "OTHER" }); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + const next = structuredClone(f.database.state); + next.local.b = "remote-b.md"; + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("SECRET")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + await f.files.apply(next); + assert(injected); + assert.equal( + Buffer.from(f.disk.userFiles().get("private.md") ?? []).toString(), + "SECRET" + ); + assert.equal(f.database.state.local.a, "private.md"); + assert.equal(f.database.state.excluded?.a, "a.md"); +}); + +test("a directory move protects each ignored descendant during a journal", async () => { + const f = await fixture({ "folder/a.md": "PUBLIC", "b.md": "OTHER" }); + await f.settings.setSetting("ignorePatterns", ["private/**"]); + const next = structuredClone(f.database.state); + next.local.b = "remote-b.md"; + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("folder/a.md", "private/a.md"); + await f.disk.userWrite("private/a.md", bytes("SECRET")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "folder", + relativePath: "private" + }); + }; + await f.files.apply(next); + assert(injected); + assert.equal( + Buffer.from(f.disk.userFiles().get("private/a.md") ?? []).toString(), + "SECRET" + ); + assert.equal(f.database.state.local["folder/a"], "private/a.md"); + assert.equal(f.database.state.excluded?.["folder/a"], "folder/a.md"); +}); + +test("a later editor save replaces earlier unsynced text after a merge was prepared", async () => { + const f = await fixture({ "a.md": "base" }); + const next = structuredClone(f.database.state); + let saves = 0; + f.disk.boundary = async (label) => { + if (saves === 0 && label.startsWith("before:rename:a.md->")) { + saves++; + await f.disk.userWrite("a.md", bytes("base ONE")); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } else if ( + saves === 1 && + label.startsWith("before:write:") && + label.endsWith(".output") + ) { + saves++; + await f.disk.userWrite("a.md", bytes("base TWO")); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + }; + await f.files.apply(next, { + a: { + expected: await toStoredSnapshot({ content: bytes("base") }), + replacement: await toStoredSnapshot({ + content: bytes("base REMOTE") + }) + } + }); + assert.equal(saves, 2); + assert.equal( + Buffer.from(f.disk.userFiles().get("a.md")!).toString(), + "base TWO REMOTE" + ); + assert.deepEqual(f.database.state.local, { a: "a.md" }); +}); + +test("a content save during remote application remains on the same document", async () => { + const f = await fixture({ "a.md": "base" }); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + const next = structuredClone(f.database.state); + next.documents.a.base = { + ...head("a", 2, "base REMOTE"), + hash: (await toStoredSnapshot({ content: bytes("base REMOTE") })).hash + }; + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + "durable:rename:a.md->.vault-link-sync/transactions/" + ) + ) + return; + injected = true; + await f.disk.userWrite("a.md", bytes("base LOCAL")); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); + }; + await f.files.apply(next, { + a: { + expected: await toStoredSnapshot({ content: bytes("base") }), + replacement: await toStoredSnapshot({ + content: bytes("base REMOTE") + }) + } + }); + + assert.equal(f.disk.userFiles().size, 1); + const result = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert(result.includes("LOCAL") && result.includes("REMOTE")); +}); + +async function setup() { + const f = await fixture({ "a.md": "ORIGINAL", "b.md": "OTHER" }); + f.database.state.documents.a.base = { + ...head("a", 1, "ORIGINAL"), + hash: f.database.state.documents.a.observedHash! + }; + const next = structuredClone(f.database.state); + next.local = { a: "remote.md", b: "remote-b.md" }; + next.fileManifest = { fileManifestId: 3, entries: { ...next.local } }; + const expected = await toStoredSnapshot({ content: bytes("ORIGINAL") }); + const replacement = await toStoredSnapshot({ + content: bytes("ORIGINAL REMOTE") + }); + next.documents.a.base = { + ...head("a", 2, "ORIGINAL REMOTE"), + hash: replacement.hash + }; + return { f, next, writes: { a: { expected, replacement } } }; +} +test("editor changes preserve exact bytes and identities at every filesystem read, mutation and persistence boundary", async (t) => { + const { f, next, writes } = await setup(); + const labels: string[] = []; + const trace = (l: string) => { + labels.push(l); + }; + f.disk.boundary = trace; + f.persistence.boundary = trace; + await f.files.apply(next, writes); + const failures = []; + let exercised = 0; + for (const kind of ["move", "move-recreate", "delete-recreate", "save"]) + for (let target = 0; target < labels.length; target++) { + const { f, next, writes } = await setup(); + let count = 0, + injected = false, + from: string | undefined; + const boundary = async (l: string) => { + if (injected) return; + if (count++ !== target) return; + const files = f.disk.userFiles(); + from = files.has("a.md") + ? "a.md" + : files.has("remote.md") + ? "remote.md" + : undefined; + if (kind === "save" && !from) from = "a.md"; + if (!from) return; + injected = true; + exercised++; + if (kind === "save") { + await f.disk.userWrite( + from, + bytes( + Buffer.from( + files.get(from) ?? bytes("ORIGINAL") + ).toString() + " LOCAL" + ) + ); + await f.syncer.syncLocallyUpdatedFile({ + relativePath: from + }); + } else if (kind === "delete-recreate") { + await f.disk.userDelete(from); + await f.syncer.syncLocallyDeletedFile(from); + } else { + await f.disk.userRename(from, "mine.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: from, + relativePath: "mine.md" + }); + } + if (kind !== "move" && kind !== "save") { + await f.disk.userWrite(from, bytes("REPLACEMENT")); + await f.syncer.syncLocallyCreatedFile(from); + } + }; + f.disk.boundary = boundary; + f.persistence.boundary = boundary; + let error; + try { + await f.files.apply(next, writes); + } catch (e) { + error = String(e); + } + f.disk.boundary = () => {}; + f.persistence.boundary = () => {}; + try { + await f.database.recoverPersistence(); + await f.files.recover(); + await f.internals.scan(); + } catch (e) { + error += ";retry=" + String(e); + } + if (!injected) continue; + const fs = Object.fromEntries( + [...f.disk.userFiles()].map(([p, b]) => [ + p, + Buffer.from(b).toString() + ]) + ); + const bad = + kind === "save" + ? ![ + "ORIGINAL LOCAL REMOTE", + "ORIGINAL REMOTE LOCAL" + ].includes(fs[f.database.state.local.a]) || + Object.keys(f.database.state.local).length !== 2 + : kind === "delete-recreate" + ? f.database.state.local.a !== undefined + : f.database.state.local.a !== "mine.md" || + fs["mine.md"] !== "ORIGINAL REMOTE"; + if ( + bad || + (kind !== "move" && + kind !== "save" && + fs[from!] !== "REPLACEMENT") + ) + failures.push({ + kind, + target, + label: labels[target], + error, + ids: f.database.state.local, + files: fs + }); + } + assert( + exercised > 50, + "The sweep must actually exercise the injected actions" + ); + t.diagnostic( + `${labels.length} boundaries; ${exercised} editor interleavings` + ); + assert.deepEqual(failures, []); +}); diff --git a/frontend/test-support/journal-protocol.test.ts b/frontend/test-support/journal-protocol.test.ts new file mode 100644 index 00000000..568b9688 --- /dev/null +++ b/frontend/test-support/journal-protocol.test.ts @@ -0,0 +1,612 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; + +for (const exclusion of ["size", "ignore"] as const) + for (const phase of [ + "planned", + ...(exclusion === "size" ? ["staged", "prepared"] : []), + "installed" + ]) + for (const recovery of [ + "none", + "restart", + "before:save", + "durable:save" + ] as const) + test(`reenabling ${exclusion} exclusion at ${phase} preserves the content base through ${recovery}`, async () => { + const f = await fixture(); + try { + const installed = phase === "installed"; + const id = randomUUID(); + const base = await f.put( + "/documents/" + id, + f.content("BASE\n") + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.client.setSettings({ + maxFileSizeMB: + exclusion === "size" ? 100 / (1024 * 1024) : 1, + ignorePatterns: ["private.md"] + }); + const remote = await f.put( + "/documents/" + id, + f.content("REMOTE\nBASE\n", base.vaultUpdateId) + ); + const local = "BASE\n" + "LOCAL\n".repeat(30); + const path = exclusion === "size" ? "a.md" : "private.md"; + let injected = false, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + const atBoundary = + phase === "installed" + ? label.startsWith("durable:rename:") && + label.endsWith(".output->a.md") + : phase === "staged" + ? label.startsWith("durable:rename:a.md->") + : phase === "prepared" + ? label.startsWith("before:write:") && + label.endsWith(".output") + : label.startsWith( + "before:mkdir:.vault-link-sync/transactions/" + ); + if (injected || !atBoundary) return; + injected = true; + if (exclusion === "ignore") + await f.disk.userRename("a.md", path); + // When installation already completed, deleting REMOTE + // is a subsequent local edit, which must not be undone. + await f.disk.userWrite(path, bytes(local)); + await f.client.syncLocallyUpdatedFile({ + relativePath: path, + ...(exclusion === "ignore" + ? { oldPath: "a.md" } + : {}) + }); + armed = true; + }; + f.store.boundary = (label) => { + if (!armed || crashed || label !== recovery) return; + crashed = true; + f.disk.crash(true); + throw new Error("interrupted exclusion save"); + }; + if (recovery.endsWith(":save")) { + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /interrupted exclusion save/ + ); + assert(crashed); + } else await f.client.setSetting("isSyncEnabled", true); + assert(injected); + f.disk.boundary = f.store.boundary = () => {}; + if (recovery !== "none") { + await f.reopenClient(); + await f.client.start(); + } + + assert.deepEqual( + (await f.disk.readSnapshot(path))?.content, + bytes(local) + ); + const stored = f.store.snapshot().database!; + assert.equal( + stored.documents![id].base?.vaultUpdateId, + installed ? remote.vaultUpdateId : base.vaultUpdateId + ); + assert.equal( + stored.remoteHeads![id].vaultUpdateId, + remote.vaultUpdateId + ); + assert.equal( + (await f.get("/documents/" + id)).contentBase64, + Buffer.from("REMOTE\nBASE\n").toString("base64") + ); + await f.client.setSettings({ + maxFileSizeMB: 1, + ignorePatterns: [] + }); + const expected = installed ? local : "REMOTE\n" + local; + const verify = async () => { + assert.deepEqual( + (await f.get("/file-manifest")).entries, + { [id]: path } + ); + assert.deepEqual( + (await f.disk.readSnapshot(path))?.content, + bytes(expected) + ); + assert.equal( + (await f.get("/documents/" + id)).contentBase64, + Buffer.from(expected).toString("base64") + ); + }; + await verify(); + await f.client.reset(); + await verify(); + } finally { + await f.dispose(); + } + }); + +for (const at of [undefined, "before:save", "durable:save"]) + for (const powerLoss of at ? [false, true] : [false]) + test(`repeated saves retain only the latest local edit through ${at ?? "completion"} and powerLoss=${powerLoss}`, async () => { + const f = await fixture(); + try { + const id = randomUUID(); + const base = await f.put("/documents/" + id, f.content("base")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put( + "/documents/" + id, + f.content("base REMOTE", base.vaultUpdateId) + ); + let saves = 0, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + if ( + saves === 0 && + label.startsWith("before:rename:a.md->") + ) { + saves++; + await f.disk.userWrite("a.md", bytes("base ONE")); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.md" + }); + } else if ( + saves === 1 && + label.startsWith("before:write:") && + label.endsWith(".output") + ) { + saves++; + await f.disk.userWrite("a.md", bytes("base TWO")); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.md" + }); + armed = true; + } + }; + f.store.boundary = (label) => { + if (!armed || crashed || label !== at) return; + crashed = true; + f.disk.crash(powerLoss); + throw new Error("injected repeated-save crash"); + }; + if (at) { + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /injected repeated-save crash/ + ); + assert(crashed); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } else await f.client.setSetting("isSyncEnabled", true); + assert.equal(saves, 2); + const verify = async () => { + assert.deepEqual((await f.get("/file-manifest")).entries, { + [id]: "a.md" + }); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base TWO REMOTE") + ); + assert.equal( + Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(), + "base TWO REMOTE" + ); + }; + await verify(); + await f.client.reset(); + await verify(); + } finally { + await f.dispose(); + } + }); + +for (const timing of ["before", "durable"]) + for (const crash of [false, true]) + test(`oversized save ${timing} staging stays local across crash=${crash}`, async () => { + const f = await fixture(); + try { + const id = randomUUID(), + large = "PRIVATE".repeat(100); + const base = await f.put( + "/documents/" + id, + f.content("PUBLIC") + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.bin" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.client.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + await f.put( + "/documents/" + id, + f.content("REMOTE", base.vaultUpdateId) + ); + let injected = false, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + `${timing}:rename:a.bin->.vault-link-sync/transactions/` + ) + ) + return; + injected = true; + await f.disk.userWrite("a.bin", bytes(large)); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.bin" + }); + armed = true; + }; + f.store.boundary = (label) => { + if (!crash || !armed || crashed || label !== "durable:save") + return; + crashed = true; + f.disk.crash(true); + throw new Error("injected oversized-save crash"); + }; + if (crash) { + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /injected oversized-save crash/ + ); + assert(crashed); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } else await f.client.setSetting("isSyncEnabled", true); + assert(injected); + const verify = async () => { + assert.deepEqual( + (await f.disk.readSnapshot("a.bin"))?.content, + bytes(large) + ); + assert.deepEqual((await f.get("/file-manifest")).entries, { + [id]: "a.bin" + }); + assert.equal( + Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(), + "REMOTE" + ); + }; + await verify(); + await f.client.reset(); + await verify(); + } finally { + await f.dispose(); + } + }); + +test("a tracked file moved into an ignored path during remote application never uploads its new bytes", async () => { + const f = await fixture(); + try { + const a = randomUUID(), + b = randomUUID(); + await f.put("/documents/" + a, f.content("PUBLIC")); + await f.put("/documents/" + b, f.content("OTHER")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [a]: "a.md", [b]: "b.md" } + }); + await f.client.setSetting("ignorePatterns", ["private.md"]); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [a]: "a.md", [b]: "remote-b.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("SECRET")); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + await f.client.setSetting("isSyncEnabled", true); + assert(injected); + const manifest = await f.get("/file-manifest"); + const contents = await Promise.all( + Object.keys(manifest.entries).map((id) => f.get("/documents/" + id)) + ); + assert( + !contents.some( + (c) => + Buffer.from(c.contentBase64, "base64").toString() === + "SECRET" + ) + ); + assert.deepEqual( + (await f.disk.readSnapshot("private.md"))?.content, + bytes("SECRET") + ); + assert.equal(manifest.entries[a], "a.md"); + } finally { + await f.dispose(); + } +}); +test("remote creation plus unrelated notification must not delete remote membership on retry", async () => { + const f = await fixture(); + try { + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + const id = randomUUID(); + await f.put("/documents/" + id, f.content("REMOTE")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "remote.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userWrite("local.md", bytes("LOCAL")); + await f.client.syncLocallyCreatedFile("local.md"); + }; + await f.client.setSetting("isSyncEnabled", true); + assert(injected); + f.disk.boundary = () => {}; + await f.client.reset(); + const manifest = await f.get("/file-manifest"); + + assert.equal(manifest.entries[id], "remote.md"); + } finally { + await f.dispose(); + } +}); +test("notified ignored occupant during remote rename must never upload its bytes", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + await f.put("/documents/" + id, f.content("REMOTE")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.setSetting("ignorePatterns", ["private"]); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "private/note.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userWrite("private", bytes("SECRET")); + await f.client.syncLocallyCreatedFile("private"); + }; + await f.client.setSetting("isSyncEnabled", true); + f.disk.boundary = () => {}; + const manifest = await f.get("/file-manifest"); + const contents = await Promise.all( + Object.keys(manifest.entries).map((id) => f.get("/documents/" + id)) + ); + + assert(injected); + assert.deepEqual( + (await f.disk.readSnapshot("private"))?.content, + bytes("SECRET") + ); + assert( + !contents.some( + (c) => + Buffer.from(c.contentBase64, "base64").toString() === + "SECRET" + ) + ); + } finally { + await f.dispose(); + } +}); +test("notified delete/recreate before staging retains replacement file", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + const h = await f.put("/documents/" + id, f.content("ORIGINAL")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put( + "/documents/" + id, + f.content("ORIGINAL REMOTE", h.vaultUpdateId) + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "remote.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + "before:rename:a.md->.vault-link-sync/transactions/" + ) + ) + return; + injected = true; + await f.disk.userDelete("a.md"); + await f.client.syncLocallyDeletedFile("a.md"); + await f.disk.userWrite("a.md", bytes("REPLACEMENT")); + await f.client.syncLocallyCreatedFile("a.md"); + }; + await f.client.setSetting("isSyncEnabled", true); + f.disk.boundary = () => {}; + const manifest = await f.get("/file-manifest"); + + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("REPLACEMENT") + ); + } finally { + await f.dispose(); + } +}); + +for (const mode of ["delete", "move", "save"] as const) + for (const powerLoss of [false, true]) + for (const at of ["before:save", "durable:save"]) + test(`${mode} during staging survives ${at} and ${powerLoss ? "power loss" : "process replacement"}`, async () => { + const f = await fixture(); + try { + const id = randomUUID(); + const h = await f.put( + "/documents/" + id, + f.content("ORIGINAL") + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put( + "/documents/" + id, + f.content("ORIGINAL REMOTE", h.vaultUpdateId) + ); + let injected = false, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + const point = mode === "save" ? "durable" : "before"; + if ( + injected || + !label.startsWith( + `${point}:rename:a.md->.vault-link-sync/transactions/` + ) + ) + return; + injected = true; + if (mode === "save") { + await f.disk.userWrite( + "a.md", + bytes("ORIGINAL LOCAL") + ); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.md" + }); + } else { + if (mode === "delete") { + await f.disk.userDelete("a.md"); + await f.client.syncLocallyDeletedFile("a.md"); + } else { + await f.disk.userRename("a.md", "mine.md"); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "mine.md" + }); + } + await f.disk.userWrite( + "a.md", + bytes("REPLACEMENT") + ); + await f.client.syncLocallyCreatedFile("a.md"); + } + armed = true; + }; + f.store.boundary = (label) => { + if (!armed || crashed || label !== at) return; + crashed = true; + f.disk.crash(powerLoss); + throw new Error("injected journal crash"); + }; + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /injected journal crash/ + ); + assert(injected && crashed); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + const manifest = await f.get("/file-manifest"); + if (mode === "save") { + assert.deepEqual(manifest.entries, { [id]: "a.md" }); + const actual = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(actual, "ORIGINAL LOCAL REMOTE"); + } else { + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("REPLACEMENT") + ); + assert.notEqual( + Object.keys(manifest.entries).find( + (key) => manifest.entries[key] === "a.md" + ), + id + ); + if (mode === "move") { + assert.equal(manifest.entries[id], "mine.md"); + assert.deepEqual( + (await f.disk.readSnapshot("mine.md"))?.content, + bytes("ORIGINAL REMOTE") + ); + } else assert.equal(manifest.entries[id], undefined); + } + await f.client.reset(); + assert.deepEqual( + (await f.get("/file-manifest")).entries, + manifest.entries + ); + } finally { + await f.dispose(); + } + }); diff --git a/frontend/test-support/listener-crash-child.ts b/frontend/test-support/listener-crash-child.ts new file mode 100644 index 00000000..bd703090 --- /dev/null +++ b/frontend/test-support/listener-crash-child.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { fixture } from "./sync-fixture"; +(async () => { + const f = await fixture( + {}, + { + events: async () => ({ headEventId: 0, events: [] }) + } + ); + let delivered = 0; + f.syncer.onRemainingOperationsCountChanged.add(() => { + throw new Error("observer failure"); + }); + f.syncer.onRemainingOperationsCountChanged.add(async () => { + throw new Error("async observer failure"); + }); + f.syncer.onRemainingOperationsCountChanged.add(() => { + delivered++; + }); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + const before = delivered; + f.syncer.wake(); + await f.syncer.waitUntilFinished(); + assert(delivered > before, "Background sync must remain usable"); + } finally { + await f.syncer.stop(); + } +})(); diff --git a/frontend/test-support/native-fs.c b/frontend/test-support/native-fs.c new file mode 100644 index 00000000..560c79c6 --- /dev/null +++ b/frontend/test-support/native-fs.c @@ -0,0 +1,27 @@ +/* Test-only POSIX primitives missing from Node's filesystem API. A preflight + * exists() followed by fs.rename() is not an exclusive rename. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#if defined(__linux__) +#include +#endif + +int main(int argc, char **argv) { + if (argc == 4 && strcmp(argv[1], "rename") == 0) { + int result; +#if defined(__APPLE__) + result = renamex_np(argv[2], argv[3], RENAME_EXCL); +#elif defined(__linux__) + result = syscall(SYS_renameat2, AT_FDCWD, argv[2], AT_FDCWD, argv[3], 1 /* RENAME_NOREPLACE */); +#else +#error "The real filesystem tests require macOS or Linux" +#endif + if (result != 0) { perror("exclusive rename"); return 1; } + return 0; + } + fprintf(stderr, "usage: native-fs rename FROM TO\n"); + return 2; +} diff --git a/frontend/test-support/network.ts b/frontend/test-support/network.ts new file mode 100644 index 00000000..7df61083 --- /dev/null +++ b/frontend/test-support/network.ts @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; + +export type RequestKind = "create" | "content" | "manifest"; +export interface RequestRecord { + kind: RequestKind; + method: string; + url: string; + body: Record; + status?: number; +} + +interface InjectedFault { + kind: RequestKind; + point: "before" | "after"; + fired: boolean; + interrupted?: RequestRecord; + retried: boolean; + reached: Promise; + resolve: () => void; +} + +export function classifyRequest( + input: RequestInfo | URL, + init?: RequestInit +): RequestRecord | undefined { + const url = new URL(input instanceof Request ? input.url : String(input)); + const method = ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(); + if (method !== "PUT") return undefined; + // The engine passes JSON strings. Fail loudly if that contract changes. + if (!/\/(documents\/[^/]+|file-manifest)\/?$/.test(url.pathname)) + return undefined; + assert.equal( + typeof init?.body, + "string", + "CAS request body must be observable JSON" + ); + const body = JSON.parse(init!.body as string) as Record; + const kind = /\/file-manifest\/?$/.test(url.pathname) + ? "manifest" + : body.parentVersionId === null + ? "create" + : "content"; + assert.equal( + typeof body.requestId, + "string", + "CAS request missing requestId" + ); + return { kind, method, url: url.href, body }; +} + +export class NetworkFaults { + public readonly requests: RequestRecord[] = []; + private observation?: { + released: boolean; + fired: boolean; + held: Promise; + release: () => void; + reached: Promise; + reach: () => void; + }; + public pauseObservation(): void { + assert( + !this.observation || this.observation.released, + "Observation already paused" + ); + let release!: () => void, reach!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const reached = new Promise((resolve) => { + reach = resolve; + }); + this.observation = { + released: false, + fired: false, + held, + release, + reached, + reach + }; + } + public async waitForObservation(): Promise { + assert(this.observation, "No observation checkpoint armed"); + await this.observation.reached; + } + public resumeObservation(): void { + if (this.observation) { + this.observation.released = true; + this.observation.release(); + } + } + private async observationCheckpoint( + signal?: AbortSignal | null + ): Promise { + const gate = this.observation; + if (!gate || gate.released) return; + gate.fired = true; + gate.reach(); + signal?.throwIfAborted(); + let abort = () => {}; + try { + await Promise.race([ + gate.held, + new Promise((_, reject) => { + abort = () => + reject( + signal?.reason ?? new Error("Observation aborted") + ); + signal?.addEventListener("abort", abort, { once: true }); + }) + ]); + } finally { + signal?.removeEventListener("abort", abort); + } + } + private armed?: InjectedFault; + private readonly faults: InjectedFault[] = []; + public isExpectedFailure(message: string): boolean { + return this.faults.some( + (fault) => + fault.fired && + message.includes( + `Injected ${fault.point}-commit ${fault.kind} network failure` + ) + ); + } + public arm(kind: RequestKind, point: "before" | "after" = "after"): void { + assert( + !this.armed || (this.armed.fired && this.armed.retried), + "Previous network fault was not exercised and retried" + ); + let resolve!: () => void; + const reached = new Promise((r) => { + resolve = r; + }); + this.armed = { + kind, + point, + fired: false, + retried: false, + reached, + resolve + }; + this.faults.push(this.armed); + } + public async wait(): Promise { + assert(this.armed, "No network fault armed"); + await this.armed.reached; + } + public assertConsumed(): void { + assert( + !this.observation || + (this.observation.fired && this.observation.released), + "Observation checkpoint was not exercised and released" + ); + for (const fault of this.faults) { + assert(fault.fired, "Armed network fault never fired"); + assert(fault.retried, "Interrupted CAS request was never retried"); + } + } + public wrap(fetch: typeof globalThis.fetch): typeof globalThis.fetch { + return async (input, init) => { + const url = new URL( + input instanceof Request ? input.url : String(input) + ); + const observing = + ( + init?.method ?? + (input instanceof Request ? input.method : "GET") + ).toUpperCase() === "GET" && !url.pathname.endsWith("/ping"); + const signal = + init?.signal ?? + (input instanceof Request ? input.signal : undefined); + if (observing) await this.observationCheckpoint(signal); + const record = classifyRequest(input, init); + if (record) { + // A client serializes its persisted CAS requests. Its next CAS + // after a lost response must replay the interrupted operation, + // even if it has since observed newer remote state. + const interrupted = this.armed?.interrupted; + if (interrupted && !this.armed!.retried) { + assert.equal( + record.url, + interrupted.url, + "Retry changed target" + ); + assert.deepEqual( + record.body, + interrupted.body, + "Retry changed payload/request identity" + ); + this.armed!.retried = true; + } + const previous = this.requests.find( + (r) => r.body.requestId === record.body.requestId + ); + if (previous) { + assert.equal( + record.url, + previous.url, + "Retry changed target" + ); + assert.deepEqual( + record.body, + previous.body, + "Retry changed payload/request identity" + ); + } + this.requests.push(record); + } + const fault = this.armed; + const matches = + record && fault && !fault.fired && fault.kind === record.kind; + const fail = () => { + fault!.interrupted = structuredClone(record!); + fault!.fired = true; + fault!.resolve(); + throw new TypeError( + `Injected ${fault!.point}-commit ${fault!.kind} network failure` + ); + }; + if (matches && fault.point === "before") fail(); + const response = await fetch(input, init); + if (observing) await this.observationCheckpoint(signal); + if (record) record.status = response.status; + // Only drop accepted replies, never confuse a rejected CAS with commit. + if ( + matches && + !fault.fired && + fault.point === "after" && + response.ok && + ((await response.clone().json()) as { type?: string }).type === + "Accepted" + ) { + await response.body?.cancel(); + fail(); + } + return response; + }; + } +} diff --git a/frontend/test-support/observer.test.ts b/frontend/test-support/observer.test.ts new file mode 100644 index 00000000..8fb2d5e6 --- /dev/null +++ b/frontend/test-support/observer.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +test("a status observer exception must not terminate a background sync process", () => { + const result = spawnSync( + process.execPath, + [ + resolve(__dirname, "../node_modules/tsx/dist/cli.mjs"), + resolve(__dirname, "listener-crash-child.ts") + ], + { encoding: "utf8", timeout: 10000 } + ); + assert.equal(result.status, 0, result.stderr); +}); diff --git a/frontend/test-support/oracle-regressions.test.ts b/frontend/test-support/oracle-regressions.test.ts new file mode 100644 index 00000000..fc2916a4 --- /dev/null +++ b/frontend/test-support/oracle-regressions.test.ts @@ -0,0 +1,351 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { test } from "node:test"; +import type { StoredDatabase } from "sync-client"; +import { assertCanonical } from "./canonical"; +import { assertManifest } from "./oracles"; +import { NetworkFaults } from "./network"; +import { MemoryDisk } from "./storage"; +import { DeterministicAgent } from "../deterministic-tests/src/deterministic-agent"; +import { AssertableState } from "../deterministic-tests/src/utils/assertable-state"; +import type { ExpectedDocument } from "../deterministic-tests/src/test-definition"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const id = "00000000-0000-4000-8000-000000000001"; + +test("canonical checks reject repeated clients before reading snapshots", async () => { + const client = { files: () => new Map(), database: () => ({}) }; + await assert.rejects( + assertCanonical([client, client], "http://unused", "unused"), + /itself/ + ); +}); + +test("canonical checks independently verify both persisted content hashes", async (context) => { + const content = bytes("\ufeffexact bytes\r\n"); + const hash = createHash("sha256").update(content).digest("hex"); + const document = { + documentId: id, + vaultUpdateId: 1, + contentSize: content.length, + userId: "user", + deviceId: "device", + updatedDate: "2026-09-20T00:00:00Z" + }; + const manifest = { fileManifestId: 2, entries: { [id]: "a.md" } }; + const state: Partial = { + initialized: true, + local: manifest.entries, + fileManifest: manifest, + lastSeenUpdateId: 2, + documents: { + [id]: { + materialized: true, + observedHash: hash, + base: { ...document, hash } + } + } + }; + context.mock.method( + globalThis, + "fetch", + async (url: string) => + new Response( + JSON.stringify( + url.endsWith("/vault-snapshot") + ? { + headEventId: 2, + fileManifest: manifest, + documents: [document] + } + : url.includes("/events-since") + ? { + headEventId: 2, + events: [ + { eventId: 1, requestId: "create" }, + { eventId: 2, requestId: "manifest" } + ] + } + : { + contentBase64: + Buffer.from(content).toString("base64") + } + ) + ) + ); + const client = (database = state) => ({ + files: () => new Map([["a.md", content.slice()]]), + database: () => structuredClone(database) + }); + await assertCanonical([client(), client()], "http://test", "token"); + for (const field of ["base", "observed"] as const) { + const corrupt = structuredClone(state); + if (field === "base") corrupt.documents![id].base!.hash = "wrong"; + else corrupt.documents![id].observedHash = "wrong"; + await assert.rejects( + assertCanonical( + [client(), client(corrupt)], + "http://test", + "token" + ), + new RegExp(`Wrong ${field} hash`) + ); + } +}); + +test("lost responses require an identical retry, including request identity and target", async () => { + for (const point of ["before", "after"] as const) { + const faults = new NetworkFaults(); + let sent = 0; + const send = faults.wrap(async () => { + sent++; + return new Response('{"type":"Accepted"}'); + }); + const url = "http://test/documents/a"; + const body = { + requestId: "original", + parentVersionId: 3, + content: { type: "Snapshot", value: "YQ==" } + }; + const init = { method: "PUT", body: JSON.stringify(body) }; + faults.arm("content", point); + await assert.rejects(send(url, init), /Injected/); + assert.throws(() => faults.assertConsumed(), /never retried/); + assert.throws( + () => faults.arm("manifest"), + /not exercised and retried/ + ); + const before = sent; + await assert.rejects( + send(url, { + ...init, + body: JSON.stringify({ ...body, requestId: "regenerated" }) + }), + /Retry changed/ + ); + await assert.rejects( + send(`${url}-different`, init), + /Retry changed target/ + ); + await assert.rejects( + send(url, { + ...init, + body: JSON.stringify({ ...body, parentVersionId: 4 }) + }), + /Retry changed/ + ); + assert.equal(sent, before, "Invalid retries must not reach the server"); + assert.throws(() => faults.assertConsumed(), /never retried/); + await send(url, init); + faults.assertConsumed(); + } +}); + +test("an update fault cannot be consumed by creating a new UUID", async () => { + const faults = new NetworkFaults(); + const send = faults.wrap(async () => new Response('{"type":"Accepted"}')); + faults.arm("content"); + await send("http://test/documents/new", { + method: "PUT", + body: '{"requestId":"new","parentVersionId":null}' + }); + assert.throws(() => faults.assertConsumed(), /never fired/); + const update = { + method: "PUT", + body: '{"requestId":"update","parentVersionId":1}' + }; + await assert.rejects( + send("http://test/documents/existing", update), + /Injected/ + ); + await send("http://test/documents/existing", update); + faults.assertConsumed(); +}); + +test("unlink is idempotent even after an uncertain visible deletion", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a", bytes("A")); + disk.boundary = (label) => { + if (label === "visible:unlink:a") throw new Error("uncertain unlink"); + }; + await assert.rejects(disk.deleteFile("a"), /uncertain/); + disk.boundary = () => {}; + await disk.deleteFile("a"); + disk.crash(true); + assert.equal(await disk.exists("a"), false); + await disk.deleteFile("a"); + await disk.createDirectory("directory"); + await assert.rejects(disk.deleteFile("directory"), /directory/); +}); + +test("text assertions preserve BOM, CRLF, empty content and missing final newline", async () => { + const agent = new DeterministicAgent(0, {}, () => {}); + for (const content of [ + "", + "\ufeffhello", + "\ufeffa\r\nb\r\n", + "last line" + ]) { + await agent.disk.userWrite("a.md", bytes(content)); + assert.equal(await agent.getFileContent("a.md"), content); + } +}); + +test("durably pruning a directory also removes its already-unlinked durable children", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("transaction/source", bytes("staged bytes")); + disk.boundary = (label) => { + if (label === "visible:unlink:transaction/source") { + disk.crash(false); + throw new Error("process died after unlink"); + } + }; + await assert.rejects(disk.deleteFile("transaction/source"), /process died/); + disk.boundary = () => {}; + await disk.delete("transaction"); + disk.crash(true); + assert.deepEqual(disk.userFiles(), new Map()); + assert.equal(await disk.exists("transaction"), false); +}); + +test("portable components are limited by UTF-8 bytes, not characters or total path length", () => { + for (const component of ["a".repeat(255), "é".repeat(127) + "a"]) + assertManifest({ [id]: `${component}/${component}/${component}` }); + for (const component of ["a".repeat(256), "é".repeat(128)]) + assert.throws( + () => assertManifest({ [id]: `nested/${component}` }), + /Oversized component/ + ); +}); + +function state(documents: { path: string; content: string; id: string }[]) { + return new AssertableState({ + files: new Map(documents.map((doc) => [doc.path, doc.content])), + clientFiles: [], + manifests: [], + canonical: Object.fromEntries( + documents.map((doc) => [doc.id, doc.path]) + ), + bytes: new Map(documents.map((doc) => [doc.path, bytes(doc.content)])) + }); +} + +test("complete scenario assertions reject collapsed creates, forgotten edits, reverted moves and recycled UUIDs", () => { + const identities = new Map(); + const expected: ExpectedDocument[] = [ + { key: "a", path: "a.md", content: "A\nimportant edit" }, + { key: "b", path: "b.md", content: "B" } + ]; + const original = [ + { id: "id-a", path: "a.md", content: "A\nimportant edit" }, + { id: "id-b", path: "b.md", content: "B" } + ]; + state(original).assertDocuments(expected, identities); + assert.throws( + () => + state([ + { ...original[0], content: "A\nimportant edit\nB" } + ]).assertDocuments(expected, identities), + /Expected 2 file/ + ); + assert.throws( + () => + state([ + { ...original[0], content: "A" }, + original[1] + ]).assertDocuments(expected, identities), + /Byte mismatch/ + ); + const moved = [ + { ...expected[0], path: "b.md" }, + { ...expected[1], path: "a.md" } + ]; + assert.throws( + () => state(original).assertDocuments(moved, identities), + /Byte mismatch/ + ); + assert.throws( + () => + state([ + { ...original[0], id: "id-b" }, + { ...original[1], id: "id-a" } + ]).assertDocuments(expected, identities), + /identity changed/ + ); + state([original[1]]).assertDocuments([expected[1]], identities); + assert.throws( + () => + state(original).assertDocuments( + [{ ...expected[0], key: "recreated" }, expected[1]], + identities + ), + /reused.*deleted UUID/ + ); + assert.equal(identities.has("recreated"), false); + const conflict = [ + { key: "one", path: "same.md", content: "one" }, + { key: "two", path: "same.md", conflict: true, content: "two" } + ]; + state([ + { id: "one", path: "same.md", content: "one" }, + { id: "two", path: "same (conflict two).md", content: "two" } + ]).assertDocuments(conflict, new Map()); +}); + +test("canonical checks follow pages without weakening gap or duplicate-request detection", async (context) => { + const manifest = { fileManifestId: 3, entries: {} }; + const client = () => ({ + files: () => new Map(), + database: () => ({ + initialized: true, + local: {}, + documents: {}, + fileManifest: manifest, + lastSeenUpdateId: 3 + }) + }); + let fault = "none"; + context.mock.method(globalThis, "fetch", async (input: string) => { + if (input.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 3, + fileManifest: manifest, + documents: [] + }); + const after = Number(new URL(input).searchParams.get("after")); + if (after === 0) + return Response.json({ + headEventId: 3, + endEventId: 1, + events: [{ eventId: 1, requestId: "first" }] + }); + assert.equal(after, 1); + return Response.json({ + headEventId: 3, + endEventId: fault === "empty" ? 1 : 3, + events: + fault === "empty" + ? [] + : [ + ...(fault === "gap" + ? [] + : [ + { + eventId: 2, + requestId: + fault === "duplicate" + ? "first" + : "second" + } + ]), + { eventId: 3, requestId: "third" } + ] + }); + }); + await assertCanonical([client(), client()], "http://test", "token"); + for (fault of ["gap", "duplicate", "empty"]) + await assert.rejects( + assertCanonical([client(), client()], "http://test", "token"), + /contiguous|multiple events|progress/ + ); +}); diff --git a/frontend/test-support/oracles.ts b/frontend/test-support/oracles.ts new file mode 100644 index 00000000..f50141ef --- /dev/null +++ b/frontend/test-support/oracles.ts @@ -0,0 +1,173 @@ +import assert from "node:assert/strict"; +import type { StoredDatabase } from "sync-client"; + +export function assertSameFiles( + clients: readonly Map[] +): void { + assert( + clients.length >= 2, + "Consistency requires at least two distinct clients" + ); + assert.equal( + new Set(clients).size, + clients.length, + "Cannot compare a client to itself" + ); + const sorted = (files: Map) => + [...files].sort(([a], [b]) => a.localeCompare(b)); + for (let i = 1; i < clients.length; i++) + assert.deepEqual( + sorted(clients[i]), + sorted(clients[0]), + `Client ${i} differs from client 0 (paths or bytes)` + ); +} + +export function assertManifest(entries: Record): void { + const paths = Object.values(entries); + const folded = paths.map((p) => + p.normalize("NFC").toUpperCase().normalize("NFC") + ); + const directorySpellings = new Map(); + for (const path of paths) { + assert.equal( + path, + path.normalize("NFC"), + `Noncanonical Unicode path: ${path}` + ); + const parts = path.split("/"); + parts.forEach((part, i) => { + assert( + Buffer.byteLength(part, "utf8") <= 255, + `Oversized component: ${part}` + ); + assert( + !/[<>:"\\|?*\p{Cc}]|[. ]$/u.test(part), + `Nonportable component: ${part}` + ); + assert( + !/^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$/u.test( + part.split(".")[0].trimEnd().toUpperCase() + ), + `Reserved component: ${part}` + ); + if (i < parts.length - 1) { + const prefix = parts.slice(0, i + 1).join("/"); + const key = prefix.toUpperCase().normalize("NFC"); + const previous = directorySpellings.get(key); + assert( + previous === undefined || previous === prefix, + `Aliased directory spellings: ${prefix} and ${previous}` + ); + directorySpellings.set(key, prefix); + } + }); + } + assert.equal( + new Set(folded).size, + paths.length, + "Manifest contains aliased/duplicate paths" + ); + for (const path of folded) { + assert( + path && !path.startsWith("/") && !path.includes("\\"), + `Invalid path ${path}` + ); + assert( + path.split("/").every((p) => p && p !== "." && p !== ".."), + `Invalid component ${path}` + ); + assert( + path.split("/")[0] !== ".VAULT-LINK-SYNC", + "Internal path in manifest" + ); + assert( + !folded.some( + (other) => other !== path && other.startsWith(`${path}/`) + ), + `File/ancestor conflict: ${path}` + ); + } +} + +/** Independent per-ID policy oracle, intentionally not importing merge code. */ +export function pathDecision( + base: string | undefined, + local: string | undefined, + remote: string | undefined +): string | undefined { + if (local === base) return remote; + if (remote === base) return local; + return remote; +} + +export function assertQuiescent( + state: Partial, + entries: Record, + head: number +): void { + assert.equal( + state.pending, + undefined, + "Pending CAS request at convergence" + ); + assert.equal( + state.eventReplay, + undefined, + "Unfinished event replay at convergence" + ); + assert.equal( + state.application, + undefined, + "Unfinished filesystem journal at convergence" + ); + assert.deepEqual( + state.local, + entries, + "Local UUID/path map differs from server" + ); + assert.deepEqual( + state.fileManifest?.entries, + entries, + "Client manifest differs from server" + ); + assert.equal( + state.lastSeenUpdateId, + head, + "Client event watermark differs from server" + ); + assertManifest(entries); +} + +/** Removal exemptions name exact markers and the explicit destructive action. + * Deletion mode never disables preservation for unrelated documents. */ +export class ContentLedger { + private readonly markers = new Map(); + public add(marker: string): void { + assert(!this.markers.has(marker), "Duplicate marker"); + this.markers.set(marker, undefined); + } + public removedBy(content: string, action: string): void { + assert(action.length > 0); + for (const marker of this.markers.keys()) + if (content.includes(marker)) this.markers.set(marker, action); + } + public assertPreserved(files: Map): void { + const texts = [...files.values()].map((bytes) => + Buffer.from(bytes).toString("utf8") + ); + for (const [marker, deletion] of this.markers) { + const count = texts.reduce( + (n, text) => n + text.split(marker).length - 1, + 0 + ); + assert(count <= 1, `Marker ${marker} duplicated ${count} times`); + if (!deletion) + assert.equal( + count, + 1, + `Marker ${marker} lost without an explicit delete/overwrite` + ); + } + } +} diff --git a/frontend/test-support/polling.test.ts b/frontend/test-support/polling.test.ts new file mode 100644 index 00000000..37627b5e --- /dev/null +++ b/frontend/test-support/polling.test.ts @@ -0,0 +1,317 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import type { EventBatch } from "../sync-client/src/services/types/EventBatch"; +import type { EventRecord } from "../sync-client/src/services/types/EventRecord"; +import type { PutFileContent } from "../sync-client/src/services/types/PutFileContent"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +const documentId = "00000000-0000-4000-8000-000000000001"; +const interval = 1_000; +const retryInterval = 75; +const bytes = (text: string) => new TextEncoder().encode(text); + +/** Real sync loop over in-memory disk/persistence and an observable version + * store. The transport never wakes the engine itself. */ +async function fixture(syncIntervalMs: number | undefined) { + const disk = new MemoryDisk(); + await disk.userWrite("note.bin", bytes("base")); + let headEventId = 2; // Initial content (1), then manifest (2). + const versions = new Map([[1, bytes("base")]]); + const head = (version: number) => ({ + documentId, + vaultUpdateId: version, + contentSize: versions.get(version)!.length, + updatedDate: "2026-09-20T00:00:00Z", + userId: "user", + deviceId: "device" + }); + let latest = head(1); + const events: EventRecord[] = []; + const writes: PutFileContent[] = []; + let eventReads = 0; + let failNextRead = false; + const publishContent = (content: Uint8Array, requestId: string) => { + const eventId = ++headEventId; + versions.set(eventId, new Uint8Array(content)); + latest = head(eventId); + events.push({ eventId, requestId, type: "content", document: latest }); + return latest; + }; + const batch = (after = 2): EventBatch => + structuredClone({ + headEventId, + events: events.filter((event) => event.eventId > after) + }); + const service: Partial = { + events: async (after) => { + eventReads++; + if (failNextRead) { + failNextRead = false; + throw new Error("offline"); + } + return batch(after); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => { + const content = versions.get(vaultUpdateId); + assert(content, `Missing immutable version ${vaultUpdateId}`); + return content.slice(); + }, + putFileContent: async (id, request) => { + assert.equal(id, documentId); + assert.equal(request.parentVersionId, latest.vaultUpdateId); + assert(request.content.type === "Snapshot"); + writes.push(structuredClone(request)); + return { + type: "Accepted", + ...publishContent( + Buffer.from(request.content.value, "base64"), + request.requestId + ) + }; + } + }; + const initial = emptyState("polling-test"); + initial.initialized = true; + initial.lastSeenUpdateId = 2; + initial.local = { [documentId]: "note.bin" }; + initial.fileManifest = { fileManifestId: 2, entries: { ...initial.local } }; + initial.remoteHeads = { [documentId]: latest }; + const hash = (await toStoredSnapshot({ content: bytes("base") })).hash; + initial.documents[documentId] = { + materialized: true, + observedHash: hash, + base: { ...latest, hash } + }; + const persistence = new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + initial.vaultKey, + async () => (await persistence.load()).database as StoredDatabase + ); + const settings = new Settings( + logger, + { syncIntervalMs, networkRetryIntervalMs: retryInterval }, + async () => {} + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: [] }), + reset: () => {} + } as unknown as ServerConfig; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => Promise + >() + }; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket as unknown as WebSocketManager, + new FileOperations(disk.session(), database, config), + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(0) + ); + return { + disk, + database, + syncer, + writes, + batch, + websocket, + reads: () => eventReads, + serverContent: () => versions.get(latest.vaultUpdateId), + failNextRead: () => { + failNextRead = true; + }, + changeRemote: () => { + publishContent(bytes("remote edit"), "remote-content"); + const eventId = ++headEventId; + events.push({ + eventId, + requestId: "remote-rename", + type: "fileManifest", + fileManifest: { + fileManifestId: eventId, + entries: { [documentId]: "renamed.bin" } + } + }); + } + }; +} + +for (const syncIntervalMs of [undefined, 0, interval]) { + for (const side of ["local", "remote"] as const) { + test(`${side} changes without notifications ${syncIntervalMs ? "are recovered by explicit polling" : `remain unobserved when polling is ${syncIntervalMs === 0 ? "zero" : "unset"}`}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + const f = await fixture(syncIntervalMs); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + const reads = f.reads(); + if (side === "local") + await f.disk.userWrite("note.bin", bytes("local edit")); + else f.changeRemote(); + const assertUnobserved = () => { + assert.equal( + f.reads(), + reads, + "A hidden wakeup fetched events" + ); + assert.equal(f.database.state.lastSeenUpdateId, 2); + assert.equal(f.writes.length, 0); + assert.deepEqual( + f.serverContent(), + bytes(side === "local" ? "base" : "remote edit") + ); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + [ + "note.bin", + bytes(side === "local" ? "local edit" : "base") + ] + ]) + ); + }; + context.mock.timers.tick( + syncIntervalMs ? interval - 1 : 60_000 + ); + await f.syncer.waitUntilFinished(); + assertUnobserved(); + if (syncIntervalMs) context.mock.timers.tick(1); + else if (side === "local") + await f.syncer.syncLocallyUpdatedFile({ + relativePath: "note.bin" + }); + else + await f.websocket.onRemoteVaultUpdateReceived.triggerAsync( + f.batch() + ); + await f.syncer.waitUntilFinished(); + const content = bytes( + side === "local" ? "local edit" : "remote edit" + ); + const path = side === "local" ? "note.bin" : "renamed.bin"; + assert.deepEqual(f.serverContent(), content); + assert.deepEqual( + f.disk.userFiles(), + new Map([[path, content]]) + ); + assert.deepEqual(f.database.state.local, { + [documentId]: path + }); + assert.equal( + f.database.state.lastSeenUpdateId, + side === "local" ? 3 : 4 + ); + assert.equal(f.syncer.isBusy, false); + assert.equal(f.writes.length, side === "local" ? 1 : 0); + if (syncIntervalMs) { + const beforeNextPoll = f.reads(); + context.mock.timers.tick(interval); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + beforeNextPoll + 1, + "Polling stopped after its first successful run" + ); + assert.equal( + f.writes.length, + side === "local" ? 1 : 0, + "An idle poll resubmitted unchanged content" + ); + } + await f.syncer.stop(); + const stoppedReads = f.reads(); + context.mock.timers.tick(60_000); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + stoppedReads, + "Polling continued after stop" + ); + } finally { + await f.syncer.stop(); + context.mock.timers.reset(); + } + }); + } +} + +test("transient failures retry with polling disabled, then leave the client idle", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + const f = await fixture(0); + try { + f.changeRemote(); + f.failNextRead(); + f.syncer.start(); + await assert.rejects(f.syncer.waitUntilFinished(), /offline/); + const reads = f.reads(); + context.mock.timers.tick(retryInterval - 1); + await assert.rejects(f.syncer.waitUntilFinished(), /offline/); + assert.equal(f.reads(), reads); + context.mock.timers.tick(1); + await f.syncer.waitUntilFinished(); + assert.equal(f.reads(), reads + 1); + assert.deepEqual( + f.disk.userFiles(), + new Map([["renamed.bin", bytes("remote edit")]]) + ); + assert.equal(f.database.state.lastSeenUpdateId, 4); + assert.equal(f.syncer.isBusy, false); + context.mock.timers.tick(60_000); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + reads + 1, + "Network retries became clean-state polling" + ); + } finally { + await f.syncer.stop(); + context.mock.timers.reset(); + } +}); + +test("a failed WebSocket connection catches up over HTTP even when polling is disabled", async () => { + const f = await fixture(0); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + f.changeRemote(); + f.websocket.onWebSocketStatusChanged.trigger(false); + await f.syncer.waitUntilFinished(); + assert.equal(f.database.state.lastSeenUpdateId, 4); + assert.deepEqual( + f.disk.userFiles(), + new Map([["renamed.bin", bytes("remote edit")]]) + ); + } finally { + await f.syncer.stop(); + } +}); diff --git a/frontend/test-support/protocol-audit.test.ts b/frontend/test-support/protocol-audit.test.ts new file mode 100644 index 00000000..2cb7d782 --- /dev/null +++ b/frontend/test-support/protocol-audit.test.ts @@ -0,0 +1,392 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { resolve, join } from "node:path"; +import { tmpdir } from "node:os"; +import { mkdtemp, rm } from "node:fs/promises"; +const root = resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const bytes = (s: string) => new TextEncoder().encode(s); +const headers = { + Authorization: `Bearer ${token}`, + "Device-Id": "audit", + "Content-Type": "application/json" +}; +async function put(url: string, body: unknown) { + const r = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(body) + }); + assert(r.ok, await r.clone().text()); + return r.json(); +} +const content = (s: string) => ({ + requestId: randomUUID(), + parentVersionId: null, + content: { type: "Snapshot", value: Buffer.from(s).toString("base64") } +}); + +test( + "connection admission rejects excess clients and releases disconnected slots", + { timeout: 15000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "vault-link-admission-")); + const config = join(dir, "config.yml"); + await writeFile( + config, + ( + await readFile(root + "/sync-server/config-e2e.yml", "utf8") + ).replace("max_clients_per_vault: 256", "max_clients_per_vault: 1") + ); + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + config, + new Logger() + ); + const sockets: WebSocket[] = []; + try { + await server.start(); + const url = server.remoteUri + "/vaults/" + randomUUID(); + await put(url + "/documents/" + randomUUID(), content("event")); + const connect = async () => { + const ws = new WebSocket(url.replace("http:", "ws:") + "/ws"); + sockets.push(ws); + const accepted = await new Promise((resolve) => { + ws.onopen = () => + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId: randomUUID(), + lastSeenVaultUpdateId: 0 + }) + ); + ws.onmessage = () => resolve(true); + ws.onerror = () => resolve(false); + ws.onclose = () => resolve(false); + }); + return { ws, accepted }; + }; + const first = await connect(); + assert(first.accepted); + assert.equal((await connect()).accepted, false); + await new Promise((resolve) => { + first.ws.onclose = () => resolve(); + first.ws.close(); + }); + // A completed close releases the server's owned connection permit. + assert.equal((await connect()).accepted, true); + } finally { + for (const ws of sockets) ws.close(); + await server.stop(); + await rm(dir, { recursive: true, force: true }); + } + } +); + +test( + "an unauthenticated upgraded socket expires without a handshake", + { timeout: 10000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let ws: WebSocket | undefined; + try { + await server.start(); + ws = new WebSocket( + server.remoteUri.replace("http:", "ws:") + + "/vaults/" + + randomUUID() + + "/ws" + ); + const socket = ws; + const closed = await new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), 6500); + socket.onclose = () => { + clearTimeout(timer); + resolve(true); + }; + socket.onerror = () => {}; + }); + assert(closed, "handshake deadline must release idle sockets"); + } finally { + ws?.close(); + await server.stop(); + } + } +); + +test( + "bootstrap notifications adopt the existing remote document identity", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let client: SyncClient | undefined; + try { + await server.start(); + const vault = randomUUID(), + id = randomUUID(); + const url = server.remoteUri + "/vaults/" + vault; + await put(url + "/documents/" + id, content("REMOTE")); + await put(url + "/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0 + } + }); + client = await SyncClient.create({ fs: disk, persistence: store }); + await disk.userWrite("a.md", bytes("LOCAL")); + await client.syncLocallyCreatedFile("a.md"); + await disk.userWrite("b.md", bytes("B")); + await client.syncLocallyCreatedFile("b.md"); + await client.start(); + const files = [...disk.userFiles().keys()]; + assert.equal(files.length, 2); + } finally { + await client?.destroy(); + await server.stop(); + } + } +); + +test( + "local undo survives a lost accepted reply followed by a rejected retry", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let client: SyncClient | undefined; + class SilentWebSocket { + static OPEN = 1; + readyState = 1; + onopen: ((event: object) => void) | null = null; + onclose: + | ((event: { code: number; reason: string }) => void) + | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: ((event: object) => void) | null = null; + constructor(_url: string | URL) { + queueMicrotask(() => this.onopen?.({})); + } + send(_data: string) {} + close() { + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + try { + await server.start(); + const vault = randomUUID(), + id = randomUUID(); + const url = server.remoteUri + "/vaults/" + vault; + await put(url + "/documents/" + id, content("base")); + await put(url + "/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0, + networkRetryIntervalMs: 0 + } + }); + let mode: "normal" | "lose-reply" | "reject-retry" = "normal"; + let lostId: string | undefined; + client = await SyncClient.create({ + fs: disk, + persistence: store, + webSocket: SilentWebSocket as unknown as typeof WebSocket, + fetch: async (input, init) => { + if (init?.method === "PUT" && mode === "reject-retry") { + assert.equal( + JSON.parse(String(init.body)).requestId, + lostId + ); + mode = "normal"; + return new Response("proxy body limit changed", { + status: 413 + }); + } + const response = await fetch(input, init); + if (init?.method === "PUT" && mode === "lose-reply") { + assert.equal( + (await response.clone().json()).type, + "Accepted" + ); + lostId = JSON.parse(String(init.body)).requestId; + mode = "normal"; + throw new Error("injected lost accepted response"); + } + return response; + } + }); + await client.start(); + mode = "lose-reply"; + await disk.userWrite("a.md", bytes("base A")); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + await assert.rejects(client.waitUntilFinished(), /lost accepted/); + const original = await ( + await fetch(url + "/documents/" + id, { headers }) + ).json(); + await put(url + "/documents/" + id, { + ...content("base A REMOTE"), + parentVersionId: original.vaultUpdateId + }); + mode = "reject-retry"; + await disk.userWrite("a.md", bytes("base")); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + await client.waitUntilFinished(); + const result = Buffer.from( + (await disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(result, "base REMOTE"); + } finally { + await client?.destroy(); + await server.stop(); + } + } +); + +test( + "closing an old connection cannot erase its replacement's cursors", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + const sockets: WebSocket[] = []; + try { + await server.start(); + const url = server.remoteUri + "/vaults/" + randomUUID(); + const id = randomUUID(); + await put(url + "/documents/" + id, content("example")); + const connect = async (deviceId: string) => { + const ws = new WebSocket(url.replace("http:", "ws:") + "/ws"); + sockets.push(ws); + await new Promise((resolve, reject) => { + ws.onopen = () => + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId, + lastSeenVaultUpdateId: 0 + }) + ); + ws.onmessage = () => resolve(); + ws.onerror = () => reject(new Error("connection failed")); + }); + return ws; + }; + const observer = await connect("observer"); + const first = await connect("same-device"); + const sendCursor = (ws: WebSocket, start: number) => + ws.send( + JSON.stringify({ + type: "cursorPositions", + documentsWithCursors: [ + { + document_id: id, + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start, end: start }] + } + ] + }) + ); + const seen = (start: number) => + new Promise((resolve) => { + observer.onmessage = (event) => { + const message = JSON.parse(String(event.data)); + if ( + message.type === "cursorPositions" && + message.clients.some( + (client: { + deviceId: string; + documentsWithCursors: { + cursors: { start: number }[]; + }[]; + }) => + client.deviceId === "same-device" && + client.documentsWithCursors[0]?.cursors[0] + ?.start === start + ) + ) + resolve(); + }; + }); + let observed = seen(1); + sendCursor(first, 1); + await observed; + const replacement = await connect("same-device"); + observed = seen(2); + sendCursor(replacement, 2); + await observed; + await new Promise((resolve) => { + first.onclose = () => resolve(); + first.close(); + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const current = new Promise< + { + deviceId: string; + documentsWithCursors: { cursors: { start: number }[] }[]; + }[] + >((resolve) => { + observer.onmessage = (event) => { + const message = JSON.parse(String(event.data)); + if (message.type === "cursorPositions") + resolve(message.clients); + }; + }); + observer.send( + JSON.stringify({ + type: "cursorPositions", + documentsWithCursors: [] + }) + ); + const clients = await current; + assert.equal( + clients.find((client) => client.deviceId === "same-device") + ?.documentsWithCursors[0]?.cursors[0]?.start, + 2 + ); + } finally { + for (const ws of sockets) ws.close(); + await server.stop(); + } + } +); diff --git a/frontend/test-support/protocol.test.ts b/frontend/test-support/protocol.test.ts new file mode 100644 index 00000000..d0652d79 --- /dev/null +++ b/frontend/test-support/protocol.test.ts @@ -0,0 +1,420 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { Worker } from "node:worker_threads"; +import * as path from "node:path"; +import { Logger } from "sync-client"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { MemoryDisk, type DiskImage, type StoredClient } from "./storage"; +import { getJson, type CanonicalSnapshot } from "./canonical"; +import type { RequestRecord } from "./network"; +import type { CrashWorkerInput } from "../deterministic-tests/src/crash-worker"; + +const root = path.resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const headers = { + Authorization: `Bearer ${token}`, + "Device-Id": "protocol-test", + "Content-Type": "application/json" +}; +const makeServer = () => + new ServerControl( + path.join(root, "sync-server/target/release/sync_server"), + path.join(root, "sync-server/config-e2e.yml"), + new Logger() + ); +async function put(url: string, body: unknown) { + const response = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(5_000) + }); + assert( + response.ok, + `HTTP ${response.status}: ${await response.clone().text()}` + ); + return (await response.json()) as { + type: string; + vaultUpdateId: number; + fileManifestId: number; + }; +} +const content = (text: string, parentVersionId: number | null = null) => ({ + requestId: randomUUID(), + parentVersionId, + content: { type: "Snapshot", value: Buffer.from(text).toString("base64") } +}); + +interface WorkerSnapshot { + type: string; + disk: DiskImage; + stored: StoredClient; + requests: RequestRecord[]; + editedBase?: { documentId: string; vaultUpdateId: number }; +} +async function runWorker(input: CrashWorkerInput): Promise { + const worker = new Worker( + path.join(root, "frontend/deterministic-tests/dist/crash-worker.js"), + { workerData: input } + ); + let timeout: ReturnType | undefined; + try { + return await new Promise((resolve, reject) => { + timeout = setTimeout( + () => + reject(new Error("Worker checkpoint/completion timed out")), + 15_000 + ); + worker.once("error", reject); + worker.once("exit", (code) => + reject( + new Error( + `Worker exited (${code}) before expected checkpoint` + ) + ) + ); + worker.on( + "message", + (message: WorkerSnapshot & { error?: string }) => { + if (message.type === "error") + reject(new Error(message.error)); + else { + assert.equal( + message.type, + input.stopAt ? "checkpoint" : "finished" + ); + resolve(message); + } + } + ); + }); + } finally { + clearTimeout(timeout); + // This is the fault injection, not graceful shutdown masquerading as it. + await worker.terminate(); + } +} + +for (const [kind, format, originalPath] of [ + ["create", "Snapshot", "original.md"], + ["content", "Snapshot", "original.bin"], + ["content", "Diff", "original.md"], + ["manifest", undefined, "original.md"] +] as const) { + test( + `abrupt client death after accepted ${kind} ${format ?? ""} reply, newer head, server SIGKILL, durable retry`, + { timeout: 30_000 }, + async () => { + const server = makeServer(); + try { + await server.start(); + const vault = `crash-${randomUUID()}`; + const url = `${server.remoteUri}/vaults/${vault}`; + const disk = new MemoryDisk(); + await disk.userWrite( + originalPath, + new TextEncoder().encode("preserve-original\n") + ); + let input: CrashWorkerInput = { + disk: disk.image(), + stored: { + settings: { + isSyncEnabled: true, + enableTelemetry: false, + remoteUri: server.remoteUri, + vaultName: vault, + token, + requestTimeoutMs: 5_000, + syncIntervalMs: 0 + } + } as StoredClient + }; + let originalId: string | undefined; + let originalParent: number | undefined; + let uploaded = "preserve-original\n"; + if (kind === "content" && format === "Diff") { + // Keep the established base in the worker's real content + // cache; a cold restart intentionally falls back to Snapshot. + uploaded += "local-update\n"; + input.editAfterSync = { + path: originalPath, + content: uploaded + }; + } else if (kind === "content") { + const initialized = await runWorker(input); + originalId = Object.keys( + initialized.stored.database!.local! + )[0]; + originalParent = + initialized.stored.database!.documents![originalId] + .base!.vaultUpdateId; + disk.restore(initialized.disk); + uploaded += "local-update\n"; + await disk.userWrite( + originalPath, + new TextEncoder().encode(uploaded) + ); + input = { disk: disk.image(), stored: initialized.stored }; + } + const checkpoint = await runWorker({ ...input, stopAt: kind }); + if (format === "Diff") { + assert( + checkpoint.editedBase, + "Warm update baseline was not recorded" + ); + originalId = checkpoint.editedBase.documentId; + originalParent = checkpoint.editedBase.vaultUpdateId; + } + const pending = checkpoint.stored.database!.pending!; + assert( + pending, + "The uncertain request was not persisted before sending" + ); + const interrupted = checkpoint.requests.at(-1)!; + assert.equal( + interrupted.kind, + kind, + "Wrong CAS was interrupted" + ); + if (kind !== "manifest") { + assert.equal(pending.type, "content"); + assert(pending.type === "content"); + assert.equal(pending.request.content.type, format); + assert.equal( + pending.request.parentVersionId, + kind === "create" ? null : originalParent + ); + if (originalId) + assert.equal( + pending.documentId, + originalId, + "Update created a replacement UUID" + ); + } + const accepted = await getJson( + `${url}/vault-snapshot`, + token + ); + // Unpublished content has no manifest membership yet, so it is + // deliberately absent from /vault-snapshot's document list. + const doc = + pending.type === "content" + ? await getJson<{ + documentId: string; + vaultUpdateId: number; + }>(`${url}/documents/${pending.documentId}`, token) + : accepted.documents[0]; + assert(doc); + const remote = await put( + `${url}/documents/${doc.documentId}`, + content(`${uploaded}remote-update\n`, doc.vaultUpdateId) + ); + assert.equal(remote.type, "Accepted"); + const remotePath = originalPath.replace("original", "remote"); + const renamed = await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: accepted.fileManifest.fileManifestId, + entries: { [doc.documentId]: remotePath } + }); + assert.equal(renamed.type, "Accepted"); + const before = await getJson( + `${url}/vault-snapshot`, + token + ); + await server.crash(); + await server.restart(); + assert.deepEqual( + await getJson(`${url}/vault-snapshot`, token), + before, + "Server restart lost acknowledged state" + ); + const restarted = await runWorker({ + disk: checkpoint.disk, + stored: checkpoint.stored + }); + const replay = restarted.requests.find( + (r) => r.body.requestId === pending.request.requestId + ); + assert(replay, "Restart never retried the uncertain request"); + assert.equal( + replay.url, + interrupted.url, + "Restart changed the request target" + ); + assert.deepEqual( + replay.body, + pending.request, + "Restart changed the persisted request" + ); + assert.equal(restarted.stored.database!.pending, undefined); + assert.equal(restarted.stored.database!.application, undefined); + const recovered = new MemoryDisk(); + recovered.restore(restarted.disk); + assert.deepEqual( + recovered.userFiles(), + new Map([ + [ + remotePath, + new TextEncoder().encode( + `${uploaded}remote-update\n` + ) + ] + ]) + ); + assert.deepEqual(restarted.stored.database!.local, { + [doc.documentId]: remotePath + }); + const events = await getJson<{ + events: { requestId: string }[]; + }>(`${url}/events-since?after=0`, token); + assert.equal( + events.events.filter( + (e) => e.requestId === pending.request.requestId + ).length, + 1, + "Retry emitted a duplicate event" + ); + } finally { + await server.stop(); + } + } + ); +} + +test( + "concurrent CAS commits have identical HTTP/event-log/WebSocket total order and reconnect catchup", + { timeout: 20_000 }, + async () => { + const server = makeServer(); + let ws: WebSocket | undefined; + try { + await server.start(); + const url = `${server.remoteUri}/vaults/order-${randomUUID()}`; + const received: number[] = []; + const connect = async (after: number) => { + ws = new WebSocket(`${url.replace(/^http/, "ws")}/ws`); + ws.addEventListener("message", (message) => { + const batch = JSON.parse(String(message.data)) as { + type: string; + events: { eventId: number }[]; + }; + if (batch.type === "vaultEvents") + received.push( + ...batch.events.map((event) => event.eventId) + ); + }); + await new Promise((resolve, reject) => { + ws!.onopen = () => resolve(); + ws!.onerror = () => reject(new Error("WebSocket failed")); + }); + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId: "order-observer", + lastSeenVaultUpdateId: after + }) + ); + }; + await connect(0); + const ids = Array.from({ length: 12 }, () => randomUUID()); + const requests = ids.map((id, i) => ({ + id, + body: content(`bytes-${i}`) + })); + const replies = await Promise.all( + requests.map(({ id, body }) => + put(`${url}/documents/${id}`, body) + ) + ); + assert.deepEqual( + replies.map((r) => r.vaultUpdateId).sort((a, b) => a - b), + ids.map((_, i) => i + 1) + ); + const emptyManifest = await getJson<{ + entries: Record; + }>(`${url}/file-manifest`, token); + assert.deepEqual( + emptyManifest.entries, + {}, + "Content-only CAS changed membership" + ); + const entries = Object.fromEntries( + ids.map((id, i) => [id, `${i}.md`]) + ); + await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: 0, + entries + }); + // All retries must return original receipts without emitting events. + assert.deepEqual( + await Promise.all( + requests.map(({ id, body }) => + put(`${url}/documents/${id}`, body) + ) + ), + replies + ); + const waitForEvents = async (head: number) => { + const deadline = Date.now() + 5_000; + while (received.length < head && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.deepEqual( + received, + Array.from({ length: head }, (_, i) => i + 1), + "WebSocket reordered, lost or duplicated an event" + ); + }; + await waitForEvents(13); + await new Promise((resolve) => { + ws!.onclose = () => resolve(); + ws!.close(); + }); + await put( + `${url}/documents/${ids[0]}`, + content("next", replies[0].vaultUpdateId) + ); + await connect(13); + await waitForEvents(14); + const log = await getJson<{ events: { eventId: number }[] }>( + `${url}/events-since?after=0`, + token + ); + assert.deepEqual( + received, + log.events.map((event) => event.eventId) + ); + const stale = await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: {} + }); + assert.equal(stale.type, "StaleBase"); + const invalid = await fetch(`${url}/file-manifest`, { + method: "PUT", + headers, + body: JSON.stringify({ + requestId: randomUUID(), + parentFileManifestId: 13, + entries: { [ids[0]]: "same", [ids[1]]: "same" } + }) + }); + assert.equal(invalid.status, 422); + assert.equal( + ( + await getJson( + `${url}/vault-snapshot`, + token + ) + ).headEventId, + 14, + "Rejected CAS emitted an event" + ); + } finally { + ws?.close(); + await server.stop(); + } + } +); diff --git a/frontend/test-support/random.ts b/frontend/test-support/random.ts new file mode 100644 index 00000000..f451ae47 --- /dev/null +++ b/frontend/test-support/random.ts @@ -0,0 +1,25 @@ +/** A seed controls harness choices; production UUIDs and OS scheduling remain + * nondeterministic. The concrete action trace is the stronger replay artifact. */ +export class Random { + private state: number; + public constructor(seed: number) { + if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) + throw new Error("Seed must be a uint32"); + this.state = seed; + } + public next(): number { + this.state = (this.state + 0x6d2b79f5) >>> 0; + let n = this.state; + n = Math.imul(n ^ (n >>> 15), n | 1); + n ^= n + Math.imul(n ^ (n >>> 7), n | 61); + return ((n ^ (n >>> 14)) >>> 0) / 4294967296; + } + public int(max: number): number { + if (!Number.isSafeInteger(max) || max < 1) + throw new Error("Empty random choice"); + return Math.floor(this.next() * max); + } + public pick(values: readonly T[]): T { + return values[this.int(values.length)]; + } +} diff --git a/frontend/test-support/real-disk.test.ts b/frontend/test-support/real-disk.test.ts new file mode 100644 index 00000000..3b917c81 --- /dev/null +++ b/frontend/test-support/real-disk.test.ts @@ -0,0 +1,137 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { RealDisk } from "./real-disk"; +import { Database, emptyState } from "../sync-client/src/persistence/database"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Logger } from "../sync-client/src/tracing/logger"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; + +test( + "real filesystem: exclusive operations, unsafe entries, and journal swaps", + { timeout: 20_000 }, + async () => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vault-link-real-fs-") + ); + try { + const root = path.join(directory, "vault"); + await fs.mkdir(root); + const helper = path.join(directory, "native-fs"); + await promisify(execFile)("cc", [ + "-Wall", + "-Wextra", + "-O2", + path.join(__dirname, "native-fs.c"), + "-o", + helper + ]); + const disk = new RealDisk(root, helper); + const snapshot = (text: string) => ({ + content: new TextEncoder().encode(text) + }); + const writes = await Promise.allSettled([ + disk.write("raced.bin", snapshot("first")), + disk.write("raced.bin", snapshot("second")) + ]); + assert.equal( + writes.filter((result) => result.status === "fulfilled").length, + 1, + "Exclusive create overwrote a racing writer" + ); + assert( + ["first", "second"].includes( + Buffer.from( + (await disk.readSnapshot("raced.bin"))!.content + ).toString() + ) + ); + await disk.write("a.md", snapshot("A")); + await disk.write("b.md", snapshot("B")); + // Recovery must flush the old source's ancestors even if another + // writer has replaced one of those directories with a file. + assert.equal(await disk.stat("a.md/absent.md"), undefined); + assert.equal(await disk.readSnapshot("a.md/absent.md"), undefined); + await disk.flushPaths(["a.md/absent.md"]); + await assert.rejects(disk.rename("a.md", "b.md")); + await assert.rejects(disk.delete("a.md"), /regular file/); + await fs.symlink("a.md", path.join(root, "symlink")); + await assert.rejects(disk.readSnapshot("symlink"), /Symlink/); + await fs.unlink(path.join(root, "symlink")); + await fs.link(path.join(root, "a.md"), path.join(root, "hardlink")); + await assert.rejects(disk.readSnapshot("a.md"), /hardlink/); + await fs.unlink(path.join(root, "hardlink")); + // The host may be case/normalization insensitive. Whichever model + // it exposes, an alias must never silently overwrite existing bytes. + for (const [original, alias] of [ + ["Case.bin", "case.bin"], + ["é.bin", "e\u0301.bin"] + ]) { + await disk.write(original, snapshot("original bytes")); + if (await disk.exists(alias)) + await assert.rejects( + disk.write(alias, snapshot("alias bytes")) + ); + else { + await disk.write(alias, snapshot("alias bytes")); + assert.equal( + Buffer.from( + (await disk.readSnapshot(alias))!.content + ).toString(), + "alias bytes" + ); + } + assert.equal( + Buffer.from( + (await disk.readSnapshot(original))!.content + ).toString(), + "original bytes" + ); + } + const initial = emptyState("real"); + initial.local = { a: "a.md", b: "b.md" }; + for (const [id, relative] of Object.entries(initial.local)) + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot( + (await disk.readSnapshot(relative))! + ) + ).hash + }; + const database = new Database( + new Logger(), + initial, + async () => {}, + "real", + async () => initial + ); + const operations = new FileOperations(disk, database, { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig); + const next = structuredClone(initial); + next.local = { a: "b.md", b: "a.md" }; + await operations.apply(next); + assert.equal( + Buffer.from( + (await disk.readSnapshot("a.md"))!.content + ).toString(), + "B" + ); + assert.equal( + Buffer.from( + (await disk.readSnapshot("b.md"))!.content + ).toString(), + "A" + ); + assert.equal(database.state.application, undefined); + } finally { + await fs.rm(directory, { recursive: true }); + } + } +); diff --git a/frontend/test-support/real-disk.ts b/frontend/test-support/real-disk.ts new file mode 100644 index 00000000..d3c42639 --- /dev/null +++ b/frontend/test-support/real-disk.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { constants } from "node:fs"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { randomUUID } from "node:crypto"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +const exec = promisify(execFile); +/** Small integration fixture, NOT a supported application adapter. Safety + * preflights detect static symlinks/hardlinks; they are not an openat sandbox + * against a hostile process replacing ancestors concurrently. */ +export class RealDisk implements FileSystemOperations { + public constructor( + private readonly root: string, + private readonly helper: string + ) {} + private async resolve(relative: string): Promise { + assert( + relative === "" || + relative + .split("/") + .every( + (p) => p && p !== "." && p !== ".." && !p.includes("\\") + ), + "Unsafe path" + ); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + current = path.join(current, component); + const entry = await this.info(current); + if (entry) + assert( + entry.isDirectory() || + (entry.isFile() && entry.nlink === 1), + "Symlink, hardlink or special file" + ); + } + return current; + } + private async info(absolute: string) { + try { + return await fs.lstat(absolute); + } catch (error) { + if ( + ["ENOENT", "ENOTDIR"].includes( + (error as NodeJS.ErrnoException).code ?? "" + ) + ) + return undefined; + throw error; + } + } + private async flush(absolute: string): Promise { + const handle = await fs.open( + absolute, + constants.O_RDONLY | constants.O_NOFOLLOW + ); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } + public async stat(relative: string) { + const info = await this.info(await this.resolve(relative)); + return info + ? { + kind: info.isFile() + ? ("file" as const) + : ("directory" as const), + size: info.size + } + : undefined; + } + public async exists(relative: string) { + return (await this.stat(relative)) !== undefined; + } + public async readSnapshot( + relative: string + ): Promise { + const absolute = await this.resolve(relative); + let handle; + try { + handle = await fs.open( + absolute, + constants.O_RDONLY | constants.O_NOFOLLOW + ); + } catch (error) { + if ( + ["ENOENT", "ENOTDIR"].includes( + (error as NodeJS.ErrnoException).code ?? "" + ) + ) + return undefined; + throw error; + } + try { + const before = await handle.stat({ bigint: true }); + assert( + before.isFile() && before.nlink === 1n, + "Not an independent regular file" + ); + const content = new Uint8Array(await handle.readFile()); + const after = await handle.stat({ bigint: true }); + const atPath = await fs.lstat(absolute, { bigint: true }); + assert( + before.ino === after.ino && + after.ino === atPath.ino && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs, + "File changed during read" + ); + return { content }; + } finally { + await handle.close(); + } + } + public async listFilesRecursively(root = ""): Promise { + const result: string[] = []; + for (const name of await fs.readdir(await this.resolve(root))) { + const relative = root ? `${root}/${name}` : name; + if ((await this.stat(relative))!.kind === "directory") + result.push(...(await this.listFilesRecursively(relative))); + else result.push(relative); + } + return result.sort(); + } + public async createDirectory(relative: string): Promise { + await this.resolve(relative); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + const next = path.join(current, component); + try { + await fs.mkdir(next); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") + throw error; + } + assert((await fs.lstat(next)).isDirectory()); + await this.flush(next); + await this.flush(current); + current = next; + } + } + public async write( + relative: string, + snapshot: FileSnapshot + ): Promise { + assert( + !snapshot.cursors?.length, + "This disk-only fixture has no editor selections" + ); + const destination = await this.resolve(relative); + const staging = ".vault-link-sync/adapter-staging"; + await this.createDirectory(staging); + const source = path.join(this.root, staging, randomUUID()); + const handle = await fs.open(source, "wx", 0o600); + try { + await handle.writeFile(snapshot.content); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await exec(this.helper, ["rename", source, destination]); + await this.flush(path.dirname(source)); + await this.flush(path.dirname(destination)); + } finally { + await fs.unlink(source).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }); + } + } + public async rename(from: string, to: string): Promise { + const source = await this.resolve(from), + destination = await this.resolve(to); + assert((await this.info(source))?.isFile()); + await this.flush(source); + await exec(this.helper, ["rename", source, destination]); + await this.flush(path.dirname(source)); + await this.flush(path.dirname(destination)); + } + public async flushPaths(paths: readonly string[]): Promise { + for (const relative of paths) { + let absolute = await this.resolve(relative); + for (;;) { + if (await this.info(absolute)) await this.flush(absolute); + if (absolute === this.root) break; + absolute = path.dirname(absolute); + } + } + } + public async delete(relative: string): Promise { + assert(relative, "Cannot prune root"); + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isDirectory(), "Cannot delete regular file"); + for (const child of await fs.readdir(absolute)) + await this.delete(`${relative}/${child}`); + await fs.rmdir(absolute); + await this.flush(path.dirname(absolute)); + } + public async deleteFile(relative: string): Promise { + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); + await fs.unlink(absolute); + await this.flush(path.dirname(absolute)); + } +} diff --git a/frontend/test-support/recovery.test.ts b/frontend/test-support/recovery.test.ts new file mode 100644 index 00000000..8da22c4a --- /dev/null +++ b/frontend/test-support/recovery.test.ts @@ -0,0 +1,695 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { isDeepStrictEqual } from "node:util"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import { mergeFileManifests } from "../sync-client/src/sync-operations/file-manifest"; +import { assertManifest, pathDecision } from "./oracles"; +import { + scanLocalFiles, + type LocalChange +} from "../sync-client/src/sync-operations/scan"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const config = { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) +} as ServerConfig; + +async function fixture( + contents: Record = { + a: { path: "a.md", text: "A" }, + b: { path: "b.md", text: "B" } + } +) { + const disk = new MemoryDisk(); + const initial = emptyState("recovery-test"); + for (const [id, { path, text }] of Object.entries(contents)) { + await disk.userWrite(path, bytes(text)); + initial.local[id] = path; + } + for (const [id, path] of Object.entries(initial.local)) + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot((await disk.readSnapshot(path))!) + ).hash + }; + const persistence = new MemoryPersistence({ database: initial }); + const restart = (changes: LocalChange[] = []) => { + const database = new Database( + new Logger(), + persistence.snapshot().database, + async (next) => persistence.save({ database: next }), + "recovery-test", + async () => (await persistence.load()).database as StoredDatabase + ); + return { + database, + files: new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} } + ) + }; + }; + return { + disk, + persistence, + restart, + initial, + permanentFault: (_label: string): void => {}, + expectedLocal: undefined as Record | undefined + }; +} + +async function swapFixture() { + const current = await fixture(); + const next = structuredClone(current.initial); + next.local = { a: "b.md", b: "a.md", c: "nested/c.md" }; + next.documents.c = { materialized: false }; + const writes = { + c: { replacement: await toStoredSnapshot({ content: bytes("C") }) } + }; + const expected = new Map([ + ["a.md", bytes("B")], + ["b.md", bytes("A")], + ["nested/c.md", bytes("C")] + ]); + return { ...current, next, writes, expected, retained: [] as string[] }; +} + +async function crossDirectoryFixture() { + const current = await fixture({ + a: { path: "src/a.md", text: "A" }, + // Keep src nonempty: deleting an empty source directory would mask an + // unflushed source unlink in the staging rename. + b: { path: "src/keep.md", text: "untouched sibling" } + }); + const next = structuredClone(current.initial); + next.local.a = "dst/a.md"; + const expected = new Map([ + ["dst/a.md", bytes("A")], + ["src/keep.md", bytes("untouched sibling")] + ]); + return { ...current, next, writes: {}, expected, retained: [] as string[] }; +} + +async function mergeFixture() { + const base = "first paragraph\n\nlast paragraph\n"; + const local = "local first paragraph\n\nlast paragraph\n"; + const remote = "first paragraph\n\nremote last paragraph\n"; + const current = await fixture({ a: { path: "note.md", text: local } }); + return { + ...current, + next: structuredClone(current.initial), + writes: { + a: { + expected: await toStoredSnapshot({ content: bytes(base) }), + replacement: await toStoredSnapshot({ content: bytes(remote) }) + } + }, + expected: new Map([ + [ + "note.md", + bytes("local first paragraph\n\nremote last paragraph\n") + ] + ]), + retained: [local] + }; +} + +async function deletionFixture() { + const current = await fixture({ + a: { path: "a.bin", text: "private binary edit\0" }, + b: { path: "deleted.md", text: "unsent deleted bytes" } + }); + const next = structuredClone(current.initial); + delete next.local.b; + return { + ...current, + next, + writes: { + a: { + replacement: await toStoredSnapshot({ + content: bytes("remote binary\0") + }) + } + }, + expected: new Map([["a.bin", bytes("remote binary\0")]]), + retained: ["private binary edit\0", "unsent deleted bytes"] + }; +} + +async function permanentDestinationFixture() { + const current = await fixture(); + const next = structuredClone(current.initial); + next.local = { a: "unsupported/a.md", b: "good.md" }; + return { + ...current, + next, + permanentFault: (label: string) => { + if (label === "before:mkdir:unsupported") + throw Object.assign(new Error("Unsupported destination"), { + code: "ENAMETOOLONG" + }); + }, + writes: { + a: { + expected: await toStoredSnapshot({ content: bytes("A") }), + replacement: await toStoredSnapshot({ + content: bytes("REMOTE") + }) + } + }, + expectedLocal: { a: "Recovered a.md", b: "good.md" }, + expected: new Map([ + ["Recovered a.md", bytes("REMOTE")], + ["good.md", bytes("B")] + ]), + retained: ["A"] + }; +} + +for (const [name, create] of [ + ["swap and create", swapFixture], + ["cross-directory move with surviving sibling", crossDirectoryFixture], + ["merge during recovery", mergeFixture], + ["binary replacement, deletion and retained bytes", deletionFixture], + ["permanent destination replan", permanentDestinationFixture] +] as const) { + test(`${name}: interrupt every apply and replay boundary, then power-cycle again`, async (t) => { + const baseline = await create(); + const labels: string[] = []; + const trace = (label: string) => { + if (!label.startsWith("read:")) labels.push(label); + }; + baseline.disk.boundary = (label) => { + baseline.permanentFault(label); + trace(label); + }; + baseline.persistence.boundary = trace; + await baseline.restart().files.apply(baseline.next, baseline.writes); + assert.deepEqual(baseline.disk.userFiles(), baseline.expected); + assert(labels.some((label) => label.startsWith("visible:rename:"))); + assert(labels.includes("durable:save")); + let replayCrashes = 0; + for (const powerLoss of [false, true]) { + for (let index = 0; index < labels.length; index++) { + await t.test( + `${powerLoss ? "power" : "process"} crash #${index}: ${labels[index]}`, + async () => { + const current = await create(); + const clearFaults = () => { + current.disk.boundary = current.permanentFault; + current.persistence.boundary = () => {}; + }; + const interrupt = async ( + action: () => Promise, + at: number, + power: boolean + ) => { + let position = 0, + fired = false; + const fault = (label: string) => { + if (label.startsWith("read:")) return; + if (position++ === at) { + fired = true; + current.disk.crash(power); + throw new Error("Injected interruption"); + } + }; + current.disk.boundary = (label) => { + current.permanentFault(label); + fault(label); + }; + current.persistence.boundary = fault; + try { + await assert.rejects( + action(), + /Injected interruption/ + ); + assert(fired, "Fault index did not occur"); + } finally { + clearFaults(); + } + }; + const verify = ( + runtime: ReturnType + ) => { + assert.equal( + runtime.database.state.application, + undefined + ); + assert.deepEqual( + runtime.database.state.local, + current.expectedLocal ?? current.next.local + ); + assert.deepEqual( + current.disk.userFiles(), + current.expected, + "Exact files must survive interrupted replay" + ); + const retained = current.disk + .image() + .visible.filter( + ([path, entry]) => + entry && + path.startsWith( + ".vault-link-sync/recovery/" + ) && + path.endsWith(".content") + ) + .map(([, entry]) => + Buffer.from(entry!.content).toString() + ) + .sort(); + assert.deepEqual( + retained, + [...current.retained].sort(), + "Recovery archives lost or duplicated displaced bytes" + ); + assert( + !current.disk + .image() + .visible.some( + ([path, entry]) => + entry && + path.startsWith( + ".vault-link-sync/transactions/" + ) + ), + "Completed recovery left transaction artifacts" + ); + }; + const finish = async (resubmit: boolean) => { + let runtime = current.restart(); + await runtime.files.recover(); + if (resubmit) + await runtime.files.apply( + current.next, + current.writes + ); + await runtime.files.recover(); + verify(runtime); + current.disk.crash(true); + runtime = current.restart(); + await runtime.files.recover(); + verify(runtime); + }; + await interrupt( + () => + current + .restart() + .files.apply(current.next, current.writes), + index, + powerLoss + ); + const saved = current.persistence.snapshot(); + const diskImage = current.disk.image(); + const intentWasNotSaved = isDeepStrictEqual( + saved.database, + current.initial + ); + // Trace replay from the actual interrupted state, including + // recovery-only branches such as already-visible installs. + const replayLabels: string[] = []; + const traceReplay = (label: string) => { + if (!label.startsWith("read:")) + replayLabels.push(label); + }; + current.disk.boundary = (label) => { + current.permanentFault(label); + traceReplay(label); + }; + current.persistence.boundary = traceReplay; + await current.restart().files.recover(); + clearFaults(); + await finish(intentWasNotSaved); + for (const secondPowerLoss of [false, true]) { + for ( + let replay = 0; + replay < replayLabels.length; + replay++ + ) { + current.disk.crash(); + current.disk.restore(diskImage); + await current.persistence.save(saved); + try { + await interrupt( + () => current.restart().files.recover(), + replay, + secondPowerLoss + ); + await finish(false); + replayCrashes++; + } catch (error) { + throw new Error( + `Second ${secondPowerLoss ? "power" : "process"} crash #${replay}: ${replayLabels[replay]}`, + { cause: error } + ); + } + } + } + } + ); + } + } + assert(replayCrashes > 0, "No interrupted recovery was exercised"); + t.diagnostic(`${replayCrashes} second crashes during journal replay`); + }); +} + +test("production manifest merge obeys independently specified per-document policy", () => { + const paths = [undefined, "a.md", "b.md", "nested/c.md"]; + const map = (path: string | undefined): Record => + path === undefined + ? {} + : { "00000000-0000-4000-8000-000000000001": path }; + for (const base of paths) + for (const local of paths) + for (const remote of paths) { + assert.deepEqual( + mergeFileManifests(map(base), map(local), map(remote)), + map(pathDecision(base, local, remote)) + ); + } + const conflicts = mergeFileManifests( + {}, + { a: "folder" }, + { b: "folder/note.md" } + ); + assertManifest(conflicts); + assert.equal( + conflicts.b, + "folder/note.md", + "Canonical path must win collision" + ); + assert.equal( + Object.keys(conflicts).length, + 2, + "Collision must not discard an ID" + ); +}); + +test("exhaustive two-document manifest states preserve IDs and canonical collision priority", () => { + const ids = [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002" + ]; + const states: Record[] = []; + for (const a of [undefined, "a.md", "b.md"]) + for (const b of [undefined, "a.md", "b.md"]) { + if (a && a === b) continue; + states.push({ + ...(a ? { [ids[0]]: a } : {}), + ...(b ? { [ids[1]]: b } : {}) + }); + } + for (const base of states) + for (const local of states) + for (const remote of states) { + const expected = Object.fromEntries( + ids + .map((id) => [ + id, + pathDecision(base[id], local[id], remote[id]) + ]) + .filter(([, value]) => value !== undefined) + ); + const merged = mergeFileManifests(base, local, remote); + assertManifest(merged); + assert.deepEqual( + Object.keys(merged).sort(), + Object.keys(expected).sort(), + "Merge discarded or invented an identity" + ); + for (const id of ids) + if ( + expected[id] !== undefined && + remote[id] === expected[id] + ) + assert.equal( + merged[id], + remote[id], + "Canonical claimant lost its path" + ); + assert.deepEqual( + mergeFileManifests(remote, merged, remote), + merged, + "Manifest merge is not idempotent" + ); + } +}); + +test("scan failures cannot masquerade as deletions", async () => { + for (const point of ["read:list:", "read:snapshot:a.md"]) { + const current = await fixture(); + const runtime = current.restart(); + const original = structuredClone(runtime.database.state); + let fired = false; + current.disk.boundary = (label) => { + if (label === point) { + fired = true; + throw new Error("permission denied"); + } + }; + await assert.rejects( + scanLocalFiles({ + next: structuredClone(original), + changes: [], + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }), + /permission denied/ + ); + assert(fired); + assert.deepEqual(runtime.database.state, original); + assert.deepEqual(current.persistence.snapshot().database, original); + } +}); + +for (const phase of ["before", "durable"]) { + test(`delete/recreate identity survives ${phase}-save failure and retry`, async () => { + const current = await fixture(); + await current.disk.userDelete("a.md"); + await current.disk.userWrite("a.md", bytes("new incarnation")); + const changes: LocalChange[] = [ + { type: "delete", path: "a.md" }, + { type: "create", path: "a.md" } + ]; + const scan = async () => { + const runtime = current.restart(); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + return runtime.database.state; + }; + current.persistence.boundary = (label) => { + if (label === `${phase}:save`) throw new Error("save failure"); + }; + await assert.rejects(scan(), /save failure/); + current.persistence.boundary = () => {}; + const state = await scan(); + assert.equal( + state.local.a, + undefined, + "Retry reused the deleted document's UUID" + ); + assert.equal( + Object.values(state.local).filter((path) => path === "a.md").length, + 1 + ); + }); +} + +async function scanChanges( + runtime: { database: Database; files: FileOperations }, + changes: LocalChange[] +): Promise { + await runtime.database.recoverPersistence(); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); +} + +for (const editedContent of ["new bytes", "B"]) { + test(`explicit rename then edit to ${JSON.stringify(editedContent)} preserves both UUIDs`, async () => { + const current = await fixture(); + const runtime = current.restart(); + await current.disk.userRename("a.md", "moved.md"); + await current.disk.userWrite("moved.md", bytes(editedContent)); + const changes: LocalChange[] = [ + { type: "move", oldPath: "a.md", relativePath: "moved.md" } + ]; + await scanChanges(runtime, changes); + assert.deepEqual( + runtime.database.state.local, + { a: "moved.md", b: "b.md" }, + "Matching another live document's bytes must not steal its UUID" + ); + assert.deepEqual( + current.disk.userFiles(), + new Map([ + ["moved.md", bytes(editedContent)], + ["b.md", bytes("B")] + ]) + ); + assert.equal(changes.length, 0); + }); +} + +for (const equalContent of [false, true]) { + for (const phase of ["before", "durable"]) { + test(`${equalContent ? "equal" : "different"}-content swap retains UUIDs after ${phase}-save rejection`, async () => { + const current = await fixture({ + a: { path: "a.md", text: "A" }, + b: { path: "b.md", text: equalContent ? "A" : "B" } + }); + const runtime = current.restart(); + const changes: LocalChange[] = []; + for (const [oldPath, relativePath] of [ + ["a.md", "temporary.md"], + ["b.md", "a.md"], + ["temporary.md", "b.md"] + ]) { + await current.disk.userRename(oldPath, relativePath); + changes.push({ type: "move", oldPath, relativePath }); + } + const expected = { a: "b.md", b: "a.md" }; + const expectedFiles = new Map([ + ["a.md", bytes(equalContent ? "A" : "B")], + ["b.md", bytes("A")] + ]); + let fired = false; + current.persistence.boundary = (label) => { + if (!fired && label === `${phase}:save`) { + fired = true; + throw new Error("uncertain save"); + } + }; + await assert.rejects( + scanChanges(runtime, changes), + /uncertain save/ + ); + assert(fired, "The requested save failure must be exercised"); + assert.deepEqual( + current.persistence.snapshot().database!.local, + phase === "durable" ? expected : current.initial.local, + "The fault must occur on the intended side of the durable save" + ); + current.persistence.boundary = () => {}; + // Model an uncertain save in a live client: durable state reloads, + // but the original editor notifications remain in memory. Do not + // clear, reconstruct or deduplicate that queue in the harness. + await scanChanges(runtime, changes); + assert.deepEqual( + runtime.database.state.local, + expected, + "Retry replayed an already committed swap and reversed its identities" + ); + assert.equal(changes.length, 0); + await scanChanges(runtime, changes); + assert.deepEqual(runtime.database.state.local, expected); + const restarted = current.restart(); + await scanChanges(restarted, []); + assert.deepEqual(restarted.database.state.local, expected); + assert.deepEqual(current.disk.userFiles(), expectedFiles); + }); + } +} + +test("normalization-sensitive disk does not lose a distinct decomposed filename", async () => { + const current = await fixture(); + await current.disk.userWrite("é.md", bytes("composed")); + const runtime = current.restart(); + const next = structuredClone(runtime.database.state); + next.local.composed = "é.md"; + next.documents.composed = { materialized: true }; + await runtime.database.commit(next); + await current.disk.userWrite("e\u0301.md", bytes("decomposed")); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes: [], + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + assert.equal( + Object.keys(runtime.database.state.local).length, + 4, + "Distinct Unicode file vanished from the scan" + ); +}); + +test("notified editor move during a remote swap keeps the moved document's UUID", async () => { + const current = await fixture(); + const changes: LocalChange[] = []; + const runtime = current.restart(changes); + let fired = false; + current.persistence.boundary = async (label) => { + if ( + !fired && + label === "durable:save" && + current.persistence.snapshot().database?.application + ) { + fired = true; + await current.disk.userRename("b.md", "editor.md"); + changes.push({ + type: "move", + oldPath: "b.md", + relativePath: "editor.md" + }); + } + }; + const next = structuredClone(runtime.database.state); + next.local = { a: "b.md", b: "a.md" }; + await runtime.files.apply(next); + assert(fired); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + assert.equal( + runtime.database.state.local.b, + "editor.md", + "Editor move was attributed to the wrong document" + ); + assert.equal( + Buffer.from( + current.disk.userFiles().get(runtime.database.state.local.b)! + ).toString(), + "B" + ); +}); diff --git a/frontend/test-support/server-restore.test.ts b/frontend/test-support/server-restore.test.ts new file mode 100644 index 00000000..d8598429 --- /dev/null +++ b/frontend/test-support/server-restore.test.ts @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +import { readdir, copyFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +for (const mode of [ + "same-version", + "behind", + "dirty", + "pending", + "legacy-state", + ...["before:save", "durable:save"].flatMap((boundary) => + [1, 2, 3, 4, 5].map((ordinal) => `${boundary}:${ordinal}`) + ) +] as const) + test(`restored history recovers ${mode} without false convergence or lost local work`, async () => { + const f = await fixture(); + const crashRecovery = mode.includes(":save:"); + const hasLocalEdit = + mode === "dirty" || mode === "pending" || crashRecovery; + try { + const id = randomUUID(); + const h = await f.put("/documents/" + id, f.content("BASE")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + const vaults = join(f.server.databaseDirectory, "vaults"); + const db = join( + vaults, + (await readdir(vaults)).find((p) => p.endsWith(".sqlite"))! + ); + const backup = db + ".backup"; + execFileSync("python3", [ + "-c", + "import sqlite3,sys; s=sqlite3.connect(sys.argv[1]); d=sqlite3.connect(sys.argv[2]); s.backup(d); d.close(); s.close()", + db, + backup + ]); + const old = await f.put( + "/documents/" + id, + f.content("OLD BRANCH", h.vaultUpdateId) + ); + await f.client.setSetting("isSyncEnabled", true); + await f.client.setSetting("isSyncEnabled", false); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(), + "OLD BRANCH" + ); + if (hasLocalEdit) { + await f.disk.userWrite("a.md", bytes("OLD BRANCH LOCAL")); + await f.client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + if (mode === "pending") { + // Lose an acknowledgement after the server committed, leaving a + // durable request that must not be retried against the restored DB. + f.setFetchHook(async (input, init) => { + const response = await fetch(input, init); + if (init?.method === "PUT") + throw new Error("lost acknowledgement"); + return response; + }); + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /lost acknowledgement/ + ); + await f.client.setSetting("isSyncEnabled", false); + f.setFetchHook(fetch); + } + await f.server.crash(); + await copyFile(backup, db); + await rm(db + "-wal", { force: true }); + await rm(db + "-shm", { force: true }); + await f.server.restart(); + const fork = + mode === "behind" + ? h + : await f.put( + "/documents/" + id, + f.content("NEW BRANCH", h.vaultUpdateId) + ); + if (mode !== "behind") + assert.equal(fork.vaultUpdateId, old.vaultUpdateId); + let interrupted = false, + mismatch = false; + if (crashRecovery) { + const boundary = mode.split(":").slice(0, 2).join(":"); + const ordinal = Number(mode.split(":")[2]); + let saves = 0; + f.setFetchHook(async (input, init) => { + const response = await fetch(input, init); + if ( + response.headers.get( + "x-vault-link-history-mismatch" + ) === "1" + ) + mismatch = true; + return response; + }); + f.store.boundary = (label) => { + if ( + mismatch && + !interrupted && + label === boundary && + ++saves === ordinal + ) { + // Sweep the scan, durable reset intent, checkpoint + // replacement and final bootstrap-enabling saves. + interrupted = true; + throw new Error("interrupted recovery save"); + } + }; + await assert.rejects( + f.client.setSetting("isSyncEnabled", true) + ); + assert(interrupted && mismatch); + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } + if (mode === "legacy-state") { + await f.client.destroy(); + const stored = f.store.snapshot() as ReturnType< + typeof f.store.snapshot + > & { historyCheckpoint?: unknown }; + delete stored.historyCheckpoint; + await f.store.save(stored); + await f.reopenClient(); + await f.client.start(); + } + await f.client.setSetting("isSyncEnabled", true); + const actual = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + const remote = Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(); + + assert.equal(actual, remote); + assert.equal(remote, mode === "behind" ? "BASE" : "NEW BRANCH"); + if (hasLocalEdit) { + const manifest = await f.get("/file-manifest"); + const recovered = Object.keys(manifest.entries).filter( + (other) => other !== id + ); + assert.equal(recovered.length, 1); + assert.equal( + Buffer.from( + (await f.get("/documents/" + recovered[0])) + .contentBase64, + "base64" + ).toString(), + "OLD BRANCH LOCAL" + ); + assert.equal( + Buffer.from( + f.disk.userFiles().get(manifest.entries[recovered[0]])! + ).toString(), + "OLD BRANCH LOCAL" + ); + } + await f.client.reset(); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(), + remote + ); + } finally { + await f.dispose(); + } + }); diff --git a/frontend/test-support/storage.ts b/frontend/test-support/storage.ts new file mode 100644 index 00000000..816dc122 --- /dev/null +++ b/frontend/test-support/storage.ts @@ -0,0 +1,320 @@ +import assert from "node:assert/strict"; +import type { + FileSnapshot, + FileSystemOperations, + StoredDatabase, + SyncSettings +} from "sync-client"; + +export type StoredClient = Partial<{ + settings: Partial; + database: Partial; +}>; +export type Boundary = (label: string) => void | Promise; +type Entry = FileSnapshot | null; // null is a directory; absence is not a directory. +export interface DiskImage { + visible: [string, Entry][]; + durable: [string, Entry][]; +} +const parent = (path: string) => path.split("/").slice(0, -1).join("/"); +export const isInternal = (path: string) => + path.normalize("NFC").toUpperCase().split("/")[0] === ".VAULT-LINK-SYNC"; + +/** The adapter and the editor are deliberately separate APIs. Engine writes do + * not generate editor notifications. All returned snapshots are owned copies. + * + * This models atomic namespace operations, not a particular filesystem's + * journal. At a visible boundary a power loss may roll back the operation; + * a process death alone does not roll back the kernel's dirty state. + */ +export class MemoryDisk implements FileSystemOperations { + private visible = new Map([["", null]]); + private durable = new Map([["", null]]); + private generation = 0; + public boundary: Boundary = () => {}; + + public image(): DiskImage { + return structuredClone({ + visible: [...this.visible], + durable: [...this.durable] + }); + } + + public restore(image: DiskImage): void { + this.visible = new Map(structuredClone(image.visible)); + this.durable = new Map(structuredClone(image.durable)); + } + + public crash(powerLoss = false): void { + this.generation++; + if (powerLoss) this.visible = structuredClone(this.durable); + } + + /** Fence an abandoned runtime, including calls it makes after a restart. */ + public session(): FileSystemOperations { + const generation = this.generation; + return new Proxy(this, { + get: (target, key) => { + const value = Reflect.get(target, key); + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + assert.equal( + this.generation, + generation, + "Access from crashed filesystem session" + ); + return value.apply(target, args); + }; + } + }); + } + + private check(path: string): void { + assert( + !path.startsWith("/") && + !path.includes("\\") && + !path.includes("\0"), + `Unsafe path: ${path}` + ); + assert( + path === "" || + path + .split("/") + .every( + (part) => part !== "" && part !== "." && part !== ".." + ), + `Unsafe path: ${path}` + ); + for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor)) + assert( + !this.visible.has(ancestor) || + this.visible.get(ancestor) === null, + `File ancestor: ${ancestor}` + ); + } + + private async change( + label: string, + paths: string[], + mutate: () => void + ): Promise { + const generation = this.generation; + await this.boundary(`before:${label}`); + assert.equal( + this.generation, + generation, + "Mutation resumed after crash" + ); + mutate(); + await this.boundary(`visible:${label}`); + assert.equal(this.generation, generation, "Flush resumed after crash"); + for (const path of paths) { + if (this.visible.has(path)) + this.durable.set( + path, + structuredClone(this.visible.get(path)!) + ); + else this.durable.delete(path); + } + await this.boundary(`durable:${label}`); + assert.equal( + this.generation, + generation, + "Operation returned after crash" + ); + } + + public async stat(path: string) { + this.check(path); + await this.boundary(`read:stat:${path}`); + const entry = this.visible.get(path); + return entry === undefined + ? undefined + : entry === null + ? { kind: "directory" as const, size: 0 } + : { kind: "file" as const, size: entry.content.length }; + } + + public async exists(path: string): Promise { + return (await this.stat(path)) !== undefined; + } + + public async readSnapshot(path: string): Promise { + this.check(path); + await this.boundary(`read:snapshot:${path}`); + const entry = this.visible.get(path); + assert(entry !== null, `Cannot read directory: ${path}`); + return structuredClone(entry); + } + + public async listFilesRecursively(root = ""): Promise { + this.check(root); + await this.boundary(`read:list:${root}`); + assert.equal( + this.visible.get(root), + null, + `Cannot list non-directory: ${root}` + ); + return [...this.visible] + .filter( + ([p, entry]) => + entry !== null && (!root || p.startsWith(`${root}/`)) + ) + .map(([p]) => p) + .sort(); + } + + public userFiles(): Map { + return new Map( + [...this.visible] + .filter(([p, e]) => e !== null && !isInternal(p)) + .map(([p, e]) => [p, e!.content.slice()]) + ); + } + + public async createDirectory(path: string): Promise { + this.check(path); + const paths: string[] = []; + for (let p = path; p; p = parent(p)) paths.unshift(p); + await this.change(`mkdir:${path}`, paths, () => { + for (const p of paths) + assert( + !this.visible.has(p) || this.visible.get(p) === null, + `File blocks directory: ${p}` + ); + for (const p of paths) this.visible.set(p, null); + }); + } + + public async write(path: string, snapshot: FileSnapshot): Promise { + this.check(path); + await this.change(`write:${path}`, [path], () => { + assert(!this.visible.has(path), `Destination exists: ${path}`); + assert.equal( + this.visible.get(parent(path)), + null, + `Parent missing: ${path}` + ); + this.visible.set(path, structuredClone(snapshot)); + }); + } + + public async rename(from: string, to: string): Promise { + this.check(from); + this.check(to); + await this.change(`rename:${from}->${to}`, [from, to], () => { + const source = this.visible.get(from); + assert(source, `Source is not a regular file: ${from}`); + assert(!this.visible.has(to), `Destination exists: ${to}`); + assert.equal( + this.visible.get(parent(to)), + null, + `Parent missing: ${to}` + ); + this.visible.set(to, source); + this.visible.delete(from); + }); + } + + public async flushPaths(paths: readonly string[]): Promise { + const affected = new Set(paths); + for (const path of paths) + for (let p = parent(path); p; p = parent(p)) affected.add(p); + await this.change(`flush:${paths.join(",")}`, [...affected], () => {}); + } + + public async delete(path: string): Promise { + this.check(path); + assert(path !== "", "Cannot remove vault root"); + const visiblePaths = [...this.visible.keys()].filter( + (p) => p === path || p.startsWith(`${path}/`) + ); + // A durable rmdir removes access to the whole subtree. A child whose + // unlink was visible but unflushed cannot reappear beneath a durably + // removed directory when this flat namespace is restored. + const paths = [ + ...new Set([ + ...visiblePaths, + ...[...this.durable.keys()].filter( + (p) => p === path || p.startsWith(`${path}/`) + ) + ]) + ]; + await this.change(`rmdir:${path}`, paths, () => { + for (const p of visiblePaths) + assert.equal( + this.visible.get(p), + null, + `Cannot unlink regular file: ${p}` + ); + for (const p of visiblePaths) this.visible.delete(p); + }); + } + + public async deleteFile(path: string): Promise { + this.check(path); + await this.change(`unlink:${path}`, [path], () => { + const entry = this.visible.get(path); + assert(entry !== null, `Cannot unlink directory: ${path}`); + this.visible.delete(path); + }); + } + + /** External editor mutations are durable by default and may replace files. */ + public async userWrite(path: string, content: Uint8Array): Promise { + this.check(path); + await this.createDirectory(parent(path)); + assert( + this.visible.get(path) !== null, + `Cannot overwrite directory: ${path}` + ); + this.visible.set(path, { content: content.slice() }); + this.durable.set(path, { content: content.slice() }); + } + + public async userDelete(path: string): Promise { + this.check(path); + assert(this.visible.get(path), `Cannot delete missing file: ${path}`); + this.visible.delete(path); + this.durable.delete(path); + } + + public async userRename(from: string, to: string): Promise { + this.check(from); + this.check(to); + await this.createDirectory(parent(to)); + const source = this.visible.get(from); + assert(source, `Source is not a regular file: ${from}`); + assert( + this.visible.get(to) !== null, + `Cannot overwrite directory: ${to}` + ); + // Editors may explicitly replace a destination. This is NOT the + // engine's rename API, which must never overwrite it. + this.visible.delete(from); + this.durable.delete(from); + this.visible.set(to, structuredClone(source)); + this.durable.set(to, structuredClone(source)); + } +} + +/** Atomic replacement with both definite failure and uncertain success. */ +export class MemoryPersistence { + private durable: StoredClient; + public boundary: Boundary = () => {}; + public constructor(initial: StoredClient = {}) { + this.durable = structuredClone(initial); + } + public async load(): Promise { + return structuredClone(this.durable); + } + public snapshot(): StoredClient { + return structuredClone(this.durable); + } + public async save(next: StoredClient): Promise { + const owned = structuredClone(next); + await this.boundary("before:save"); + this.durable = owned; + await this.boundary("durable:save"); + } +} diff --git a/frontend/test-support/sync-audit.test.ts b/frontend/test-support/sync-audit.test.ts new file mode 100644 index 00000000..ae842a83 --- /dev/null +++ b/frontend/test-support/sync-audit.test.ts @@ -0,0 +1,900 @@ +import assert from "node:assert/strict"; +import { PermanentSyncError } from "../sync-client/src/errors/errors"; +import { test } from "node:test"; +import type { Database } from "../sync-client/src/persistence/database"; +import { emptyState } from "../sync-client/src/persistence/database"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +import { fixture, bytes, head } from "./sync-fixture"; + +test("ignored rename must not produce a duplicate submitted path", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + for (const [id, path] of Object.entries(f.database.state.local)) { + f.database.state.documents[id].base = { + ...head(id, 1, id.toUpperCase()), + hash: f.database.state.documents[id].observedHash! + }; + } + await f.disk.userRename("a.md", "ignored.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "ignored.md" + }); + await f.disk.userRename("b.md", "a.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "a.md" + }); + await f.settings.setSettings({ ignorePatterns: ["ignored.md"] }); + await f.internals.scan(); + f.internals.stopped = false; + await f.internals.preparePush(); + const p = f.database.state.pending; + assert(p?.type === "fileManifest"); + assert.equal( + new Set(Object.values(p.request.entries)).size, + Object.keys(p.request.entries).length + ); +}); + +test("ignored untracked ancestor must remain in place", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("remote") } + ); + await f.disk.userWrite("private", bytes("LOCAL PRIVATE")); + await f.settings.setSettings({ ignorePatterns: ["private"] }); + f.database.state.remoteHeads.a = head("a", 2, "remote"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "private/note.md" } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + assert.deepEqual(await f.disk.readSnapshot("private"), { + content: bytes("LOCAL PRIVATE") + }); +}); + +test("oversized untracked file must remain in place", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("R") } + ); + await f.disk.userWrite("a.md", bytes("LOCAL OVERSIZED")); + await f.settings.setSettings({ maxFileSizeMB: 5 / (1024 * 1024) }); + f.database.state.remoteHeads.a = head("a", 2, "R"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "a.md" } + }); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("LOCAL OVERSIZED") + ); +}); + +test("rejected unknown success must use submitted content as merge base", async () => { + const f = await fixture( + { "a.md": "base" }, + { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes( + vaultUpdateId === 1 + ? "base" + : vaultUpdateId === 2 + ? "base A" + : "base A REMOTE" + ) + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("base A")); + f.internals.stopped = false; + await f.internals.preparePush(); + const req = f.database.state.pending!.request.requestId; + await f.internals.finishPending(); + await f.disk.userWrite("a.md", bytes("base")); + await f.internals.incorporateEventBatch({ + headEventId: 3, + events: [ + { + eventId: 2, + requestId: req, + type: "content", + document: head("a", 2, "base A") + }, + { + eventId: 3, + requestId: "other", + type: "content", + document: head("a", 3, "base A REMOTE") + } + ] + }); + const result = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(result, "base REMOTE"); +}); + +test("bootstrap adopts same-path identity even after multiple notifications", async () => { + const f = await fixture({}); + f.database.state.initialized = false; + f.database.state.fileManifest = { fileManifestId: 0, entries: {} }; + await f.disk.userWrite("a.md", bytes("local a")); + await f.syncer.syncLocallyCreatedFile("a.md"); + await f.disk.userWrite("b.md", bytes("local b")); + await f.syncer.syncLocallyCreatedFile("b.md"); + const initial = { + headEventId: 2, + fileManifest: { fileManifestId: 2, entries: { remoteA: "a.md" } }, + documents: [head("remoteA", 1, "remote a")] + }; + await f.internals.scan(initial); + assert.equal(f.database.state.local.remoteA, "a.md"); +}); + +test("oversized remote head must not cause identical stale uploads forever", async () => { + let attempts = 0; + const remoteText = "R".repeat(20); + const f = await fixture( + { "a.md": "edit" }, + { + putFileContent: async () => { + attempts++; + return { + type: "StaleBase", + ...head("a", 2, remoteText), + contentBase64: Buffer.from(remoteText).toString("base64") + }; + } + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 2, remoteText); + await f.settings.setSettings({ maxFileSizeMB: 10 / (1024 * 1024) }); + f.internals.stopped = false; + // Even a small local edit must wait until its required remote base is readable. + assert.equal(await f.internals.preparePush(), false); + assert.equal(attempts, 0); + assert.equal(f.database.state.pending, undefined); +}); + +test("local undo of accepted-but-rejected manifest must survive replay", async () => { + const f = await fixture( + { "a.md": "A" }, + { + pushFileManifest: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async () => bytes("A") + } + ); + f.database.state.lastSeenUpdateId = 1; + f.database.state.documents.a.base = { + ...head("a", 1, "A"), + hash: f.database.state.documents.a.observedHash! + }; + await f.disk.userRename("a.md", "b.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "b.md" + }); + await f.internals.scan(); + f.internals.stopped = false; + await f.internals.preparePush(); + const req = f.database.state.pending!.request.requestId; + f.internals.stopped = true; + await f.internals.finishPending(); + await f.disk.userRename("b.md", "a.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "a.md" + }); + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId: req, + type: "fileManifest", + fileManifest: { fileManifestId: 2, entries: { a: "b.md" } } + } + ] + }); + assert.equal(f.database.state.local.a, "a.md"); +}); + +test("disabled startup should recover an active journal before returning", async () => { + const disk = new MemoryDisk(); + const key = JSON.stringify(["http://offline.test", "test"]); + const state = emptyState(key); + state.initialized = true; + state.local = { a: "a.md" }; + state.documents = { a: { materialized: true } }; + const next = structuredClone(state); + next.local = { a: "b.md" }; + const prefix = ".vault-link-sync/transactions/active/a"; + await disk.userWrite(prefix + ".source", bytes("ONLY COPY")); + state.application = { + id: "active", + extensions: ["md"], + next, + steps: [ + { + documentId: "a", + from: "a.md", + to: "b.md", + staged: prefix + ".source", + output: prefix + ".output", + phase: "staged" + } + ] + }; + const store = new MemoryPersistence({ + settings: { + remoteUri: "http://offline.test", + vaultName: "test", + isSyncEnabled: false + }, + database: state + }); + const client = await SyncClient.create({ fs: disk, persistence: store }); + try { + await client.start(); + assert.deepEqual( + (await disk.readSnapshot("b.md"))?.content, + bytes("ONLY COPY") + ); + } finally { + await client.destroy(); + } +}); + +test("a settings update resumes the enabled engine", async () => { + const f = await settingsLifecycleFixture(); + try { + await f.client.setSettings({ maxFileSizeMB: 20 }); + assert.equal(f.client.getSettings().maxFileSizeMB, 20); + assert.equal(f.store.snapshot().settings?.maxFileSizeMB, 20); + const before = f.reads(); + await f.client.syncLocallyUpdatedFile({ relativePath: "absent.md" }); + await f.client.waitUntilFinished(); + assert( + f.reads() > before, + "notifications must still wake the enabled engine" + ); + } finally { + await f.client.destroy(); + } +}); + +test("a dirty cursor update must invalidate old precise positions", async () => { + const { CursorTracker } = await import( + "../sync-client/src/sync-operations/cursor-tracker" + ); + const { FileChangeNotifier } = await import( + "../sync-client/src/sync-operations/file-change-notifier" + ); + const f = await fixture({ "a.md": "base" }); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + const ws = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: () => {} + }; + const tracker = new CursorTracker( + f.database, + ws as unknown as WebSocketManager, + f.files, + new FileChangeNotifier() + ); + let latest: import("../sync-client/src/types/maybe-outdated-client-cursors").MaybeOutdatedClientCursors[] = + []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + latest = cursors; + }); + const clean = { + userName: "other", + deviceId: "other", + documentsWithCursors: [ + { + document_id: "a", + relative_path: "a.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 0 }] + } + ] + }; + await ws.onRemoteCursorsUpdateReceived.triggerAsync([clean]); + await ws.onRemoteCursorsUpdateReceived.triggerAsync([ + { + ...clean, + documentsWithCursors: [ + { + ...clean.documentsWithCursors[0], + vault_update_id: null, + cursors: [{ start: 100, end: 100 }] + } + ] + } + ]); + assert.equal(latest[0]?.isOutdated, true); + tracker.reset(); +}); + +test("silent open WebSocket must eventually trigger catchup or reconnect", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout", "setInterval"] }); + let opens = 0, + eventReads = 0; + class SilentWebSocket { + static OPEN = 1; + readyState = 1; + onopen: ((event: object) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = + null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: ((event: object) => void) | null = null; + constructor(_url: string | URL) { + opens++; + queueMicrotask(() => this.onopen?.({})); + } + send(_message: string) {} + close() { + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + const store = new MemoryPersistence({ + settings: { + isSyncEnabled: true, + remoteUri: "http://silent.test", + vaultName: "test", + token: "test", + syncIntervalMs: 0 + } + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: store, + webSocket: SilentWebSocket as unknown as typeof WebSocket, + fetch: async (input) => { + const path = String(input); + if (path.endsWith("/ping")) + return Response.json({ + supportedApiVersion: 5, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (path.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + if (path.includes("/events-since")) { + eventReads++; + return Response.json({ headEventId: 0, events: [] }); + } + throw new Error("Unexpected request " + path); + } + }); + try { + await client.start(); + const before = eventReads; + context.mock.timers.tick(45_000); + context.mock.timers.tick(3500); + for (let i = 0; i < 30; i++) await Promise.resolve(); + assert(opens > 1 || eventReads > before); + } finally { + await client.destroy(); + } +}); + +test("oversized files should be filtered before reading complete snapshots", async () => { + const f = await fixture({}); + await f.disk.userWrite("huge.bin", bytes("too big")); + await f.settings.setSettings({ maxFileSizeMB: 1 / (1024 * 1024) }); + let reads = 0; + f.disk.boundary = (label) => { + if (label === "read:snapshot:huge.bin") reads++; + }; + await f.internals.scan(); + assert.equal(reads, 0); +}); + +test("permanent destination failure replans the affected file and finishes the batch", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + f.disk.boundary = (label) => { + if (label === "before:mkdir:unsupported") + throw Object.assign(new Error("filesystem path limit"), { + code: "ENAMETOOLONG" + }); + }; + const next = structuredClone(f.database.state); + next.local = { a: "unsupported/a.md", b: "good.md" }; + next.fileManifest = { fileManifestId: 2, entries: { ...next.local } }; + await f.files.apply(next); + await f.settings.setSettings({ ignorePatterns: ["unsupported/**"] }); + await f.files.recover(); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((b) => Buffer.from(b).toString()) + .sort(), + ["A", "B"] + ); +}); + +test("conflict allocation terminates when a truncated extension ends in a space", async () => { + const { spawnSync } = await import("node:child_process"); + const modulePath = require.resolve( + "../sync-client/src/utils/portable-path" + ); + const result = spawnSync( + process.execPath, + [ + "--require", + require.resolve("tsx/cjs"), + "-e", + ` + const {allocatePortablePath,validatePortablePaths} = require(${JSON.stringify(modulePath)}); + const name = 'a.' + 'x'.repeat(62) + ' ' + 'x'.repeat(100); + const allocated = allocatePortablePath(name, 'new', {old:name}); + validatePortablePaths([name, allocated]); + if (Buffer.byteLength(allocated) > 255) process.exit(2); + ` + ], + { timeout: 3000, encoding: "utf8" } + ); + assert.equal(result.status, 0, String(result.error ?? result.stderr)); +}); + +test("paged catchup folds remote edits and reversions before touching local bytes", async () => { + const f = await fixture( + { "a.md": "base LOCAL" }, + { getDocumentVersionContent: async () => bytes("base") } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 1, "base"); + f.database.state.lastSeenUpdateId = 1; + const first = { + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId: "remote-1", + type: "content" as const, + document: head("a", 2, "base REMOTE") + } + ] + }; + await f.internals.incorporateEventBatch(first); + assert.equal( + f.database.state.lastSeenUpdateId, + 1, + "partial history must not become an incorporated base" + ); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); + assert.equal( + await f.internals.preparePush(), + false, + "do not submit against a partially replayed history" + ); + const last = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "remote-2", + type: "content" as const, + document: head("a", 3, "base") + } + ] + }; + await f.internals.incorporateEventBatch(last); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); +}); + +test("an ignored occupant arriving after the scan keeps its path and privacy", async () => { + let f: Awaited>; + f = await fixture( + {}, + { + getDocumentVersionContent: async () => { + await f.disk.userWrite("private", bytes("SECRET")); + return bytes("remote"); + } + } + ); + await f.settings.setSettings({ ignorePatterns: ["private"] }); + f.database.state.remoteHeads.a = head("a", 2, "remote"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "private/note.md" } + }); + assert.deepEqual( + (await f.disk.readSnapshot("private"))?.content, + bytes("SECRET") + ); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert( + pending?.type !== "content" || + Buffer.from(pending.snapshot.contentBase64, "base64").toString() !== + "SECRET" + ); +}); + +test("content notifications during every snapshot do not starve a complete namespace scan", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let notifications = 0; + f.disk.boundary = async (label) => { + if ( + label.startsWith("read:snapshot:") && + !label.includes(".vault-link-sync") + ) { + notifications++; + await f.syncer.syncLocallyUpdatedFile({ + relativePath: label.slice("read:snapshot:".length) + }); + } + }; + await f.internals.scan(); + assert.equal(notifications, 2); + assert.deepEqual(f.database.state.local, { a: "a.md", b: "b.md" }); +}); + +for (const boundary of ["before:save", "durable:save"]) { + test(`receipt incorporation after paged replay survives ${boundary} and process replacement`, async () => { + const service: Partial = { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes( + vaultUpdateId === 1 + ? "base" + : vaultUpdateId === 2 + ? "base A" + : "base A REMOTE" + ) + }; + let f = await fixture({ "a.md": "base" }, service); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("base A")); + f.internals.stopped = false; + await f.internals.preparePush(); + const { requestId } = f.database.state.pending!.request; + await f.internals.finishPending(); + await f.disk.userWrite("a.md", bytes("base")); + await f.internals.incorporateEventBatch({ + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "base A") + } + ] + }); + const page = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "other", + type: "content" as const, + document: head("a", 3, "base A REMOTE") + } + ] + }; + f.persistence.boundary = (label) => { + if (label === boundary) throw new Error("injected crash"); + }; + await assert.rejects( + f.internals.incorporateEventBatch(page), + /injected crash/ + ); + f.persistence.boundary = () => {}; + f.disk.crash(true); + f = await fixture({}, service, f); + await f.files.recover(); + assert.equal(f.database.state.eventReplay?.after, 2); + await f.internals.incorporateEventBatch(page); + assert.equal(f.database.state.eventReplay, undefined); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base REMOTE") + ); + }); +} + +test("initial vault settings bind the engine and survive restart", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence({ + settings: { + remoteUri: "http://test", + vaultName: "old", + isSyncEnabled: false + } + }); + class Socket { + static OPEN = 1; + readyState = 3; + onopen = null; + onclose = null; + onerror = null; + onmessage = null; + send() {} + close() {} + } + const options = { + fs: disk, + persistence, + webSocket: Socket as unknown as typeof WebSocket, + fetch: async (input: RequestInfo | URL) => { + if (String(input).endsWith("/ping")) + return Response.json({ + supportedApiVersion: 5, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (String(input).endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + return Response.json({ headEventId: 0, events: [] }); + } + }; + let client = await SyncClient.create(options); + try { + await client.setSettings({ vaultName: "new", isSyncEnabled: true }); + await client.start(); + await client.destroy(); + client = await SyncClient.create(options); + assert.equal(client.getSettings().vaultName, "new"); + await client.start(); + } finally { + await client.destroy(); + } +}); + +test("disabling sync persists across reset without resuming the transport", async () => { + const f = await settingsLifecycleFixture(); + try { + const before = f.reads(); + await f.client.setSettings({ isSyncEnabled: false }); + assert.equal(f.store.snapshot().settings?.isSyncEnabled, false); + await f.client.reset(); + assert.equal(f.client.getSettings().isSyncEnabled, false); + assert.equal(f.reads(), before); + } finally { + await f.client.destroy(); + } +}); + +for (const receipt of [false, true]) { + test(`${receipt ? "replayed" : "direct"} upload acknowledgement survives a local deletion`, async () => { + const f = await fixture( + { "a.md": "base" }, + { + putFileContent: async () => { + if (receipt) + throw new PermanentSyncError( + "HTTP 413 after lost success" + ); + return { type: "Accepted", ...head("a", 2, "edited") }; + } + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("edited")); + f.internals.stopped = false; + await f.internals.preparePush(); + const { requestId } = f.database.state.pending!.request; + if (receipt) await f.internals.finishPending(); + // Recovery folds queued deletions into the durable namespace before + // an outstanding request or its delayed receipt is handled. + f.internals.stopped = true; + await f.disk.userDelete("a.md"); + await f.syncer.syncLocallyDeletedFile("a.md"); + await f.internals.scan(); + assert.deepEqual(f.database.state.local, {}); + if (receipt) { + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "edited") + } + ] + }); + assert.equal(f.database.state.lastSeenUpdateId, 2); + assert.equal(f.database.state.unconfirmed?.length ?? 0, 0); + } else await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.deepEqual(f.database.state.local, {}); + assert.equal(await f.disk.readSnapshot("a.md"), undefined); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + }); +} + +async function settingsLifecycleFixture() { + let eventReads = 0; + let onClose = () => {}; + class Socket { + static OPEN = 1; + readyState = 1; + onopen: (() => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = + null; + constructor() { + queueMicrotask(() => this.onopen?.()); + } + send() {} + close() { + onClose(); + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + const store = new MemoryPersistence({ + settings: { + remoteUri: "http://test", + vaultName: "test", + isSyncEnabled: true + } + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: store, + webSocket: Socket as unknown as typeof WebSocket, + fetch: async (input) => { + const path = String(input); + if (path.endsWith("/ping")) + return Response.json({ + supportedApiVersion: 5, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (path.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + if (path.includes("/events-since")) { + eventReads++; + return Response.json({ headEventId: 0, events: [] }); + } + throw new Error("Unexpected request " + path); + } + }); + await client.start(); + return { + client, + store, + reads: () => eventReads, + onClose: (fn: () => void) => { + onClose = fn; + } + }; +} + +for (const boundary of ["before:save", "durable:save"]) { + test(`settings recovery resumes after the rejection-reset save fails at ${boundary}`, async () => { + const f = await settingsLifecycleFixture(); + try { + const { database } = f.client as unknown as { database: Database }; + await database.commit({ + ...database.state, + rejectedManifest: { + entries: { retired: "old.md" }, + message: "retired rejection" + } + }); + let saves = 0, + failed = false; + f.store.boundary = (label) => { + if (label === "before:save") saves++; + if (saves === 2 && label === boundary && !failed) { + failed = true; + throw new Error("rejection-reset save failed"); + } + }; + await assert.rejects( + f.client.setSettings({ maxFileSizeMB: 20 }), + /rejection-reset save failed/ + ); + assert(failed); + assert.equal(f.client.getSettings().maxFileSizeMB, 20); + const before = f.reads(); + await f.client.syncLocallyUpdatedFile({ + relativePath: "absent.md" + }); + await f.client.waitUntilFinished(); + assert( + f.reads() > before, + "the enabled engine must resume after any settings-phase save failure" + ); + } finally { + await f.client.destroy(); + } + }); +} + +test("a caller cannot mutate settings while transports are being paused", async () => { + const f = await settingsLifecycleFixture(); + const change = { + maxFileSizeMB: 20, + vaultName: "test", + ignorePatterns: ["private/**"] + }; + try { + f.onClose(() => { + change.vaultName = "wrong-vault"; + change.ignorePatterns.push("**"); + }); + await f.client.setSettings(change); + assert.equal(f.client.getSettings().vaultName, "test"); + assert.deepEqual(f.client.getSettings().ignorePatterns, ["private/**"]); + assert.equal( + f.store.snapshot().database?.vaultKey, + JSON.stringify(["http://test", "test"]) + ); + } finally { + await f.client.destroy(); + } +}); diff --git a/frontend/test-support/sync-fixture.ts b/frontend/test-support/sync-fixture.ts new file mode 100644 index 00000000..da54142a --- /dev/null +++ b/frontend/test-support/sync-fixture.ts @@ -0,0 +1,136 @@ +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { globsToRegexes } from "../sync-client/src/utils/globs-to-regexes"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import type { LocalChange } from "../sync-client/src/sync-operations/local-changes"; + +export const bytes = (text: string) => new TextEncoder().encode(text); +export const head = (id: string, version: number, content: string) => ({ + documentId: id, + vaultUpdateId: version, + contentSize: bytes(content).length, + updatedDate: "2026-09-19T00:00:00Z", + userId: "user", + deviceId: "device" +}); + +export async function fixture( + contents: Record = { "a.md": "local" }, + service: Partial = {}, + recovered?: { disk: MemoryDisk; persistence: MemoryPersistence } +) { + const disk = recovered?.disk ?? new MemoryDisk(); + const initial = recovered + ? (recovered.persistence.snapshot().database as StoredDatabase) + : emptyState("test"); + if (!recovered) { + initial.initialized = true; + for (const [path, content] of Object.entries(contents)) { + await disk.userWrite(path, bytes(content)); + const id = path.split(".")[0]; + initial.local[id] = path; + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot({ content: bytes(content) }) + ).hash + }; + } + initial.fileManifest = { + fileManifestId: 1, + entries: { ...initial.local } + }; + } + const persistence = + recovered?.persistence ?? new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + "test", + async () => (await persistence.load()).database as StoredDatabase + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + const changes: LocalChange[] = []; + const files = new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} }, + (path, size) => + size > settings.getSettings().maxFileSizeMB * 1024 * 1024 || + globsToRegexes(settings.getSettings().ignorePatterns, logger).some( + (pattern) => pattern.test(path) + ) + ); + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteVaultUpdateReceived: new EventListeners() + } as unknown as WebSocketManager; + const contentCache = new FixedSizeDocumentCache(1024); + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket, + files, + config, + new SyncHistory(logger), + contentCache, + { entries: changes, save: async () => {} } + ); + const internals = syncer as unknown as { + finishPending: () => Promise; + preparePush: () => Promise; + scan: ( + initial?: import("../sync-client/src/services/types/VaultSnapshot").VaultSnapshot + ) => Promise; + incorporateEventBatch: ( + batch: import("../sync-client/src/services/types/EventBatch").EventBatch + ) => Promise; + stopped: boolean; + incorporateFileManifest: (remote: { + fileManifestId: number; + entries: Record; + }) => Promise; + incorporateContent: (remote: ReturnType) => Promise; + }; + return { + disk, + database, + files, + settings, + syncer, + internals, + persistence, + changes, + contentCache + }; +} diff --git a/frontend/test-support/sync-server-fixture.ts b/frontend/test-support/sync-server-fixture.ts new file mode 100644 index 00000000..8e0d438f --- /dev/null +++ b/frontend/test-support/sync-server-fixture.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { randomUUID } from "node:crypto"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +const root = resolve(__dirname, "../.."); +const token = "test-token-change-me"; +export const bytes = (s: string) => new TextEncoder().encode(s); +const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "Device-Id": "audit" +}; +export async function fixture() { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + await server.start(); + const vault = randomUUID(), + url = server.remoteUri + "/vaults/" + vault; + const put = async (path: string, data: unknown) => { + const r = await fetch(url + path, { + method: "PUT", + headers, + body: JSON.stringify(data) + }); + assert(r.ok, await r.clone().text()); + return r.json(); + }; + const get = async (path: string) => { + const r = await fetch(url + path, { headers }); + assert(r.ok); + return r.json(); + }; + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0, + networkRetryIntervalMs: 0, + requestTimeoutMs: 5000 + } + }); + let fetchHook: typeof fetch | undefined; + const createClient = () => + SyncClient.create({ + fs: disk, + persistence: store, + fetch: (...args) => + fetchHook ? fetchHook(...args) : fetch(...args) + }); + let client = await createClient(); + const content = (text: string, parentVersionId: number | null = null) => ({ + requestId: randomUUID(), + parentVersionId, + content: { + type: "Snapshot", + value: Buffer.from(text).toString("base64") + } + }); + return { + server, + disk, + store, + get client() { + return client; + }, + reopenClient: async () => { + await client.destroy(); + client = await createClient(); + }, + url, + headers, + put, + get, + content, + setFetchHook: (hook: typeof fetch) => { + fetchHook = hook; + }, + dispose: async () => { + await client.destroy(); + await server.stop(); + } + }; +} + +export { randomUUID }; diff --git a/frontend/test-support/tsconfig.json b/frontend/test-support/tsconfig.json new file mode 100644 index 00000000..42eb5312 --- /dev/null +++ b/frontend/test-support/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "skipLibCheck": true, + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "node", + "esModuleInterop": true, + "types": ["node"], + "lib": ["DOM", "ES2024"] + }, + "include": ["*.ts"] +} diff --git a/logs/harness-review/deterministic-results.json b/logs/harness-review/deterministic-results.json new file mode 100644 index 00000000..7ac2c0b3 --- /dev/null +++ b/logs/harness-review/deterministic-results.json @@ -0,0 +1,1121 @@ +[ + { + "name": "v4-binary-bytes-and-independent-identities", + "result": { + "success": true, + "duration": 100 + } + }, + { + "name": "v4-three-file-cycle-preserves-identities", + "result": { + "success": true, + "duration": 170 + } + }, + { + "name": "v4-case-alias-and-file-directory-conflicts", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "v4-create-before-commit-network-loss", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "v4-create-after-commit-network-loss", + "result": { + "success": true, + "duration": 64 + } + }, + { + "name": "v4-content-before-commit-network-loss", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "v4-content-after-commit-network-loss", + "result": { + "success": true, + "duration": 91 + } + }, + { + "name": "v4-manifest-before-commit-network-loss", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "v4-manifest-after-commit-network-loss", + "result": { + "success": true, + "duration": 72 + } + }, + { + "name": "rename-create-conflict", + "result": { + "success": true, + "duration": 121 + } + }, + { + "name": "rename-chain", + "result": { + "success": true, + "duration": 52 + } + }, + { + "name": "rename-update-conflict", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "delete-rename-conflict", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "multi-file-operations", + "result": { + "success": true, + "duration": 129 + } + }, + { + "name": "delete-recreate-same-path", + "result": { + "success": true, + "duration": 63 + } + }, + { + "name": "offline-rename-and-edit", + "result": { + "success": true, + "duration": 79 + } + }, + { + "name": "simultaneous-create-delete-same-path", + "result": { + "success": true, + "duration": 54 + } + }, + { + "name": "sequential-create-duplicate-content", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "mc-three-client-rename-offline-update", + "result": { + "success": true, + "duration": 102 + } + }, + { + "name": "mc-multi-delete-offline-rename", + "result": { + "success": true, + "duration": 115 + } + }, + { + "name": "mc-cross-create-rename-same-target", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "mc-delete-then-offline-rename", + "result": { + "success": true, + "duration": 79 + } + }, + { + "name": "offline-mixed-operations", + "result": { + "success": true, + "duration": 117 + } + }, + { + "name": "offline-concurrent-renames", + "result": { + "success": true, + "duration": 76 + } + }, + { + "name": "offline-multiple-edits", + "result": { + "success": true, + "duration": 74 + } + }, + { + "name": "rename-swap", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "rename-circular", + "result": { + "success": true, + "duration": 113 + } + }, + { + "name": "rename-roundtrip", + "result": { + "success": true, + "duration": 95 + } + }, + { + "name": "offline-rename-remote-create-old-path", + "result": { + "success": true, + "duration": 83 + } + }, + { + "name": "offline-edit-remote-rename", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "rename-chain-then-delete", + "result": { + "success": true, + "duration": 86 + } + }, + { + "name": "offline-delete-remote-rename", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "overlapping-edits-same-section", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "rapid-updates-after-merge", + "result": { + "success": true, + "duration": 117 + } + }, + { + "name": "delete-recreate-concurrent-update", + "result": { + "success": true, + "duration": 90 + } + }, + { + "name": "move-and-concurrent-remote-update", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "double-offline-cycle", + "result": { + "success": true, + "duration": 104 + } + }, + { + "name": "offline-update-both-then-delete-one", + "result": { + "success": true, + "duration": 100 + } + }, + { + "name": "offline-create-same-path-mergeable", + "result": { + "success": true, + "duration": 68 + } + }, + { + "name": "three-client-rename-create-delete", + "result": { + "success": true, + "duration": 106 + } + }, + { + "name": "rename-to-path-of-unconfirmed-delete", + "result": { + "success": true, + "duration": 94 + } + }, + { + "name": "offline-edit-then-move-same-content", + "result": { + "success": true, + "duration": 92 + } + }, + { + "name": "delete-recreate-different-content", + "result": { + "success": true, + "duration": 102 + } + }, + { + "name": "offline-move-then-remote-delete", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "reset-clears-recently-deleted-resurrection", + "result": { + "success": true, + "duration": 75 + } + }, + { + "name": "move-then-delete-stale-path", + "result": { + "success": true, + "duration": 65 + } + }, + { + "name": "offline-delete-vs-remote-update", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "update-does-not-survive-remote-delete", + "result": { + "success": true, + "duration": 64 + } + }, + { + "name": "move-preserves-remote-update", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "recently-deleted-cleared-on-reconnect", + "result": { + "success": true, + "duration": 107 + } + }, + { + "name": "watermark-advances-on-skip", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "watermark-gap-remote-update-not-recorded", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "queue-reset-loses-coalesced-local-edit", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "rename-to-pending-path-fallback", + "result": { + "success": true, + "duration": 55 + } + }, + { + "name": "move-remote-update-reverts-rename", + "result": { + "success": true, + "duration": 99 + } + }, + { + "name": "local-edit-lost-during-create-merge", + "result": { + "success": true, + "duration": 52 + } + }, + { + "name": "create-rename-response-skips-file", + "result": { + "success": true, + "duration": 73 + } + }, + { + "name": "online-create-rename-concurrent-create-orphan", + "result": { + "success": true, + "duration": 69 + } + }, + { + "name": "concurrent-rename-first-wins", + "result": { + "success": true, + "duration": 90 + } + }, + { + "name": "binary-to-text-transition", + "result": { + "success": true, + "duration": 151 + } + }, + { + "name": "text-pending-create-not-displaced", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "binary-pending-create-not-displaced", + "result": { + "success": true, + "duration": 80 + } + }, + { + "name": "coalesce-update-remote-update-data-loss", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "coalesced-remote-update-watermark-loss", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "concurrent-delete-during-remote-update", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "concurrent-edit-exact-same-position", + "result": { + "success": true, + "duration": 97 + } + }, + { + "name": "concurrent-rename-and-create-at-target-rename-first", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "concurrent-rename-and-create-at-target-create-first", + "result": { + "success": true, + "duration": 109 + } + }, + { + "name": "concurrent-rename-same-target", + "result": { + "success": true, + "duration": 110 + } + }, + { + "name": "concurrent-update-diff-consistency", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "user-parenthesized-file-not-deleted", + "result": { + "success": true, + "duration": 116 + } + }, + { + "name": "create-delete-noop", + "result": { + "success": true, + "duration": 30 + } + }, + { + "name": "create-merge-delete", + "result": { + "success": true, + "duration": 73 + } + }, + { + "name": "move-identical-content-ambiguity", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "create-during-reconciliation", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "create-merge-preserves-renamed-update", + "result": { + "success": true, + "duration": 106 + } + }, + { + "name": "create-rename-create-same-path", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "move-chain-three-files", + "result": { + "success": true, + "duration": 116 + } + }, + { + "name": "delete-by-other-client-then-recreate", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "online-delete-recreate-rapid-cycle", + "result": { + "success": true, + "duration": 169 + } + }, + { + "name": "online-edit-vs-delete-convergence", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "rapid-edit-delete-online-convergence", + "result": { + "success": true, + "duration": 144 + } + }, + { + "name": "online-both-create-same-path-deconflict", + "result": { + "success": true, + "duration": 92 + } + }, + { + "name": "online-create-update-while-other-creates-same-path", + "result": { + "success": true, + "duration": 82 + } + }, + { + "name": "displaced-file-not-marked-deleted", + "result": { + "success": true, + "duration": 112 + } + }, + { + "name": "remote-update-resurrects-deleted-doc", + "result": { + "success": false, + "error": "Convergence timed out after 60000ms: Client 1 differs from client 0 (paths or bytes)\n+ actual - expected\n\n+ [\n+ [\n+ 'P.md',\n+ Uint8Array(17) [\n+ 118,\n+ 50,\n+ 49,\n+ 32,\n+ 99,\n+ 111,\n+ 110,\n+ 116,\n+ 101,\n+ 110,\n+ 116,\n+ 32,\n+ 40,\n+ 68,\n+ 50,\n+ 41,\n+ 10\n+ ]\n+ ]\n+ ]\n- []\n", + "diagnostics": [ + { + "client": 0, + "database": { + "vaultKey": "[\"http://localhost:62272\",\"test-df2b81ab-03e9-432a-878a-d6aa1c6c9147\"]", + "initialized": true, + "fileManifest": { + "fileManifestId": 4, + "entries": {} + }, + "local": {}, + "documents": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "materialized": true, + "base": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40, + "hash": "229b36522c531631df2139e070b94a0e1b8d8833ae49d4662cb0d7a8bd158e9b" + }, + "bootstrap": false, + "observedHash": "229b36522c531631df2139e070b94a0e1b8d8833ae49d4662cb0d7a8bd158e9b" + } + }, + "remoteHeads": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40 + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17 + } + }, + "lastSeenUpdateId": 5 + }, + "disk": { + "visible": [ + [ + "", + null + ], + [ + ".vault-link-sync", + null + ], + [ + ".vault-link-sync/transactions", + null + ] + ], + "durable": [ + [ + "", + null + ], + [ + ".vault-link-sync", + null + ], + [ + ".vault-link-sync/transactions", + null + ] + ] + }, + "requests": [ + { + "kind": "content", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "d970973e-d042-4383-9edd-6be699df0c0f", + "parentVersionId": 3, + "content": { + "type": "Diff", + "value": [ + -11, + "v17local edit by client 0", + 15 + ] + } + }, + "status": 200 + } + ] + }, + { + "client": 1, + "database": { + "vaultKey": "[\"http://localhost:62272\",\"test-df2b81ab-03e9-432a-878a-d6aa1c6c9147\"]", + "initialized": true, + "fileManifest": { + "fileManifestId": 6, + "entries": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + } + }, + "local": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + }, + "documents": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "materialized": true, + "bootstrap": false, + "observedHash": "106554e74ef5b4dc8c8668c00ff9e822ebb2573232e6af3c71849bf200685e26", + "base": { + "vaultUpdateId": 3, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.876666Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 26, + "hash": "106554e74ef5b4dc8c8668c00ff9e822ebb2573232e6af3c71849bf200685e26" + } + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "materialized": true, + "bootstrap": false, + "observedHash": "8dd4b135f5ee76639d62ae49fbfc7404962fc5a67f8f60ced508d5cb8ce5f4df", + "base": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17, + "hash": "8dd4b135f5ee76639d62ae49fbfc7404962fc5a67f8f60ced508d5cb8ce5f4df" + } + } + }, + "remoteHeads": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40 + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17 + } + }, + "lastSeenUpdateId": 7 + }, + "disk": { + "visible": [ + [ + "", + null + ], + [ + "P.md", + { + "content": { + "0": 118, + "1": 50, + "2": 49, + "3": 32, + "4": 99, + "5": 111, + "6": 110, + "7": 116, + "8": 101, + "9": 110, + "10": 116, + "11": 32, + "12": 40, + "13": 68, + "14": 50, + "15": 41, + "16": 10 + } + } + ] + ], + "durable": [ + [ + "", + null + ], + [ + "P.md", + { + "content": { + "0": 118, + "1": 50, + "2": 49, + "3": 32, + "4": 99, + "5": 111, + "6": 110, + "7": 116, + "8": 101, + "9": 110, + "10": 116, + "11": 32, + "12": 40, + "13": 68, + "14": 50, + "15": 41, + "16": 10 + } + } + ] + ] + }, + "requests": [ + { + "kind": "create", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "f2abf8a1-8c5d-49d7-8ca8-4d4df56a2aff", + "parentVersionId": null, + "content": { + "type": "Snapshot", + "value": "djggY29udGVudAo=" + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "41f31b38-1edc-4f97-8519-6973e4a04f9f", + "parentFileManifestId": 0, + "entries": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": "P.md" + } + }, + "status": 200 + }, + { + "kind": "content", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "4bd1dcf9-c5d2-42f4-aa96-f1b921362e55", + "parentVersionId": 1, + "content": { + "type": "Diff", + "value": [ + -2, + "v17", + 8, + " from client 1", + 1 + ] + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "3b0cc3a0-b288-47d0-998b-d608f885d4b0", + "parentFileManifestId": 2, + "entries": {} + }, + "status": 200 + }, + { + "kind": "create", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/c6316ddf-539d-4604-8039-5ff6022efef2", + "body": { + "requestId": "edb11740-8c0c-4407-91d2-bd264e26281f", + "parentVersionId": null, + "content": { + "type": "Snapshot", + "value": "djIxIGNvbnRlbnQgKEQyKQo=" + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "c9615094-e693-4b11-8a3d-5c415d3dcdd0", + "parentFileManifestId": 4, + "entries": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + } + }, + "status": 200 + } + ] + } + ], + "duration": 60283 + } + }, + { + "name": "local-update-survives-remote-rename", + "result": { + "success": true, + "duration": 58 + } + }, + { + "name": "catchup-create-and-update-not-skipped", + "result": { + "success": true, + "duration": 55 + } + }, + { + "name": "remote-rename-collides-with-pending-local-create", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "remote-quick-write-rename-before-record", + "result": { + "success": true, + "duration": 42 + } + }, + { + "name": "v4-server-sigkill-retains-acknowledged-state", + "result": { + "success": true, + "duration": 220 + } + }, + { + "name": "idempotency-after-server-pause", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "server-pause-both-clients-create", + "result": { + "success": true, + "duration": 60 + } + }, + { + "name": "server-pause-update-and-create", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "server-pause-rename-edit-resume", + "result": { + "success": true, + "duration": 70 + } + }, + { + "name": "delete-during-pending-create", + "result": { + "success": true, + "duration": 43 + } + }, + { + "name": "rapid-create-update-delete-cycle", + "result": { + "success": true, + "duration": 51 + } + }, + { + "name": "server-pause-both-edit-same-file", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "update-during-create-processing", + "result": { + "success": true, + "duration": 65 + } + }, + { + "name": "interrupted-delete-retry", + "result": { + "success": true, + "duration": 67 + } + }, + { + "name": "rename-pending-create-before-response", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "create-update-coalesce-server-pause", + "result": { + "success": true, + "duration": 67 + } + }, + { + "name": "server-pause-delete-recreate", + "result": { + "success": true, + "duration": 71 + } + }, + { + "name": "merging-update-response-survives-user-rename", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "local-rename-survives-remote-rename", + "result": { + "success": true, + "duration": 82 + } + }, + { + "name": "rename-chain-during-pending-create", + "result": { + "success": true, + "duration": 57 + } + }, + { + "name": "remote-update-survives-user-rename", + "result": { + "success": true, + "duration": 111 + } + }, + { + "name": "same-doc-id-collapse-on-local-create-after-remote-create", + "result": { + "success": true, + "duration": 54 + } + }, + { + "name": "renamed-pending-create-reused-path-then-delete", + "result": { + "success": true, + "duration": 185 + } + }, + { + "name": "rename-pending-create-onto-pending-delete-path", + "result": { + "success": true, + "duration": 166 + } + }, + { + "name": "rename-overwrites-pending-create-then-delete", + "result": { + "success": true, + "duration": 80 + } + }, + { + "name": "same-doc-id-collapse-after-remote-quick-write-and-pending-rename", + "result": { + "success": true, + "duration": 220 + } + }, + { + "name": "delete-recreated-pending-create-with-stale-deleting-record", + "result": { + "success": true, + "duration": 234 + } + }, + { + "name": "queued-create-delete-does-not-hijack-reused-path", + "result": { + "success": true, + "duration": 173 + } + }, + { + "name": "self-merge-pending-rename-aliases-second-create", + "result": { + "success": true, + "duration": 186 + } + } +] \ No newline at end of file diff --git a/logs/harness-review/result-42.json b/logs/harness-review/result-42.json new file mode 100644 index 00000000..fa5492ce --- /dev/null +++ b/logs/harness-review/result-42.json @@ -0,0 +1,4 @@ +{ + "success": true, + "duration": 262 +} \ No newline at end of file diff --git a/logs/harness-review/result-43.json b/logs/harness-review/result-43.json new file mode 100644 index 00000000..cbf0df26 --- /dev/null +++ b/logs/harness-review/result-43.json @@ -0,0 +1,4 @@ +{ + "success": true, + "duration": 273 +} \ No newline at end of file diff --git a/logs/harness-review/results.json b/logs/harness-review/results.json new file mode 100644 index 00000000..b14cc528 --- /dev/null +++ b/logs/harness-review/results.json @@ -0,0 +1,111 @@ +{ + "options": { + "workers": 2, + "seed": 42, + "iterations": 3, + "timeout": 180000 + }, + "results": [ + { + "name": "build-server", + "command": "cargo", + "args": [ + "build", + "--release" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "build-clients", + "command": "npm", + "args": [ + "run", + "build", + "--workspace", + "sync-client", + "--workspace", + "deterministic-tests", + "--workspace", + "test-client" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "harness-self-tests", + "command": "npm", + "args": [ + "run", + "test:harness" + ], + "code": 1, + "signal": null, + "timedOut": false, + "success": false + }, + { + "name": "crash-and-protocol", + "command": "npm", + "args": [ + "run", + "test:protocol" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "deterministic", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/deterministic-tests/dist/cli.js", + "--concurrency", + "2" + ], + "code": 1, + "signal": null, + "timedOut": false, + "success": false + }, + { + "name": "fuzz-42", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/test-client/dist/cli.js", + "--seed", + "42", + "--iterations", + "3", + "--artifacts", + "/Users/schmelczer/Projects/vault-link/logs/harness-review" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "fuzz-43", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/test-client/dist/cli.js", + "--seed", + "43", + "--iterations", + "3", + "--artifacts", + "/Users/schmelczer/Projects/vault-link/logs/harness-review" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + } + ] +} \ No newline at end of file diff --git a/logs/harness-review/trace-42.json b/logs/harness-review/trace-42.json new file mode 100644 index 00000000..c27de220 --- /dev/null +++ b/logs/harness-review/trace-42.json @@ -0,0 +1,368 @@ +{ + "version": 1, + "seed": 42, + "iterations": 3, + "steps": [ + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-0/café/a.md", + "content": "[seed-42-round-0-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-0/café/b.md", + "content": "[seed-42-round-0-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-0/café/a.md", + "key": "round-0/café/a.md" + }, + { + "type": "remember-identity", + "path": "round-0/café/b.md", + "key": "round-0/café/b.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/a.md", + "newPath": "round-0/café/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/c.md", + "newPath": "round-0/café/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/a.md", + "newPath": "round-0/café/c.md" + }, + { + "type": "update", + "client": 0, + "path": "round-0/café/c.md", + "content": "[seed-42-round-0-A]\nrapid final edit" + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-0/café/c.md", + "key": "round-0/café/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-0/café/c.md": "[seed-42-round-0-A]\nrapid final edit", + "round-0/café/b.md": "[seed-42-round-0-B]" + }, + "absent": [ + "round-0/café/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-1/日本語/a.md", + "content": "[seed-42-round-1-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-1/日本語/b.md", + "content": "[seed-42-round-1-B]" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-1/日本語/a.md", + "key": "round-1/日本語/a.md" + }, + { + "type": "remember-identity", + "path": "round-1/日本語/b.md", + "key": "round-1/日本語/b.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/a.md", + "newPath": "round-1/日本語/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/c.md", + "newPath": "round-1/日本語/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/a.md", + "newPath": "round-1/日本語/c.md" + }, + { + "type": "update", + "client": 0, + "path": "round-1/日本語/c.md", + "content": "[seed-42-round-1-A]\nrapid final edit" + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-1/日本語/c.md", + "key": "round-1/日本語/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-1/日本語/c.md": "[seed-42-round-1-A]\nrapid final edit", + "round-1/日本語/b.md": "[seed-42-round-1-B]" + }, + "absent": [ + "round-1/日本語/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]", + "[seed-42-round-1-A]", + "[seed-42-round-1-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-2/café/a.md", + "content": "[seed-42-round-2-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-2/café/b.md", + "content": "[seed-42-round-2-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-2/café/a.md", + "key": "round-2/café/a.md" + }, + { + "type": "remember-identity", + "path": "round-2/café/b.md", + "key": "round-2/café/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/a.md", + "newPath": "round-2/café/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/b.md", + "newPath": "round-2/café/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/c.md", + "newPath": "round-2/café/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-2/café/b.md", + "content": "[seed-42-round-2-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-2/café/b.md", + "key": "round-2/café/a.md" + }, + { + "type": "assert-identity", + "path": "round-2/café/a.md", + "key": "round-2/café/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-2/café/a.md": "[seed-42-round-2-B]", + "round-2/café/b.md": "[seed-42-round-2-A]\nextra line\n" + }, + "absent": [ + "round-2/café/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]", + "[seed-42-round-1-A]", + "[seed-42-round-1-B]", + "[seed-42-round-2-A]", + "[seed-42-round-2-B]" + ], + "removed": [] + } + ] +} \ No newline at end of file diff --git a/logs/harness-review/trace-43.json b/logs/harness-review/trace-43.json new file mode 100644 index 00000000..4d764671 --- /dev/null +++ b/logs/harness-review/trace-43.json @@ -0,0 +1,404 @@ +{ + "version": 1, + "seed": 43, + "iterations": 3, + "steps": [ + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-0/space folder/a.md", + "content": "[seed-43-round-0-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-0/space folder/b.md", + "content": "[seed-43-round-0-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "drop-response", + "client": 0, + "kind": "create", + "point": "after" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-0/space folder/a.md", + "key": "round-0/space folder/a.md" + }, + { + "type": "remember-identity", + "path": "round-0/space folder/b.md", + "key": "round-0/space folder/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/space folder/a.md", + "newPath": "round-0/space folder/c.md" + }, + { + "type": "update", + "client": 1, + "path": "round-0/space folder/a.md", + "content": "[seed-43-round-0-A]\nremote addition\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-0/space folder/c.md", + "key": "round-0/space folder/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-0/space folder/c.md": "[seed-43-round-0-A]\nremote addition\n", + "round-0/space folder/b.md": "[seed-43-round-0-B]" + }, + "absent": [ + "round-0/space folder/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-1/space folder/a.md", + "content": "[seed-43-round-1-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-1/space folder/b.md", + "content": "[seed-43-round-1-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-1/space folder/a.md", + "key": "round-1/space folder/a.md" + }, + { + "type": "remember-identity", + "path": "round-1/space folder/b.md", + "key": "round-1/space folder/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/a.md", + "newPath": "round-1/space folder/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/b.md", + "newPath": "round-1/space folder/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/c.md", + "newPath": "round-1/space folder/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-1/space folder/b.md", + "content": "[seed-43-round-1-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-1/space folder/b.md", + "key": "round-1/space folder/a.md" + }, + { + "type": "assert-identity", + "path": "round-1/space folder/a.md", + "key": "round-1/space folder/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-1/space folder/a.md": "[seed-43-round-1-B]", + "round-1/space folder/b.md": "[seed-43-round-1-A]\nextra line\n" + }, + "absent": [ + "round-1/space folder/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]", + "[seed-43-round-1-A]", + "[seed-43-round-1-B]" + ], + "removed": [] + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-2/nested/a.md", + "content": "[seed-43-round-2-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-2/nested/b.md", + "content": "[seed-43-round-2-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-2/nested/a.md", + "key": "round-2/nested/a.md" + }, + { + "type": "remember-identity", + "path": "round-2/nested/b.md", + "key": "round-2/nested/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/a.md", + "newPath": "round-2/nested/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/b.md", + "newPath": "round-2/nested/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/c.md", + "newPath": "round-2/nested/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-2/nested/b.md", + "content": "[seed-43-round-2-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-2/nested/b.md", + "key": "round-2/nested/a.md" + }, + { + "type": "assert-identity", + "path": "round-2/nested/a.md", + "key": "round-2/nested/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-2/nested/a.md": "[seed-43-round-2-B]", + "round-2/nested/b.md": "[seed-43-round-2-A]\nextra line\n" + }, + "absent": [ + "round-2/nested/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]", + "[seed-43-round-1-A]", + "[seed-43-round-1-B]", + "[seed-43-round-2-A]", + "[seed-43-round-2-B]" + ], + "removed": [] + } + ] +} \ No newline at end of file diff --git a/sync-server/README.md b/sync-server/README.md index fc54337c..27e68c20 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -1,22 +1,23 @@ # Sync server -## API v4 +## API v5 The server is a compare-and-swap store. Content versions do not contain paths or deletions. A separately versioned file manifest owns the namespace and membership. All routes below are under `/vaults/:vault_id` and require bearer authentication (except the WebSocket, which authenticates its first message). -| Endpoint | Purpose | -| --- | --- | -| `PUT /documents/:document_id` | CAS an immutable content version. | -| `GET /documents/:document_id` | Current content version, including base64 bytes. | -| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. | -| `GET /file_manifest` | Current file manifest. | -| `PUT /file_manifest` | CAS the complete file manifest. | -| `GET /vault_snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. | -| `GET /events_since?after=N` | Every event after N, in order, with the current watermark. | -| `GET /ws` | Ordered event delivery and ephemeral cursor positions. | +| Endpoint | Purpose | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `PUT /documents/:document_id` | CAS an immutable content version. | +| `GET /documents/:document_id/metadata` | Current version metadata and size without content bytes. | +| `GET /documents/:document_id` | Current content version, including base64 bytes. | +| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. | +| `GET /file-manifest` | Current file manifest. | +| `PUT /file-manifest` | CAS the complete file manifest. | +| `GET /vault-snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. | +| `GET /events-since?after=N` | Every event after N, in order, with the current watermark. | +| `GET /ws` | Ordered event delivery and ephemeral cursor positions. | A content push includes a `Device-Id` header and this JSON body: @@ -54,13 +55,14 @@ UUID-to-path mapping in `file_manifest_entries`. Each version has a header row even when its entries are empty. Entries are unique by document ID and exact path within a version; portable path validation also rejects aliases and directory conflicts. Reads reconstruct the complete map, and vault snapshots join these -rows to the latest content versions. The version, entries, and complete file -manifest event in `events.event_json` commit atomically. Event replay and retry -acknowledgements continue to use `event_json`. +rows to the latest content versions. Event replay and retry acknowledgements are +also reconstructed from normalized content/manifest rows, without storing a +second complete map in the event table. Both pushes return a tagged `Accepted` or `StaleBase` response. An accepted file manifest response returns its assigned file manifest ID. A stale content response -includes bytes; a stale file manifest response includes the complete map. +includes metadata and byte length only; clients fetch immutable bytes after applying +their size and exclusion policy. A stale file manifest response includes the complete map. Accepted requests store their unique request ID and fingerprint in the event row, atomically with the version. Retrying the exact request reconstructs the original `Accepted` response from that event before checking the current head, even after @@ -70,8 +72,8 @@ identity is not part of the fingerprint. File manifest validation rejects duplicate UUID keys (including aliases), missing content, path aliases, inconsistent directory spelling, and file/ancestor -conflicts. Paths use `/`, NFC normalization, at most 240 UTF-8 bytes, and portable -Windows filename restrictions. Alias comparison is NFC → Unicode uppercase → +conflicts. Paths use `/`, NFC normalization, and portable Windows filename +restrictions without imposing a universal total-path byte limit. Alias comparison is NFC → Unicode uppercase → NFC. `.vault-link-sync` and its aliases are reserved. The server never sanitizes or allocates a replacement name. @@ -87,11 +89,19 @@ determine event ordering. The WebSocket handshake retains `lastSeenVaultUpdateId`. The server sends `{type: "vaultEvents", headEventId, events}` in increasing order, including events originated by that client. Each connection drains the database through one sender. -In-memory notifications only wake it; polling covers missed wakes and cancellation -after commit. Reconnect with the last durably processed event ID. The HTTP replay -endpoint uses the same history, so losing the final notification is recoverable. -A fresh client reads `/vault_snapshot`, applies it durably, then replays after its -watermark. +In-memory notifications only wake it; polling covers missed wakes and requests +interrupted after commit. Reconnect with the last durably processed event ID. The +HTTP replay endpoint uses the same history, so losing the final notification is +recoverable. +A fresh client reads `/vault-snapshot`, applies it durably, then replays after its +watermark. Successful HTTP responses include `X-Vault-Link-History`, containing +an event ID and a random event-incarnation token. Clients persist and echo this +checkpoint on later HTTP requests. The server returns HTTP 409 with +`X-Vault-Link-History-Mismatch: 1` before executing a request whose checkpoint is +absent from the current database. Restored backups retain their original prefix; +new events get new tokens even when numeric IDs or request UUIDs are reused. +Restore databases while the server is stopped. Live replacement of open SQLite +files is not supported. Cursor notifications are ephemeral and do not consume event IDs. Cursor expiry and disconnect broadcast the remaining clients, including an empty list when the last client leaves. @@ -116,8 +126,20 @@ flushes database directory entries; the filesystem must support those durability operations. Content, file manifests, and events (including request fingerprints) are retained indefinitely. -API v4 requires fresh vault databases and upgraded clients. Existing database -migration checksums intentionally reject earlier schemas; there is no automatic -erasure or migration. Use a new database directory and preserve old data. +Deploy API v5 clients and server together. The v4 database schema migrates by +adding event-incarnation tokens without deleting content or receipts. Schemas +from before v4 still require a separate database directory and explicit recovery. +Vault databases open and migrate independently on first access; one corrupt or +incompatible vault does not prevent healthy vaults from serving requests. + +Diffs are limited to 10,000 items and reconstructed in one pass over Unicode +scalars on a blocking worker, before taking the SQLite writer transaction. The +transaction rechecks request identity and the parent version before committing. +Clients send a snapshot when an edit exceeds the diff-operation budget. Decoded +content is bounded by `max_body_size_mb` as well as the encoded request limit. + +A missing configuration file is initialized with durably stored credentials. +Existing configurations must explicitly supply `users.user_configs`; omitted +credentials are rejected instead of generating an unpersisted token at startup. Base64 increases snapshot body size by about one third; request limits apply to the encoded JSON body. diff --git a/sync-server/src/app_state/cursors.rs b/sync-server/src/app_state/cursors.rs index 65b941ed..e535d5b2 100644 --- a/sync-server/src/app_state/cursors.rs +++ b/sync-server/src/app_state/cursors.rs @@ -2,6 +2,7 @@ use core::time::Duration; use std::{collections::HashMap, sync::Arc}; use tokio::sync::Mutex; +use uuid::Uuid; use super::{ database::models::{DeviceId, VaultId}, @@ -19,6 +20,7 @@ pub struct Cursors { config: DatabaseConfig, broadcasts: Broadcasts, vault_to_cursors: Arc>>>, + sessions: Arc>>>, } impl Cursors { @@ -27,16 +29,39 @@ impl Cursors { config: config.clone(), broadcasts: broadcasts.clone(), vault_to_cursors: Arc::new(Mutex::new(HashMap::new())), + sessions: Arc::default(), } } + pub async fn register_connection(&self, vault_id: &str, device_id: &str) -> Uuid { + let mut sessions = self.sessions.lock().await; + let session = Uuid::new_v4(); + sessions + .entry(vault_id.to_owned()) + .or_default() + .insert(device_id.to_owned(), session); + self.remove_device_cursors(vault_id, device_id).await; + session + } + pub async fn update_cursors( &self, vault_id: VaultId, user_name: String, device_id: &DeviceId, + session: Uuid, document_to_cursors: Vec, ) { + // A reconnect can overlap an old connection's buffered messages and + // cleanup. Only the most recently authenticated session owns presence. + let sessions = self.sessions.lock().await; + if sessions + .get(&vault_id) + .and_then(|devices| devices.get(device_id)) + != Some(&session) + { + return; + } let mut vault_to_cursors = self.vault_to_cursors.lock().await; let all_device_cursors = vault_to_cursors @@ -100,7 +125,22 @@ impl Cursors { .await; } - pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { + pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str, session: Uuid) { + let mut sessions = self.sessions.lock().await; + let Some(devices) = sessions.get_mut(vault_id) else { + return; + }; + if devices.get(device_id) != Some(&session) { + return; + } + devices.remove(device_id); + if devices.is_empty() { + sessions.remove(vault_id); + } + self.remove_device_cursors(vault_id, device_id).await; + } + + async fn remove_device_cursors(&self, vault_id: &str, device_id: &str) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { @@ -147,32 +187,42 @@ mod tests { async fn disconnect_broadcasts_remaining_clients_and_the_final_empty_state() { let broadcasts = Broadcasts::new(&ServerConfig::default()); let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + let mut sessions = HashMap::new(); for device in ["one", "two"] { + let session = cursors.register_connection("vault", device).await; + sessions.insert(device, session); cursors .update_cursors( "vault".to_owned(), "user".to_owned(), &device.to_owned(), + session, vec![], ) .await; } let mut receiver = broadcasts.get_receiver("vault".to_owned()).await; let mut other = broadcasts.get_receiver("other".to_owned()).await; - cursors.remove_cursors_of_device("vault", "one").await; + cursors + .remove_cursors_of_device("vault", "one", sessions["one"]) + .await; let Notification::Cursors(update) = receiver.try_recv().unwrap() else { panic!("expected cursors") }; assert_eq!(update.clients.len(), 1); assert_eq!(update.clients[0].device_id, "two"); - cursors.remove_cursors_of_device("vault", "two").await; + cursors + .remove_cursors_of_device("vault", "two", sessions["two"]) + .await; let Notification::Cursors(update) = receiver.try_recv().unwrap() else { panic!("expected cursors") }; assert!(update.clients.is_empty()); assert!(other.try_recv().is_err()); - cursors.remove_cursors_of_device("vault", "two").await; + cursors + .remove_cursors_of_device("vault", "two", sessions["two"]) + .await; assert!(receiver.try_recv().is_err()); } @@ -182,11 +232,13 @@ mod tests { let config = DatabaseConfig::default(); let cursors = Cursors::new(&config, &broadcasts); for (vault, device) in [("vault", "expired"), ("vault", "live"), ("other", "live")] { + let session = cursors.register_connection(vault, device).await; cursors .update_cursors( vault.to_owned(), "user".to_owned(), &device.to_owned(), + session, vec![], ) .await; @@ -229,4 +281,39 @@ mod tests { assert!(receiver.try_recv().is_err()); assert!(other.try_recv().is_err()); } + #[tokio::test] + async fn replacement_session_fences_old_updates_and_disconnects() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + let vault = "vault".to_owned(); + let device = "device".to_owned(); + let old = cursors.register_connection(&vault, &device).await; + cursors + .update_cursors(vault.clone(), "old".to_owned(), &device, old, vec![]) + .await; + let new = cursors.register_connection(&vault, &device).await; + cursors + .update_cursors(vault.clone(), "new".to_owned(), &device, new, vec![]) + .await; + let mut receiver = broadcasts.get_receiver(vault.clone()).await; + cursors + .update_cursors( + vault.clone(), + "late old update".to_owned(), + &device, + old, + vec![], + ) + .await; + cursors.remove_cursors_of_device(&vault, &device, old).await; + assert!(receiver.try_recv().is_err()); + assert_eq!(cursors.get_cursors(&vault).await[0].user_name, "new"); + cursors.remove_cursors_of_device(&vault, &device, new).await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(cursors.get_cursors(&vault).await.is_empty()); + assert!(cursors.sessions.lock().await.is_empty()); + } } diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 9aa99db5..eca69a44 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -2,7 +2,7 @@ use core::time::Duration; use std::{ collections::HashMap, path::{Path, PathBuf}, - sync::Arc, + sync::{Arc, Weak}, }; use anyhow::{Context as _, Result}; @@ -48,6 +48,7 @@ impl std::fmt::Debug for PoolWithTimestamp { pub struct Database { config: DatabaseConfig, connection_pools: Arc>>, + opening_pools: Arc>>>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; @@ -66,25 +67,16 @@ impl Database { ) })?; - info!("Applying pending database migrations"); + // SQLite flushes its own files, not ancestors created by this process. + Self::flush_directory_ancestors(&vault_directory).await?; - // Filenames encode only a digest, so their vault names cannot be - // recovered here. Check migrations now and open pools lazily by name. - let mut entries = tokio::fs::read_dir(&vault_directory).await?; - while let Some(entry) = entries.next_entry().await? { - if entry.file_type().await?.is_file() - && entry.path().extension().is_some_and(|ext| ext == "sqlite") - { - Self::open_database(config, &entry.path()) - .await? - .close() - .await; - } - } + // Open and migrate each vault lazily under its own lock. A corrupt or + // incompatible database must fail that vault, not global startup. let database = Self { config: config.clone(), connection_pools: Arc::default(), + opening_pools: Arc::default(), }; // Start background task to cleanup idle connection pools @@ -127,9 +119,32 @@ impl Database { Self::run_migrations(&pool).await?; + if let Some(directory) = file_name.parent() { + Self::flush_directory_ancestors(directory).await?; + } + Ok(pool) } + async fn flush_directory_ancestors(directory: &Path) -> Result<()> { + #[cfg(unix)] + { + let directory = tokio::fs::canonicalize(directory).await?; + tokio::task::spawn_blocking(move || -> Result<()> { + for ancestor in directory.ancestors() { + std::fs::File::open(ancestor)?.sync_all().with_context(|| { + format!("Cannot flush directory {}", ancestor.display()) + })?; + } + Ok(()) + }) + .await??; + } + #[cfg(not(unix))] + let _ = directory; + Ok(()) + } + async fn run_migrations(pool: &Pool) -> Result<()> { sqlx::migrate!("src/app_state/database/migrations") .run(pool) @@ -140,28 +155,49 @@ impl Database { async fn get_connection_pool(&self, vault: &VaultId) -> Result> { let vault = normalize_string(vault); validate_vault_id(&vault)?; - let mut pools = self.connection_pools.lock().await; - - if !pools.contains_key(&vault) { - let file_name = Self::database_path(&self.config, &vault); - let pool = Self::open_database(&self.config, &file_name).await?; - pools.insert( - vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, - ); + if let Some(pool) = self.existing_pool(&vault).await { + return Ok(pool); } - let pool_with_timestamp = pools - .get_mut(&vault) - .expect("Pool was just inserted or already exists"); + // Serialize initialization per vault. SQLite lock waits, migrations and + // directory flushes must never hold the registry for unrelated vaults. + // Weak entries also allow cancelled/failed opens to be retried safely. + let opening = { + let mut openings = self.opening_pools.lock().await; + openings.retain(|_, opening| opening.strong_count() > 0); + if let Some(opening) = openings.get(&vault).and_then(Weak::upgrade) { + opening + } else { + let opening = Arc::new(Mutex::new(())); + openings.insert(vault.clone(), Arc::downgrade(&opening)); + opening + } + }; + let _opening = opening.lock().await; + if let Some(pool) = self.existing_pool(&vault).await { + return Ok(pool); + } + let file_name = Self::database_path(&self.config, &vault); + let pool = Self::open_database(&self.config, &file_name).await?; + self.connection_pools.lock().await.insert( + vault, + PoolWithTimestamp { + pool: pool.clone(), + last_accessed: Instant::now(), + }, + ); + Ok(pool) + } - // Update last accessed time - pool_with_timestamp.last_accessed = Instant::now(); - - Ok(pool_with_timestamp.pool.clone()) + async fn existing_pool(&self, vault: &VaultId) -> Option> { + self.connection_pools + .lock() + .await + .get_mut(vault) + .map(|entry| { + entry.last_accessed = Instant::now(); + entry.pool.clone() + }) } /// Attempting to write from this transaction might result in a @@ -196,16 +232,22 @@ impl Database { .iter() .filter(|(_, pool_with_timestamp)| { now.duration_since(pool_with_timestamp.last_accessed) > idle_timeout + && pool_with_timestamp.pool.num_idle() + == pool_with_timestamp.pool.size() as usize }) .map(|(vault_id, _)| vault_id.clone()) .collect(); - // Close and remove idle pools - for vault_id in &vaults_to_remove { - if let Some(pool_with_timestamp) = pools.remove(vault_id) { - info!("Closing idle database connection pool for vault `{vault_id}`"); - pool_with_timestamp.pool.close().await; - } + let closing: Vec<_> = vaults_to_remove + .into_iter() + .filter_map(|vault| pools.remove(&vault).map(|entry| (vault, entry.pool))) + .collect(); + drop(pools); + // A checkout racing cleanup can delay close. It must never retain the + // mutex used to access every other vault in the process. + for (vault, pool) in closing { + info!("Closing idle database connection pool for vault `{vault}`"); + pool.close().await; } } diff --git a/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql b/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql new file mode 100644 index 00000000..dc96b259 --- /dev/null +++ b/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql @@ -0,0 +1,4 @@ +-- Tokens identify an event's incarnation, including when a restored backup +-- reuses its numeric ID or a client resubmits the same request UUID. +ALTER TABLE events ADD COLUMN event_token TEXT; +UPDATE events SET event_token = lower(hex(randomblob(16))); diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index 57c4cd94..edb3f885 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -123,5 +123,9 @@ pub struct VaultSnapshot { pub struct EventBatch { #[ts(as = "f64")] pub head_event_id: VaultUpdateId, + /// Last event in this page; `head_event_id` is the vault's current head. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional, as = "Option")] + pub end_event_id: Option, pub events: Vec, } diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs index b0af9e4a..6585d102 100644 --- a/sync-server/src/app_state/database/mutations.rs +++ b/sync-server/src/app_state/database/mutations.rs @@ -10,14 +10,15 @@ impl Database { request_id: uuid::Uuid, fingerprint: &[u8], ) -> Result { - Ok( - sqlx::query("INSERT INTO events(request_id, request_fingerprint) VALUES (?, ?)") - .bind(request_id.to_string()) - .bind(fingerprint) - .execute(&mut **tx) - .await? - .last_insert_rowid(), + Ok(sqlx::query( + "INSERT INTO events(request_id, request_fingerprint, event_token) VALUES (?, ?, ?)", ) + .bind(request_id.to_string()) + .bind(fingerprint) + .bind(uuid::Uuid::new_v4().simple().to_string()) + .execute(&mut **tx) + .await? + .last_insert_rowid()) } pub async fn insert_document_version( diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs index 5ba1a97d..6afec8d7 100644 --- a/sync-server/src/app_state/database/queries.rs +++ b/sync-server/src/app_state/database/queries.rs @@ -8,6 +8,22 @@ use super::{ use anyhow::{Context as _, Result, ensure}; impl Database { + pub async fn missing_document( + tx: &mut Transaction<'_>, + ids: &[DocumentId], + ) -> Result> { + // A single covering-index lookup per requested identity; never load a + // content blob or decode a version while validating manifest references. + Ok(sqlx::query_scalar( + "SELECT value FROM json_each(?) AS requested + WHERE NOT EXISTS (SELECT 1 FROM documents WHERE document_id = requested.value) + LIMIT 1", + ) + .bind(serde_json::to_string(ids)?) + .fetch_optional(&mut **tx) + .await?) + } + pub async fn get_request_event( tx: &mut Transaction<'_>, request_id: uuid::Uuid, @@ -168,24 +184,72 @@ impl Database { ); let rows: Vec<(VaultUpdateId, String)> = sqlx::query_as( - "SELECT event_id, request_id FROM events WHERE event_id > ? ORDER BY event_id", + "SELECT event_id, request_id FROM events WHERE event_id > ? ORDER BY event_id LIMIT 64", ) .bind(after) .fetch_all(&mut *tx) .await?; let mut events = Vec::with_capacity(rows.len()); + let mut bytes = 0; + let mut end_event_id = after; for (event_id, request_id) in rows { - events.push(EventRecord { + let event = EventRecord { event_id, request_id: request_id.parse()?, event: Self::event_by_id(&mut tx, event_id).await?, - }); + }; + let size = serde_json::to_vec(&event)?.len(); + // One manifest is atomic and may exceed this budget. Always allow + // one event so even such a page advances; never accumulate history. + if !events.is_empty() && bytes + size > 1024 * 1024 { + break; + } + bytes += size; + end_event_id = event_id; + events.push(event); } Ok(EventBatch { head_event_id, + end_event_id: Some(end_event_id), events, }) } } + +impl Database { + pub async fn history_checkpoint(&self, vault: &VaultId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let row: Option<(i64, String)> = sqlx::query_as( + "SELECT event_id, event_token FROM events ORDER BY event_id DESC LIMIT 1", + ) + .fetch_optional(&mut *tx) + .await?; + Ok(row.map_or_else( + || "0:empty".to_owned(), + |(id, token)| format!("{id}:{token}"), + )) + } + + pub async fn contains_checkpoint(&self, vault: &VaultId, checkpoint: &str) -> Result { + if checkpoint == "0:empty" { + return Ok(true); + } + let Some((id, token)) = checkpoint.split_once(':') else { + return Ok(false); + }; + let Ok(id) = id.parse::() else { + return Ok(false); + }; + let mut tx = self.create_readonly_transaction(vault).await?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM events WHERE event_id = ? AND event_token = ?)", + ) + .bind(id) + .bind(token) + .fetch_one(&mut *tx) + .await?; + Ok(exists) + } +} diff --git a/sync-server/src/app_state/database/tests.rs b/sync-server/src/app_state/database/tests.rs index 9b5b1e62..b3578f46 100644 --- a/sync-server/src/app_state/database/tests.rs +++ b/sync-server/src/app_state/database/tests.rs @@ -94,6 +94,32 @@ async fn rejects_invalid_vault_names_at_the_database_boundary() { assert!(database.connection_pools.lock().await.is_empty()); } +#[tokio::test] +async fn migrations_can_be_reapplied_without_losing_requests_or_content() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + let vault = "migrations".to_owned(); + let version = insert_content(&database, &vault, uuid::Uuid::new_v4(), b"only copy").await; + let pool = database.get_connection_pool(&vault).await.unwrap(); + Database::run_migrations(&pool).await.unwrap(); + Database::run_migrations(&pool).await.unwrap(); + let reopened = Database::try_new(&config).await.unwrap(); + assert_eq!( + reopened + .get_document_version(&vault, version.vault_update_id, None) + .await + .unwrap() + .unwrap() + .content, + b"only copy" + ); + assert_eq!( + reopened.events_after(&vault, 0).await.unwrap().events.len(), + 1 + ); +} + #[tokio::test] async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { let directory = tempfile::tempdir().unwrap(); @@ -134,3 +160,246 @@ async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { ); } } + +#[tokio::test] +async fn replay_pages_are_bounded_and_cover_every_event_in_order() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "pages".to_owned(); + let id = uuid::Uuid::new_v4(); + for _ in 0..150 { + insert_content(&database, &vault, id, b"edit").await; + } + let mut after = 0; + while after < 150 { + let batch = database.events_after(&vault, after).await.unwrap(); + assert!(batch.events.len() <= 64, "catchup must bound each page"); + assert!(!batch.events.is_empty()); + assert_eq!(batch.head_event_id, 150); + for event in batch.events { + assert_eq!(event.event_id, after + 1); + after = event.event_id; + } + } + assert!( + database + .events_after(&vault, after) + .await + .unwrap() + .events + .is_empty() + ); +} + +#[tokio::test] +async fn idle_cleanup_does_not_wait_for_an_active_vault() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + database.vault_snapshot(&"other".to_owned()).await.unwrap(); + let vault = "slow".to_owned(); + let transaction = database.create_write_transaction(&vault).await.unwrap(); + database + .connection_pools + .lock() + .await + .get_mut(&vault) + .unwrap() + .last_accessed = Instant::now() - Duration::from_secs(301); + tokio::time::timeout(Duration::from_millis(200), database.cleanup_idle_pools()) + .await + .expect("cleanup must leave an active pool alone"); + let other = tokio::time::timeout( + Duration::from_millis(200), + database.vault_snapshot(&"other".to_owned()), + ) + .await + .expect("other vaults must remain accessible") + .unwrap(); + assert_eq!(other.head_event_id, 0); + assert!( + !database + .get_connection_pool(&vault) + .await + .unwrap() + .is_closed() + ); + drop(transaction); +} + +#[tokio::test] +async fn manifest_existence_checks_are_independent_of_payload_decoding() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "existence".to_owned(); + let id = uuid::Uuid::new_v4(); + insert_content(&database, &vault, id, b"content").await; + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + // Make full-row deserialization fail: checking a foreign identity must not + // decode unrelated version fields or read its potentially enormous blob. + sqlx::query("UPDATE documents SET updated_date = 'unreadable timestamp'") + .execute(&mut *tx) + .await + .unwrap(); + assert_eq!( + Database::missing_document(&mut tx, &[id]).await.unwrap(), + None + ); + let missing = uuid::Uuid::new_v4(); + assert_eq!( + Database::missing_document(&mut tx, &[id, missing]) + .await + .unwrap(), + Some(missing.to_string()) + ); + assert_eq!( + Database::missing_document(&mut tx, &[]).await.unwrap(), + None + ); +} + +#[tokio::test] +async fn replay_byte_budget_prevents_accumulating_large_historical_manifests() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "large-manifests".to_owned(); + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + // Each event fits independently, but fewer than 64 fit the response budget. + let entries: std::collections::BTreeMap<_, _> = (0..1500) + .map(|index| { + ( + uuid::Uuid::new_v4(), + format!("{index}-{}.md", "x".repeat(180)), + ) + }) + .collect(); + for _ in 0..8 { + let file_manifest_id = Database::allocate_event(&mut tx, uuid::Uuid::new_v4(), b"test") + .await + .unwrap(); + Database::insert_file_manifest( + &mut tx, + &FileManifest { + file_manifest_id, + entries: entries.clone(), + }, + ) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + let batch = database.events_after(&vault, 0).await.unwrap(); + assert!(!batch.events.is_empty()); + assert!(batch.events.len() < 8); + assert!(serde_json::to_vec(&batch).unwrap().len() < 1024 * 1024 + 1024); + assert_eq!( + batch.end_event_id, + batch.events.last().map(|event| event.event_id) + ); +} + +#[tokio::test] +async fn opening_a_locked_vault_does_not_block_unrelated_vaults() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + database.vault_snapshot(&"other".to_owned()).await.unwrap(); + let vault = "locked-opening".to_owned(); + // A second process holds the writer lock while this process lazily checks + // migrations. Even already-open, unrelated vaults must remain available. + let external = Database::open_database(&config, &Database::database_path(&config, &vault)) + .await + .unwrap(); + let transaction = external.begin_with("BEGIN IMMEDIATE").await.unwrap(); + let opening_database = database.clone(); + let opening = tokio::spawn(async move { opening_database.get_connection_pool(&vault).await }); + tokio::time::sleep(Duration::from_millis(100)).await; + let opening_is_blocked = !opening.is_finished(); + let other = tokio::time::timeout( + Duration::from_millis(300), + database.vault_snapshot(&"other".to_owned()), + ) + .await; + transaction.rollback().await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), opening) + .await + .unwrap() + .unwrap() + .unwrap(); + external.close().await; + assert!( + opening_is_blocked, + "the test must exercise an in-progress database open" + ); + assert_eq!( + other + .expect("a blocked migration must not lock every vault") + .unwrap() + .head_event_id, + 0 + ); +} + +#[tokio::test] +async fn concurrent_vault_opens_share_a_pool_and_cancelled_opens_can_retry() { + let directory = tempfile::tempdir().unwrap(); + let mut config = config(&directory); + config.max_connections_per_vault = 1; + let database = Database::try_new(&config).await.unwrap(); + let vault = "cancelled-opening".to_owned(); + let external = Database::open_database(&config, &Database::database_path(&config, &vault)) + .await + .unwrap(); + let transaction = external.begin_with("BEGIN IMMEDIATE").await.unwrap(); + let opener = database.clone(); + let name = vault.clone(); + let cancelled = tokio::spawn(async move { opener.get_connection_pool(&name).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!cancelled.is_finished()); + cancelled.abort(); + assert!(cancelled.await.unwrap_err().is_cancelled()); + transaction.rollback().await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..8 { + let database = database.clone(); + let vault = vault.clone(); + tasks.push(tokio::spawn(async move { + database.get_connection_pool(&vault).await.unwrap() + })); + } + let pools = tokio::time::timeout(Duration::from_secs(5), futures::future::join_all(tasks)) + .await + .unwrap(); + let pools: Vec<_> = pools.into_iter().map(Result::unwrap).collect(); + let checked_out = pools[0].acquire().await.unwrap(); + for pool in &pools[1..] { + assert!( + tokio::time::timeout(Duration::from_millis(25), pool.acquire()) + .await + .is_err(), + "all callers must share the single configured connection" + ); + } + drop(checked_out); + external.close().await; +} + +#[tokio::test] +async fn one_corrupt_vault_does_not_prevent_startup_or_healthy_vault_access() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let bad = Database::database_path(&config, "broken"); + tokio::fs::create_dir_all(bad.parent().unwrap()) + .await + .unwrap(); + tokio::fs::write(&bad, b"not a SQLite database") + .await + .unwrap(); + let database = Database::try_new(&config).await.unwrap(); + assert!(database.vault_snapshot(&"broken".to_owned()).await.is_err()); + let saved = insert_content(&database, "healthy", uuid::Uuid::new_v4(), b"safe").await; + assert_eq!(saved.content, b"safe"); + assert_eq!( + tokio::fs::read(bad).await.unwrap(), + b"not a SQLite database" + ); +} diff --git a/sync-server/src/app_state/websocket/broadcasts.rs b/sync-server/src/app_state/websocket/broadcasts.rs index 1b157b13..0c5a8f83 100644 --- a/sync-server/src/app_state/websocket/broadcasts.rs +++ b/sync-server/src/app_state/websocket/broadcasts.rs @@ -12,23 +12,39 @@ pub enum Notification { #[derive(Debug, Clone)] pub struct Broadcasts { capacity: usize, + admission: Arc>>>, senders: Arc>>>, } impl Broadcasts { pub fn new(config: &ServerConfig) -> Self { Self { - capacity: config.max_clients_per_vault.max(1), + capacity: config.max_clients_per_vault, + admission: Arc::default(), senders: Arc::default(), } } + pub fn try_admit(&self, vault: &VaultId) -> Option { + let mut limits = self.admission.lock().unwrap(); + limits.retain(|_, limit| limit.strong_count() > 0); + let limit = limits + .get(vault) + .and_then(std::sync::Weak::upgrade) + .unwrap_or_else(|| { + let limit = Arc::new(tokio::sync::Semaphore::new(self.capacity)); + limits.insert(vault.clone(), Arc::downgrade(&limit)); + limit + }); + limit.try_acquire_owned().ok() + } + async fn sender(&self, vault: VaultId) -> broadcast::Sender { self.senders .lock() .await .entry(vault) - .or_insert_with(|| broadcast::channel(self.capacity).0) + .or_insert_with(|| broadcast::channel(self.capacity.max(1)).0) .clone() } diff --git a/sync-server/src/app_state/websocket/utils.rs b/sync-server/src/app_state/websocket/utils.rs index e72a3775..a1423e09 100644 --- a/sync-server/src/app_state/websocket/utils.rs +++ b/sync-server/src/app_state/websocket/utils.rs @@ -49,9 +49,13 @@ pub async fn send_update_over_websocket( .context("Failed to serialize update") .map_err(server_error)?; - sender - .send(Message::Text(serialized_update)) - .await - .context("Failed to send message over websocket") - .map_err(server_error) + tokio::time::timeout( + std::time::Duration::from_secs(10), + sender.send(Message::Text(serialized_update)), + ) + .await + .context("WebSocket send deadline expired") + .map_err(server_error)? + .context("Failed to send message over websocket") + .map_err(server_error) } diff --git a/sync-server/src/config.rs b/sync-server/src/config.rs index 6a003d2e..c4246403 100644 --- a/sync-server/src/config.rs +++ b/sync-server/src/config.rs @@ -20,7 +20,6 @@ pub struct Config { pub database: DatabaseConfig, #[serde(default)] pub server: ServerConfig, - #[serde(default)] pub users: UserConfig, #[serde(default)] pub logging: LoggingConfig, @@ -28,23 +27,52 @@ pub struct Config { impl Config { pub async fn read_or_create(path: &Path) -> Result { - let config = if path.exists() { - info!( - "Loading configuration from `{}`", - path.canonicalize().unwrap().display() - ); - Self::load_from_file(path).await? - } else { - Self::default() - }; - - config.write(path).await?; - info!( - "Updated configuration at `{}`", - path.canonicalize().unwrap().display() - ); - - Ok(config) + match Self::load_from_file(path).await { + Ok(config) => return Ok(config), + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => {} + Err(error) => return Err(error), + } + let contents = serde_yaml::to_string(&Self::default())?; + let destination = path.to_owned(); + tokio::task::spawn_blocking(move || -> Result<()> { + use std::io::Write; + let path = destination; + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let temporary = parent.join(format!(".vault-link-config-{}.tmp", uuid::Uuid::new_v4())); + let result = (|| -> Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + file.write_all(contents.as_bytes())?; + file.sync_all()?; + // Publish a complete, durable file without replacing a config + // another process may have created while we generated defaults. + match std::fs::hard_link(&temporary, &path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })(); + let _ = std::fs::remove_file(&temporary); + result + }) + .await??; + info!("Loading configuration from `{}`", path.display()); + Self::load_from_file(path).await } pub async fn load_from_file(path: &Path) -> Result { @@ -57,12 +85,54 @@ impl Config { serde_yaml::from_str(&contents).context("Failed to parse configuration") } +} - pub async fn write(&self, path: &Path) -> Result<()> { - let contents = serde_yaml::to_string(&self).context("Failed to serialize configuration")?; +#[cfg(test)] +mod tests { + use super::*; - fs::write(path, contents) - .await - .context("Failed to write configuration to disk") + #[tokio::test] + async fn startup_preserves_existing_configuration_bytes_and_permissions() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + let original = b"# administrator's comments and formatting\nserver: {port: 9123}\nusers: {user_configs: []}\n"; + fs::write(&path, original).await.unwrap(); + let mut permissions = fs::metadata(&path).await.unwrap().permissions(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).await.unwrap(); + let config = Config::read_or_create(&path).await.unwrap(); + assert_eq!(config.server.port, 9123); + assert_eq!(fs::read(&path).await.unwrap(), original); + } + + #[tokio::test] + async fn existing_configuration_cannot_generate_unpersisted_credentials() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + for contents in ["server: {port: 9123}\n", "users: {}\n"] { + fs::write(&path, contents).await.unwrap(); + for _ in 0..2 { + assert!( + Config::read_or_create(&path).await.is_err(), + "Existing credentials must be explicit; random defaults are only for initialization" + ); + } + assert_eq!(fs::read_to_string(&path).await.unwrap(), contents); + } + } + + #[tokio::test] + async fn concurrent_initialization_never_overwrites_configuration() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + let results = + futures::future::join_all((0..16).map(|_| Config::read_or_create(&path))).await; + let config = Config::load_from_file(&path).await.unwrap(); + for result in results { + assert_eq!( + serde_yaml::to_string(&result.unwrap()).unwrap(), + serde_yaml::to_string(&config).unwrap() + ); + } } } diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index 59d1480f..fac5cf98 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -85,8 +85,9 @@ mod tests { let path = directory.path().join("config.yml"); let fresh = Config::read_or_create(&path).await.unwrap(); let reread = Config::load_from_file(&path).await.unwrap(); - let omitted: Config = serde_yaml::from_str("{}").unwrap(); - let explicit: Config = serde_yaml::from_str("server: {}").unwrap(); + let omitted: Config = serde_yaml::from_str("users: {user_configs: []}").unwrap(); + let explicit: Config = + serde_yaml::from_str("server: {}\nusers: {user_configs: []}").unwrap(); for config in [fresh, reread, omitted, explicit] { assert_eq!(config.server.host, DEFAULT_HOST); assert_eq!(config.server.port, DEFAULT_PORT); diff --git a/sync-server/src/config/user_config.rs b/sync-server/src/config/user_config.rs index 0bb3379b..985a5c2a 100644 --- a/sync-server/src/config/user_config.rs +++ b/sync-server/src/config/user_config.rs @@ -9,7 +9,7 @@ use crate::{ #[derive(Debug, Deserialize, Serialize, Clone)] pub struct UserConfig { - #[serde(default = "default_users", deserialize_with = "validate_users")] + #[serde(deserialize_with = "validate_users")] pub user_configs: Vec, } diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 9eb66172..4df81e76 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -63,6 +63,11 @@ pub async fn create_server(config: Config) -> Result<()> { http::header::CONTENT_TYPE, http::header::AUTHORIZATION, DEVICE_ID_HEADER_NAME.clone(), + http::header::HeaderName::from_static("x-vault-link-history"), + ]) + .expose_headers([ + http::header::HeaderName::from_static("x-vault-link-history"), + http::header::HeaderName::from_static("x-vault-link-history-mismatch"), ]) .allow_methods([Method::GET, Method::PUT]), ) @@ -113,6 +118,10 @@ fn get_authed_routes(app_state: AppState) -> Router { get(endpoints::fetch_latest_document_version::fetch_latest_document_version) .put(endpoints::put_file_content::put_file_content), ) + .route( + "/vaults/:vault_id/documents/:document_id/metadata", + get(endpoints::fetch_latest_document_version::fetch_latest_document_metadata), + ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id/content", get(endpoints::fetch_document_version_content::fetch_document_version_content), diff --git a/sync-server/src/server/auth.rs b/sync-server/src/server/auth.rs index c601ef54..c7a49378 100644 --- a/sync-server/src/server/auth.rs +++ b/sync-server/src/server/auth.rs @@ -2,8 +2,9 @@ use std::collections::HashMap; use axum::{ extract::{Path, Request, State}, + http::{HeaderValue, StatusCode}, middleware::Next, - response::Response, + response::{IntoResponse, Response}, }; use axum_extra::{ TypedHeader, @@ -14,7 +15,9 @@ use log::info; use crate::{ app_state::{AppState, database::models::VaultId}, config::user_config::{AllowListedVaults, User, VaultAccess}, - errors::{SyncServerError, client_error, permission_denied_error, unauthenticated_error}, + errors::{ + SyncServerError, client_error, permission_denied_error, server_error, unauthenticated_error, + }, utils::normalize_vault_id::{normalize_string, validate_vault_id}, }; @@ -34,9 +37,42 @@ pub async fn auth_middleware( let user = auth(&state, token, &vault_id)?; + // A restore happens while the server is stopped. Validate before executing + // any old request, including retries whose numeric parent IDs were reused. + if let Some(checkpoint) = req.headers().get("x-vault-link-history") { + let checkpoint = checkpoint + .to_str() + .map_err(|error| client_error(error.into()))?; + if !state + .database + .contains_checkpoint(&vault_id, checkpoint) + .await + .map_err(server_error)? + { + let mut response = (StatusCode::CONFLICT, "Server history changed").into_response(); + response.headers_mut().insert( + "x-vault-link-history-mismatch", + HeaderValue::from_static("1"), + ); + return Ok(response); + } + } + req.extensions_mut().insert(user); - Ok(next.run(req).await) + let mut response = next.run(req).await; + if response.status().is_success() { + let checkpoint = state + .database + .history_checkpoint(&vault_id) + .await + .map_err(server_error)?; + response.headers_mut().insert( + "x-vault-link-history", + HeaderValue::from_str(&checkpoint).map_err(|error| server_error(error.into()))?, + ); + } + Ok(response) } pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result { diff --git a/sync-server/src/server/endpoints/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs index 870ac669..3a51f182 100644 --- a/sync-server/src/server/endpoints/fetch_latest_document_version.rs +++ b/sync-server/src/server/endpoints/fetch_latest_document_version.rs @@ -9,7 +9,7 @@ use super::VaultPath; use crate::{ app_state::{ AppState, - database::models::{DocumentId, DocumentVersion}, + database::models::{DocumentId, DocumentVersion, DocumentVersionWithoutContent}, }, errors::{SyncServerError, not_found_error, server_error}, }; @@ -37,3 +37,21 @@ pub async fn fetch_latest_document_version( Ok(Json(latest_version.into())) } + +pub async fn fetch_latest_document_metadata( + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, + State(state): State, +) -> Result, SyncServerError> { + let mut tx = state + .database + .create_readonly_transaction(&vault_id) + .await + .map_err(server_error)?; + let metadata = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT vault_update_id, document_id, updated_date, user_id, device_id, length(content) AS content_size + FROM documents WHERE document_id = ? ORDER BY vault_update_id DESC LIMIT 1" + ).bind(document_id.hyphenated()).fetch_optional(&mut *tx).await + .map_err(|error| server_error(error.into()))? + .ok_or_else(|| not_found_error(anyhow!("Document does not exist")))?; + Ok(Json(metadata)) +} diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index 4af4504c..2796e71c 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -23,7 +23,7 @@ use axum::{ use axum_extra::TypedHeader; use base64::{Engine as _, engine::general_purpose::STANDARD}; use log::debug; -use reconcile_text::{BuiltinTokenizer, EditedText, NumberOrText}; +use reconcile_text::NumberOrText; use sha2::{Digest, Sha256}; #[axum::debug_handler] @@ -41,6 +41,30 @@ pub async fn put_file_content( .map_err(|error| server_error(error.into()))?, ); + // Build bytes against an immutable parent off the async executor and before + // taking the writer lock. The transaction below rechecks CAS and receipts. + let initial_parent = state + .database + .get_latest_document_version(&vault_id, &document_id, None) + .await + .map_err(server_error)?; + let prepared = if initial_parent.as_ref().map(|p| p.vault_update_id) == push.parent_version_id { + let limit = state + .config + .server + .max_body_size_mb + .saturating_mul(1024 * 1024); + Some( + tokio::task::spawn_blocking(move || { + decode_content(initial_parent, push.content, limit) + }) + .await + .map_err(|error| server_error(error.into()))?, + ) + } else { + None + }; + let mut tx = state .database .create_write_transaction(&vault_id) @@ -78,33 +102,13 @@ pub async fn put_file_content( ))); } - let content = match push.content { - PushContent::Snapshot(bytes) => STANDARD - .decode(bytes) - .map_err(|error| client_error(error.into()))?, - - PushContent::Diff(diff) => { - if diff - .iter() - .any(|n| matches!(n, NumberOrText::Number(i64::MIN))) - { - return Err(client_error(anyhow!("Invalid diff length"))); - } - - let parent = parent - .as_ref() - .ok_or_else(|| client_error(anyhow!("Diff requires a parent")))?; - - let text = - str::from_utf8(&parent.content).map_err(|error| client_error(error.into()))?; - - EditedText::from_diff(text, diff, &*BuiltinTokenizer::Word) - .map_err(|error| client_error(error.into()))? - .apply() - .text() - .into_bytes() - } - }; + let content = prepared + .ok_or_else(|| { + server_error(anyhow!( + "Document changed while preparing content; retry the request" + )) + })? + .map_err(client_error)?; let version = StoredDocumentVersion { vault_update_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) @@ -132,8 +136,65 @@ pub async fn put_file_content( Ok(Json(response)) } +const MAX_DIFF_ITEMS: usize = 10_000; + +fn decode_content( + parent: Option, + content: PushContent, + limit: usize, +) -> anyhow::Result> { + let bytes = match content { + PushContent::Snapshot(encoded) => STANDARD.decode(encoded)?, + PushContent::Diff(diff) => { + anyhow::ensure!( + diff.len() <= MAX_DIFF_ITEMS, + "Diff exceeds operation budget; send a snapshot" + ); + let parent = parent.ok_or_else(|| anyhow!("Diff requires a parent"))?; + let text = str::from_utf8(&parent.content)?; + apply_diff(text, diff, limit)? + } + }; + anyhow::ensure!(bytes.len() <= limit, "Content exceeds size limit"); + Ok(bytes) +} + +/// Each Unicode scalar in the parent is visited at most once. No tokenization +/// or repeated scans of a prefix are needed to apply the wire diff. +fn apply_diff(text: &str, diff: Vec, limit: usize) -> anyhow::Result> { + let mut source = text.chars(); + let mut result = String::new(); + for item in diff { + match item { + NumberOrText::Text(insert) => { + anyhow::ensure!( + insert.len() <= limit.saturating_sub(result.len()), + "Content exceeds size limit" + ); + result.push_str(&insert); + } + NumberOrText::Number(length) => { + for _ in 0..length.unsigned_abs() { + let ch = source + .next() + .ok_or_else(|| anyhow!("Diff exceeds parent length"))?; + if length > 0 { + anyhow::ensure!( + ch.len_utf8() <= limit.saturating_sub(result.len()), + "Content exceeds size limit" + ); + result.push(ch); + } + } + } + } + } + Ok(result.into_bytes()) +} + #[cfg(test)] mod tests { + use reconcile_text::EditedText; use std::path::PathBuf; use super::*; @@ -278,6 +339,29 @@ mod tests { } } + #[tokio::test] + async fn fragmented_diff_budget_rejects_without_committing_an_event() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted( + vault + .send(snapshot(document, None, &"a".repeat(1024 * 1024))) + .await, + ); + let mut push = snapshot(document, Some(base.vault_update_id), ""); + let mut fragments = vec![NumberOrText::Number(0); 10_001]; + fragments.push(NumberOrText::Number(1024 * 1024)); + push.content = PushContent::Diff(fragments); + assert!(matches!( + push.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(document).await.vault_update_id, + base.vault_update_id + ); + } + #[tokio::test] async fn json_snapshot_preserves_bytes_and_rejects_invalid_base64_without_a_version() { let vault = TestVault::new().await; @@ -363,11 +447,8 @@ mod tests { outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"), }; let latest = vault.latest(document).await; - assert_eq!(winner.vault_update_id, rejected.metadata.vault_update_id); - assert_eq!( - STANDARD.decode(rejected.content_base64).unwrap(), - latest.content - ); + assert_eq!(winner.vault_update_id, rejected.vault_update_id); + assert_eq!(rejected.content_size, latest.content.len()); assert!(latest.content == b"left" || latest.content == b"right"); let original = vault .state @@ -577,7 +658,7 @@ mod tests { assert!(matches!( vault.send(foreign_base).await, DocumentUpdateResponse::StaleBase(version) - if version.metadata.vault_update_id == updated.vault_update_id + if version.vault_update_id == updated.vault_update_id )); let binary_document = uuid::Uuid::new_v4(); diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs index 98b559d2..05a26e11 100644 --- a/sync-server/src/server/endpoints/put_file_manifest.rs +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -62,18 +62,14 @@ pub async fn put_file_manifest( return Ok(Json(FileManifestUpdateResponse::StaleBase(latest))); } - for id in push.entries.keys() { - if state - .database - .get_latest_document_version(&vault_id, id, Some(&mut tx)) + if let Some(id) = + Database::missing_document(&mut tx, &push.entries.keys().copied().collect::>()) .await .map_err(server_error)? - .is_none() - { - return Err(client_error(anyhow!( - "File manifest references missing content: {id}" - ))); - } + { + return Err(client_error(anyhow!( + "File manifest references missing content: {id}" + ))); } let file_manifest = FileManifest { diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index ad1fe8c0..3a97911b 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -15,7 +15,8 @@ use axum::{ Path, State, ws::{Message, WebSocket, WebSocketUpgrade}, }, - response::Response, + http::StatusCode, + response::{IntoResponse, Response}, }; use futures::StreamExt; use log::{debug, info}; @@ -32,11 +33,22 @@ pub async fn websocket_handler( ) -> Result { debug!("Upgrading WebSocket connection for vault `{vault_id}`"); - Ok(ws.on_upgrade(move |socket| async move { - if let Err(error) = websocket(state, socket, vault_id).await { - debug!("WebSocket disconnected: {error}"); - } - })) + let Some(permit) = state.broadcasts.try_admit(&vault_id) else { + return Ok(( + StatusCode::TOO_MANY_REQUESTS, + "Vault connection limit reached", + ) + .into_response()); + }; + Ok(ws + .max_message_size(1024 * 1024) + .max_frame_size(1024 * 1024) + .on_upgrade(move |socket| async move { + let _permit = permit; + if let Err(error) = websocket(state, socket, vault_id).await { + debug!("WebSocket disconnected: {error}"); + } + })) } async fn websocket( @@ -48,10 +60,15 @@ async fn websocket( let authenticated = get_authenticated_handshake( &state, &vault, - receiver.next().await.transpose().unwrap_or_default(), + tokio::time::timeout(Duration::from_secs(5), receiver.next()) + .await + .map_err(|_| client_error(anyhow::anyhow!("WebSocket handshake deadline expired")))? + .transpose() + .unwrap_or_default(), )?; let device = authenticated.handshake.device_id; + let session = state.cursors.register_connection(&vault, &device).await; let mut after = authenticated .handshake .last_seen_vault_update_id @@ -69,7 +86,7 @@ async fn websocket( let batch = state.database.events_after(&vault, after).await.map_err(server_error)?; if !batch.events.is_empty() { - let head = batch.head_event_id; + let head = batch.end_event_id.unwrap_or(batch.head_event_id); send_update_over_websocket(&WebSocketServerMessage::VaultEvents(batch), &mut sender).await?; after = head; } @@ -92,7 +109,7 @@ async fn websocket( match message { WebSocketClientMessage::Handshake(_) => return Err(client_error(anyhow::anyhow!("Unexpected handshake"))), WebSocketClientMessage::CursorPositions(positions) => { - state.cursors.update_cursors(vault.clone(), authenticated.user.name.clone(), &device, positions.documents_with_cursors).await; + state.cursors.update_cursors(vault.clone(), authenticated.user.name.clone(), &device, session, positions.documents_with_cursors).await; let clients = state.cursors.get_cursors(&vault).await.into_iter().filter(|c| c.device_id != device).collect(); send_update_over_websocket(&WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }), &mut sender).await?; } @@ -109,7 +126,7 @@ async fn websocket( state .cursors - .remove_cursors_of_device(&vault, &device) + .remove_cursors_of_device(&vault, &device, session) .await; result diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index 66c5b5cb..e8d85687 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -2,7 +2,7 @@ use serde::{self, Deserialize, Serialize}; use ts_rs::TS; use crate::app_state::database::models::{ - DocumentVersion, DocumentVersionWithoutContent, FileManifest, VaultUpdateId, + DocumentVersionWithoutContent, FileManifest, VaultUpdateId, }; /// Response to a ping request. @@ -34,7 +34,7 @@ pub enum DocumentUpdateResponse { Accepted(DocumentVersionWithoutContent), /// Nothing was written. The client must incorporate this version before retrying. - StaleBase(DocumentVersion), + StaleBase(DocumentVersionWithoutContent), } #[derive(TS, Debug, Clone, Serialize, Deserialize)] diff --git a/sync-server/src/utils/portable_path.rs b/sync-server/src/utils/portable_path.rs index 7c170848..3c4bf4f8 100644 --- a/sync-server/src/utils/portable_path.rs +++ b/sync-server/src/utils/portable_path.rs @@ -16,7 +16,9 @@ pub fn path_key(path: &str) -> String { } pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { - let mut nodes: BTreeMap = BTreeMap::new(); + // Store one component per node. Expanding every prefix makes a short, + // deeply nested path consume quadratic memory and CPU before any I/O. + let mut nodes: BTreeMap<(usize, String), (usize, &str, bool)> = BTreeMap::new(); for path in entries.values() { ensure!(!path.is_empty(), "Path must not be empty"); @@ -33,6 +35,7 @@ pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { "Reserved sync directory" ); + let mut parent = 0; for (index, part) in parts.iter().enumerate() { ensure!( !part.is_empty() && *part != "." && *part != "..", @@ -73,15 +76,18 @@ pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { ); } - let prefix = parts[..=index].join("/"); let is_file = index == parts.len() - 1; + let key = (parent, path_key(part)); - if let Some((spelling, previous_is_file)) = nodes.get(&path_key(&prefix)) { - if *spelling != prefix || *previous_is_file || is_file { + if let Some((node, spelling, previous_is_file)) = nodes.get(&key) { + if spelling != part || *previous_is_file || is_file { bail!("Conflicting path: {path}"); } + parent = *node; } else { - nodes.insert(path_key(&prefix), (prefix, is_file)); + let node = nodes.len() + 1; + nodes.insert(key, (node, part, is_file)); + parent = node; } } } @@ -139,6 +145,18 @@ mod tests { } } + #[test] + fn deep_paths_do_not_expand_every_prefix() { + let path = vec!["a"; 10_000].join("/"); + let started = std::time::Instant::now(); + validate_file_manifest(&manifest(&[&path])).unwrap(); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "A 20 KB path exhausted the validation work budget: {:?}", + started.elapsed() + ); + } + #[test] fn rejects_invalid_components_characters_and_non_normalized_paths() { for path in [