diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..5b0ee4b0 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +TS_RS_EXPORT_DIR = { value = "frontend/sync-client/src/services/types", relative = true } diff --git a/.forgejo/workflows/e2e.yml b/.forgejo/workflows/e2e.yml index eb8d1e54..ce30c205 100644 --- a/.forgejo/workflows/e2e.yml +++ b/.forgejo/workflows/e2e.yml @@ -15,6 +15,8 @@ concurrency: env: RUSTFLAGS: "-Dwarnings" + E2E_ARTIFACTS: logs/e2e + E2E_WORKERS: 8 jobs: build: @@ -30,6 +32,10 @@ jobs: with: node-version: "25.x" + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -44,28 +50,12 @@ jobs: sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 - name: E2E tests - run: | - cd sync-server - cargo run config-e2e.yml --color never & - SERVER_PID=$! - cd .. - - scripts/e2e.sh 8 - EXIT_CODE=$? - - kill $SERVER_PID 2>/dev/null || true - wait $SERVER_PID 2>/dev/null || true - - exit $EXIT_CODE + run: scripts/e2e.sh - name: Upload e2e logs if: always() uses: actions/upload-artifact@v4 with: name: e2e-logs - path: logs/ + path: ${{ env.E2E_ARTIFACTS }} retention-days: 30 - - - name: Cleanup - if: always() - run: scripts/clean-up.sh diff --git a/.gitignore b/.gitignore index 967b2b65..ab97af82 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ frontend/*/dist # Rust build folders sync-server/target sync-server/artifacts -sync-server/bindings/*.ts # build folders sync-server/db.sqlite3* diff --git a/.vscode/settings.json b/.vscode/settings.json index 98187650..4296a815 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,5 +6,8 @@ "**/node_modules": true, "**/.sqlx": true, "**/target": true - } -} + }, + "cSpell.words": [ + "Syncer" + ] +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 39161e39..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,155 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project shape - -VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo: - -- `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket. -- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI. - -The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server. - -### Frontend workspaces - -- `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`. -- `obsidian-plugin` — Obsidian plugin built from `sync-client`. -- `local-client-cli` — same engine wrapped as a standalone CLI. -- `history-ui` — vault-history web UI. -- `test-client` — fuzz E2E harness (random ops across N processes). -- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server. - -## Common commands - -Pre-push hygiene (formats, lints, runs tests, requires clean git state): - -```sh -scripts/check.sh --fix -``` - -Run the fuzz E2E (N parallel processes): - -```sh -scripts/e2e.sh 12 -# Logs land in logs/log_.log. Clean with scripts/clean-up.sh -``` - -Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves): - -```sh -cd sync-server && cargo build --release && cd .. -cd frontend -npm run build -w sync-client -w deterministic-tests -node deterministic-tests/dist/cli.js # all -node deterministic-tests/dist/cli.js --filter=rename # subset -node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism -``` - -Run a single sync-client unit test by file: - -```sh -cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts' -``` - -Server: dev runs from `sync-server/` against `config-e2e.yml`: - -```sh -cd sync-server -cargo run config-e2e.yml # dev -cargo build --release # used by both e2e harnesses -cargo test # unit + ts-rs binding export tests -``` - -Frontend dev (sync-client + obsidian-plugin watch in parallel): - -```sh -cd frontend && npm install && npm run dev -``` - -Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`): - -```sh -scripts/update-api-types.sh -``` - -## SQLite / sqlx - -The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it: - -```sh -cd sync-server -sqlx database create --database-url sqlite://db.sqlite3 -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 -cargo sqlx prepare --workspace -``` - -New migrations: `sqlx migrate add --source src/app_state/database/migrations `. - -## Sync engine architecture - -Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry). - -The engine is **two independent loops with separate invariants**: - -- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement. -- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries. - -**`SyncEventQueue`** (`sync-event-queue.ts`) holds: - -- `byDocId: Map` — primary record store. -- `byLocalPath: Map` — derived index for path lookups, maintained at every mutation point. -- `events: SyncEvent[]` — pending wire ops in FIFO drain order. - -```ts -DocumentRecord = { - documentId, - parentVersionId, - remoteHash?, - remoteRelativePath, - localPath: RelativePath | undefined -} -``` - -`localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`). - -Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`. - -**Pending creates** use a `Promise` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked. - -**Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up. - -**Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise. - -## Edge-case patterns the sync engine has to survive - -The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left: - -**Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids. - -**`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server. - -**`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison. - -**Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename. - -**Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves). - -**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event. - -**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split. - -**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd. - -## Two complementary E2E harnesses - -- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy. -- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.). - -When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass. - -## Style - -- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent. -- Rust: `rustfmt.toml` enforces 4-space spaces, LF. -- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`. diff --git a/README.md b/README.md index 74c6ee97..f13c18a4 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ scripts/bump-version.sh patch #### Run E2E tests ```sh -scripts/e2e.sh 8 +E2E_WORKERS=8 scripts/e2e.sh ``` And to clean up the logs & database files, run `scripts/clean-up.sh` diff --git a/SYNC_ENGINE_HANDOVER.md b/SYNC_ENGINE_HANDOVER.md new file mode 100644 index 00000000..9b5a6372 --- /dev/null +++ b/SYNC_ENGINE_HANDOVER.md @@ -0,0 +1,652 @@ +> Historical handover: its client journaling, recovery-archive and filesystem +> durability descriptions are superseded by the direct-application model in +> [the sync client README](frontend/sync-client/README.md). The current client +> saves complete metadata and reconciles current disk contents after interruption. + +# Sync engine v4 handover + +Prepared on 2026-09-19. This records the implementation delivered in the preceding +work session: its starting point, intended behavior, changes and their reasons, +validation, and integration work still required. + +Follow-up on 2026-09-19: file modification times have since been removed from the +server schema/protocol, client snapshots, and recovery records. Concurrent binary +or unmergeable edits now keep the server version, with displaced local bytes +retained in recovery; changes on only one side still keep that side. References +to `mtime` below describe the earlier implementation and are superseded by this +change. See the server and client READMEs for the current contracts. + +Further follow-up on 2026-09-19: request tracking is now part of `events`, using +unique `request_id` and `request_fingerprint` columns. The separate +`request_receipts` table and duplicated response JSON have been removed. Retries +check the fingerprint and reconstruct the original `Accepted` response from the +stored event before CAS. References to separate receipts below describe the +earlier implementation and are superseded by this change. + +File manifest storage follow-up on 2026-09-19: complete maps are now stored as rows in +`file_manifest_entries`, keyed by `(file_manifest_id, document_id)`, with exact path +uniqueness per version. `file_manifests` retains a header for every accepted version, +including empty maps. File manifest reads reconstruct the map from these rows, and +vault snapshots join them to the latest content versions. Event rows contain only +ordering and request-deduplication metadata; replay and retry acknowledgements +reconstruct events from normalized document/manifest rows. References below to JSON file +manifest storage describe the earlier implementation. + +The baseline is commit `1619c51` (`Migrate server to CAS only`). The implementation +changes are currently in the working tree/index, rather than a new implementation +commit. The file inventory below describes the rewrite relative to that baseline. + +The following changes must not be attributed to this implementation: + +- While this handover was being written, additional unstaged edits appeared in + `sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql`, + `sync-server/src/server/endpoints/get_file_manifest.rs`, + `sync-server/src/server/endpoints/put_file_manifest.rs`, + `sync-server/src/server/endpoints/vault_snapshot.rs`, + `sync-server/src/server/endpoints/put_file_content.rs`, and + `sync-server/src/server/endpoints/put_file_content/tests.rs`. They were left alone + and are outside this handover's implementation inventory. The earlier passing + validation applies to the rewrite as delivered, represented by the staged + implementation at the start of this handover, not subsequent edits. Review and + validate the final SQL before starting a new database. + +This handover task added only this document; it did not modify implementation +files or change staging decisions. + +## Initial state + +The repository was in the middle of a protocol migration. It was not a functioning +v4 implementation with a few missing routes. + +| Area | Starting behavior | Why it needed to change | +| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Server protocol | Advertised API v3. `PUT /documents/:id` already used a parent version, request UUID, immutable versions, SHA-256 request fingerprints, and durable acknowledgements. Concurrent text merging had already been removed from the server. | Preserve the existing content CAS design and extend its durability model to namespace changes. This rewrite did not originate server-side CAS or idempotency. | +| Content and names | A document version also contained `relative_path` and `is_deleted`; pushes carried a path and supported `Delete`. The server sanitized and allocated available paths. | Content history and namespace decisions were coupled. Whole rename sequences and cross-document conflicts could not be decided atomically as a file manifest. | +| Server persistence | `documents`, a latest-version view, and `push_acknowledgements`; new IDs were allocated from the maximum document update ID inside a write transaction. | There was no common durable event stream for content and file manifests, or file manifest history. | +| Broadcasts | A completed write committed and then broadcast its document metadata. The originating device was skipped. Reconnection fetched latest document heads rather than every event. | Post-commit task scheduling could change notification order; a crash after commit could lose a wakeup. Latest heads are not an ordered history of namespace transitions. | +| Client protocol | Advertised API v2. It still used the earlier create/text/binary/delete service calls and expected `FastForwardUpdate` / `MergingUpdate`. | The client had not caught up with the server's previous CAS migration, let alone the proposed file manifest protocol. | +| Client scheduling | `Syncer`, `UnrestrictedSyncer`, a configurable concurrent queue, document locks, mutable records, and in-memory promises coordinated operations. | Whole-file-manifest changes need a consistent view across documents. A single durable reconciliation loop is easier to reason about when performance is secondary. | +| Client metadata | Persisted document metadata and an update watermark, with mutable runtime records and gap tracking. | No persisted exact request/response workflow or filesystem application journal tied disk changes to metadata advancement. | +| Filesystem | Read/write/rename/delete operations, an atomic text updater, an in-memory safety wrapper, and native line-ending conversion. | The interface did not promise durable data and directory updates or crash-recoverable multi-file moves. | +| Content comparison | A 32-bit rolling checksum. | Collisions could incorrectly hide edits or identify unrelated files as a rename. | +| Reset/settings | Reset cleared local tracking; settings could cause a vault reset. Persistence had no explicit power-loss contract. | Clearing uncertain requests or application state could lose recovery information or apply it to the wrong vault. | + +The Obsidian plugin, CLI client, legacy test fixtures, and their adapters were +explicitly outside the implementation scope. + +## Expected result and selected policies + +The requested result is a simple server CAS store and a client that owns all +reconciliation. The content algorithm follows the fetch/merge/push model in +[Conflict-Free Three-Way Text Merging](https://schmelczer.dev/articles/reconcile-text-3-way-merge/), +including recording the sent snapshot after acceptance and retrying an uncertain +request with its original idempotency key. + +The implemented invariants are: + +1. Document UUIDs identify content; a file manifest maps UUIDs to paths and owns + membership. A content push cannot rename, delete, or restore membership. +2. File manifest acceptance replaces the entire map only when its parent is current. + Every referenced document must already have content. +3. An accepted content or file manifest mutation commits its state, event, and receipt + together. Each vault has one total order; separate vaults have separate orders. +4. Each WebSocket sends persistent events in that order, including the sender's + own events. Cursors remain ephemeral and do not consume event IDs. +5. The client persists its exact request before sending. An unknown outcome is + resolved before advancing to newer remote state. +6. The client advances a merge base to what was actually incorporated or accepted, + never to an assumption about the current live file. +7. Filesystem application has a durable journal. Every affected source is staged + before destinations are installed, so swaps and cycles do not overwrite files. +8. A received event watermark becomes durable with the corresponding state or + recoverable application obligation. Excluded downloads remain explicit work. + +The conflict policies implemented alongside these invariants are: + +| Situation | Decision | Reason | +| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Same UUID, local path unchanged | Use the server path, including absence. | Incorporate remote changes. | +| Same UUID, remote path unchanged | Use the local path, including absence. | Preserve local intent. | +| Same UUID, both paths changed | Use the server path, including absence. | The requested deterministic tie-breaker. | +| Different UUIDs claim conflicting paths | Keep both identities. A selected path matching the server wins priority; UUID order resolves other ties. Give displaced documents deterministic conflict names. | A per-ID merge alone cannot guarantee one-to-one paths. | +| Delete versus content edit | Membership follows the file manifest merge. Preserve locally displaced bytes in recovery; do not automatically recreate membership. | Avoid deletion/resurrection loops while retaining observed data. | +| First sync, existing local text at a remote path | Adopt the remote UUID and merge against an empty parent. | There is no known common ancestor; neither existing side should silently overwrite the other. | +| Concurrent binary/unmergeable changes | Larger originating mtime wins; server wins ties. | A defined policy is necessary when text reconciliation does not apply. Clock skew remains a limitation. | +| Rename notification | Preserve identity through the ordered logical move hints. | Retain identity through online rename sequences. | +| Unobserved/offline rename | Infer only from a unique, nonempty content hash match on both sides; otherwise treat as delete/create. | Identical files and rename-plus-edit sequences cannot be identified reliably from snapshots alone. | +| Deliberate notified delete/recreate | Allocate a new UUID, even at the same path with identical bytes. | A new document must not inherit the deleted document's identity accidentally. | +| Portable-name conflict | Repair names client-side; reject invalid canonical file manifests server-side. | Every client must receive one valid, unambiguous namespace. | +| Ignored, oversized, or unmaterialized remote file | Preserve membership and remember its remote head. | Skipping a download must not publish a deletion. | + +This is not a proof of a universally perfect sync engine. Power-loss recovery +depends on the adapter/storage contracts below. Snapshot-based observation cannot +recover writes that were never observed or unambiguously reconstruct offline +identity. Automatic text merging also cannot guarantee semantic correctness of +the resulting prose or code. + +## Server changes and rationale + +### Independent namespace and content CAS + +`PutFileContent` now contains `requestId`, nullable `parentVersionId`, `mtime`, and +`content`. `content` accepts `Snapshot` or `Diff`. Paths, delete payloads, and other +unknown fields are rejected. The server still reconstructs a transport diff against +the supplied parent; it does not reconcile competing edits. The existing guard +against the minimum signed integer in diff lengths was retained. + +`mtime` is the originating file time in milliseconds, stored as a finite floating +point value within the JavaScript date range. It is distinct from the server's +commit time. This lets clients apply the selected binary policy without treating +download time as a new edit. + +`PushFileManifest` contains `requestId`, `parentFileManifestId`, and `entries`, where entries +are the complete UUID-to-path map. The empty initial file manifest has ID 0. A custom +map deserializer rejects repeated UUID keys, including alternate spellings of the +same UUID; an ordinary JSON map decoder would silently keep one value. A sorted +map gives request fingerprinting a stable key order. + +Both mutation kinds return `Accepted` or `StaleBase`. A stale content response +contains the current bytes; a stale file manifest response contains the current map. +Only accepted mutations get durable receipts. Rejected requests have no state +effect and may receive a newer stale base if retried later. + +### Schema and transaction boundary + +The bootstrap schema was replaced with a fresh v4 schema: + +| Object | Purpose and reason | +| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `events` | AUTOINCREMENT IDs, request IDs, and fingerprints provide the common order and durable idempotency namespace. IDs are constrained to JavaScript's safe integer range. Event payloads are reconstructed from normalized rows. | +| `documents` | Immutable content versions reference their creating event; no path or deletion fields. UUIDs use the BLOB representation expected by SQLx's UUID binding. `mtime`, author, device, and server timestamp are retained. | +| `documents_by_id` and `latest_document_versions` | Find a UUID's current content without namespace lookups. | +| `file_manifests` and `file_manifest_entries` | Immutable manifest headers and normalized UUID/path rows keyed by their creating event ID. A file manifest version need not equal the latest event ID because content events can follow it. | + +The write transaction performs event/request lookup before CAS, then head validation, +event allocation, normalized state insertion, and commit. Request lookup must come +first: a retry of an accepted request is still successful after another +client advances the head. Reusing a request UUID with different content, parent, +document ID, or operation type is rejected. Session/device identity is excluded +from the fingerprint so a restarted client can recover its old outcome. + +Read operations were consolidated in `queries.rs`, and write operations in +`mutations.rs`. Runtime SQL queries replace the old schema-dependent SQLx macro +queries in this path. This keeps the new schema and transaction logic together +without requiring the old prepared-query cache to describe v4. + +SQLite now explicitly uses WAL, `synchronous=FULL`, `fullfsync`, and foreign keys. +On Unix, database creation also flushes the containing directory and ancestors. +The reason is durability of both transaction contents and newly created directory +entries. These settings still depend on the platform/filesystem honoring flushes. + +No migration or automatic deletion of old databases was added. Replacing the +bootstrap migration deliberately makes old migration checksums incompatible. +Use a fresh database directory and retain old data separately. + +### Endpoint and replay changes + +Paths below are relative to `/vaults/:vault_id`: + +| Endpoint | Change and rationale | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PUT /documents/:document_id` | Existing route becomes a thin content-only CAS handler. Database mutation code owns the complete atomic operation. | +| `GET /documents/:document_id` | Retained; returns current content using the new metadata shape. | +| `GET /documents/:document_id/versions/:version_id/content` | Retained; clients fetch immutable bases for merging. | +| `GET /file-manifest` | New; retrieves canonical membership and names. | +| `PUT /file-manifest` | New; validates and CAS-replaces the complete namespace. | +| `GET /vault-snapshot` | New; returns the file manifest, referenced content heads, and event watermark from one read transaction. Prevents a bootstrap gap between separately read state and history. | +| `GET /events-since?after=N` | New; returns every later event in increasing order with the transaction's watermark. Negative/ahead-of-history cursors are rejected rather than silently resetting a client. | +| `GET /ws` | Reworked to send the same ordered durable event history. | +| `GET /documents` | Removed; latest document rows no longer describe membership or preserve namespace transitions. | + +WebSocket messages use `vaultEvents` batches containing `events` and +`headEventId`. The handshake retains `lastSeenVaultUpdateId`. One sender per +connection drains the database in order; broadcasts are only wake hints. A +two-second timer also drains the log, covering lost wakes and commit-before-notify +crashes. Broadcast lag triggers another drain rather than loss of persistent +events. HTTP replay is available independently of the socket. + +Cursor broadcasts were adapted to the new notification wrapper but remain +ephemeral. Persistent events no longer filter out their originating device: +filtering them would create holes in that client's event sequence. + +### Portable file manifest validation + +Validation now checks the whole namespace, not just individual strings: + +- Relative `/`-separated paths and NFC normalization, without a universal + total-path byte limit. +- No empty, `.` or `..` components; no backslashes, controls, Windows-forbidden + characters, trailing dots/spaces, or reserved device names. The device-name + checks also cover extension forms, trailing spaces in the stem, and the + superscript COM/LPT digit variants. +- `.vault-link-sync` and case/normalization aliases are reserved. +- Comparison uses NFC → Unicode uppercase → NFC. +- No duplicate/alias paths, file-versus-ancestor conflicts, or inconsistent + directory spelling such as `Notes/a.md` alongside `notes/b.md`. +- Every UUID must refer to existing content before membership is accepted. + +The server rejects invalid input instead of allocating or sanitizing names. This +keeps the canonical update exactly equal to the file manifest whose CAS the client +requested. Client and server implement the same portable-name policy. + +## Client changes and rationale + +### Persistent state and the three file manifests + +The three conceptual file manifests are represented as follows: + +| Concept | Representation | Why | +| --------------------------- | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Incorporated server base, B | `state.fileManifest` with its version ID | A received map becomes the base only after its application is durable or journaled for recovery. | +| Last submitted file manifest, S | `state.pending.request.entries` while a file manifest request is unresolved | Freeze the exact request and parent across network failures and restarts. On acceptance, this exact S becomes B. | +| Current local view, L | `state.local`, refreshed by complete scans and logical notifications | Preserve changes made while an earlier S was in flight. It also tracks unmaterialized/excluded remote membership; it is more than a raw listing of existing disk files. | + +S is not kept as a redundant permanent third field after its outcome is resolved. +On acceptance it becomes the incorporated base. On rejection the client merges B, +the current L, and the returned remote map, then prepares a new request with a new +UUID. This preserves the three-way semantics without maintaining duplicate state. + +The new format also stores per-document incorporated content metadata and hash, +observed hash, materialization/bootstrap flags, `lastSeenUpdateId`, deferred +`remoteHeads`, the initial vault snapshot while bootstrap is incomplete, pending +requests/responses, and a filesystem application journal. + +`Database.commit` clones and saves the complete value before replacing in-memory +state. A save is marked uncertain before it is attempted. If it fails, the next +attempt reloads durable state before doing more work; a failed save may have +persisted either complete version. This avoids continuing from an assumed outcome. + +State is bound to the remote URI and vault name. Incompatible formats and live +vault rebinding are rejected instead of silently clearing recovery information. +A truly empty, uninitialized state can be rebound for initial configuration. + +### One reconciliation loop + +The configurable queue and `UnrestrictedSyncer` were replaced by one serialized +loop per vault. This intentionally trades throughput for a smaller set of states +that can overlap. Its normal progression is: + +1. Reload metadata after any uncertain save and recover any filesystem journal. + Recovery runs before a network ping, so an offline server does not block it. +2. Initialize server configuration; if necessary, persist and apply a consistent + bootstrap vault snapshot. +3. Resolve the exact pending request and persist its response before processing + newer remote state. +4. Scan local files, applying ordered logical create/delete/move hints. +5. Replay HTTP events from the durable cursor. Require contiguous IDs and the + advertised watermark; do not silently skip a missing event. +6. Apply file-manifest/content changes, retaining remote heads for deferred downloads. +7. Rescan, retry newly eligible deferred content, and prepare one next content or + file manifest submission. +8. Repeat while there is work; otherwise wake on notifications or a five-second + timer. Transient failures retry at the configured network interval. + +WebSocket batches wake this loop. The client uses HTTP replay as its authoritative +event input, avoiding separate merge paths for HTTP and socket delivery. A dirty +flag and completion handling preserve wakes arriving while an attempt finishes. +`waitUntilFinished()` follows successive active attempts and reports failures. + +The exact content snapshot, mtime, hash, request ID, parent, and optional cursors +are persisted before sending. After acceptance, the base hash comes from those +sent bytes, not a later disk read. A lost response therefore cannot cause an edit +to be merged into itself. Stale responses cause a client-side merge, and rejected +submission bytes are retained in recovery before the request is cleared. + +New UUIDs are saved before network use. Initial content is accepted before its +UUID is published in the file manifest. A crash between those operations can leave +unreferenced content, which is retained and harmless to the visible namespace. +It does not leave a file manifest pointing to absent content. + +### Scanning, identities, and conflicts + +Scans must complete or fail. A disappearing file during a scan is retried rather +than interpreted from a partial listing. Ignored paths, files above the configured +size limit, and files not yet materialized are handled explicitly so an unavailable +local copy does not masquerade as deletion. + +Known online moves are applied in notification order, including directory-prefix +moves. Logical delete/create notifications distinguish replacement from editing. +Offline inference requires exactly one missing identity and one unknown path with +the same nonempty SHA-256 hash. Initial same-path adoption uses portable path +comparison. On normalization-insensitive filesystems, an accessible known NFC +spelling is retained when a scan reports its decomposed alias. + +A local generation counter invalidates a prepared merge or push if a notification +arrives during scanning, remote fetches, or planning. The loop rescans rather than +applying the stale plan. This addresses, for example, moving a file and creating a +different file at its old path while remote content is being fetched. + +`file-manifest.ts` first performs the requested per-UUID three-way merge, then allocates +valid paths. Server-selected occupants get priority; remaining ties use UUID +ordering. Conflict suffixes contain the UUID, including file/ancestor conflicts. +Invalid components are repaired, but long paths are not silently flattened into +UUID filenames. Comparing a candidate with each +occupied path individually avoids an allocation loop when the temporary occupied +set itself contains overlapping desired and actual paths. + +### Content reconciliation + +`content.ts` centralizes snapshot serialization and merging. It compares exact +bytes for unchanged-side fast paths, fetches immutable bases when needed, and +calls `reconcile` for valid UTF-8 content with a mergeable extension. Unknown +first-contact text uses an empty parent. Binary or otherwise unmergeable concurrent +content follows originating mtime, with the server winning ties. + +Optional local cursor positions are transformed through text reconciliation and +passed to the adapter with the output. Text merge output uses the larger input +mtime. UTF-8 BOM is decoded as an actual character during merging, and native +line-ending conversion was removed. Existing bytes and CRLF are not rewritten +merely because another platform downloaded the file. + +The old rolling checksum was replaced by asynchronous Web Crypto SHA-256. Cursor +tracking was updated to await hashes, find remote cursors by document UUID, and +resolve their display path through the local file manifest. Missing/unreadable local +files mark ephemeral cursor data as prior instead of breaking synchronization. + +### Filesystem transaction and crash recovery + +`FileOperations` now bridges the metadata store and the filesystem with a persisted +application journal. It does not assume these two stores share a transaction. + +Each application gets a unique transaction directory under +`.vault-link-sync/transactions`. Each affected document has a source staging path, +an output path, expected/replacement snapshots, source/destination names, and phase: +`planned`, `staged`, `prepared`, `installing`, or `installed`. + +The application sequence and reasons are: + +| Action | Why it is needed | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Persist the complete plan before visible mutations | Restart must know what it was trying to do. The plan includes mergeable extensions so recovery is independent of server availability. | +| Move all affected sources into staging before installing any destination | Rename swaps/cycles and paths becoming ancestor directories cannot overwrite another source. Content replacement and deletion use the same retained-source machinery. | +| Detect an existing staging slot on recovery | It proves that the source move happened even if the subsequent metadata save failed. | +| Reconcile a staged source that changed after planning | Preserve observed external edits made between the expected snapshot and staging. | +| Persist the prepared result before creating its output file | Restart must not merge an already merged result a second time. | +| Persist `installing` before the atomic rename | If its origin is subsequently absent, recovery can recognize that installation completed even if its acknowledgement/save was lost. | +| Require exclusive creation and no-replace renames | A concurrent destination must cause a retry/conflict, not be silently overwritten. | +| Stage an unexpected file occupant as another UUID with a conflict path | Preserve an unrelated file that appeared at the destination or blocked an ancestor. | +| Preserve a nonempty blocking directory and rename the incoming document | Unmanaged descendants must not be recursively deleted to make space. | +| Prune only empty implicit directories | Directories are not synchronized objects; files must not be erased by cleanup. | +| Remove staged/output artifacts, then commit the planned next state | Completed applications do not become permanent byte/state archives; interrupted cleanup is retried while the active journal still exists. | + +An external deletion observed before staging prevents automatic resurrection. +Recovery artifact creation is idempotent: an existing artifact is checked for the +expected bytes instead of overwritten. + +Rejected request snapshots are stored separately under +`.vault-link-sync/requests`, with request IDs and identifying JSON. No recovery +artifact or orphan-content garbage collector was added. Ordinary transaction +artifacts are removed on completion; rejected-request snapshots remain explicit +recovery material. + +### Lifecycle, settings, transport, and compatibility + +`SyncClient.create` requires strong adapters and loads persisted state. Destruction +stops work and flushes local changes. Settings and engine saves serialize complete +replacements through the same persistence store so one cannot overwrite newer +data from the other. The client does not coordinate separate processes accessing +the same local vault. + +`reset()` restarts transports/tracking; it no longer clears pending requests or +application journals. Changing a live vault requires a separate state store and +client. Settings saves are assumed to succeed; settings are saved before becoming +visible to listeners, and getters copy the settings and ignore-pattern array to +prevent accidental in-place mutation. + +Public notification methods enqueue work instead of waiting for convergence. +Call `waitUntilFinished()` to await the active attempt. Sync status now checks the +requested path's manifest, content hash, exclusions, pending request, and active +application state. Remaining-operation reporting is a busy indicator rather than +the old concurrent queue length. Successful local submissions and remote filesystem +applications create history entries; the history UI is not a durable audit log. + +The service layer now exposes typed vault-snapshot/event/file-manifest/content operations. +Durable retry ownership moved out of its old `retryForever` request builders and +into the loop. Requests have a 30-second abort timeout; authentication errors and +permanent 4xx responses are distinguished from transient failures. The proper +`Device-Id` header is used. A rejected initial ping is no longer cached forever, +so startup while offline can recover on retry. + +The existing optional telemetry wiring was retained. Its build-injected version +constant now has a development fallback so importing source outside webpack does +not throw. No new telemetry service was introduced. + +The unused `syncConcurrency`, `diffCacheSizeMB`, and `minimumSaveIntervalMs` +settings and the old content cache/queue scaffolding were removed. +`nativeLineEndings` remains accepted by the creation type but no longer transforms +content. These compatibility surfaces should be reviewed when adapters/UI are +updated, rather than assumed to retain their former effects. + +## Required adapter contract and remaining integration + +No production Obsidian or CLI adapter was upgraded in this work. Their old +interfaces are incompatible and will require implementation changes before they +can use the v4 client. + +| Adapter responsibility | Required guarantee | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Listing and reading | Complete enumeration and coherent bytes/mtime snapshots; absence must be distinguishable from failure. Read races must reject instead of returning partial snapshots. | +| Path access | Stay inside the vault; reject symlinks, ancestor symlink traversal, hardlinks, and special files. Staging and visible files must share a filesystem. | +| `write` | Atomically create a complete file exclusively. A crash leaves the path absent or complete, never partially written. Preserve supplied mtime and supported cursor metadata. | +| `rename` | Atomic same-filesystem move without replacing an existing destination. Flush source data and both affected directories before resolving. | +| Directory operations | Durable recursive creation, including ancestors. Pruning uses only directory removal and never unlinks descendant files. | +| Metadata persistence | Atomically replace the complete state and flush data/directory entries before resolving. A failed save may leave either complete value, which must be reloadable. | +| Notifications | Report logical external operations; suppress self-generated create/delete/move notifications. Treat an editor's atomic save as a content update rather than a deliberate document delete/recreate. | +| State placement | Keep metadata outside the user-file namespace or in the reserved internal directory. | + +Known boundaries and deferred work: + +- No automatic migration of existing server databases or v2/v3 client stores. +- No guarantee for writes that occur and disappear between observations, or + perfect identity reconstruction while notifications are unavailable. Logical + notification hints themselves are not an external filesystem operation log. +- No multi-file atomic visibility to arbitrary external programs: staging is + recoverable, but an external reader can see a transaction partway through. +- No hardware power-cut testing or production-adapter durability certification. +- No bounded history/recovery storage, event pagination, large-vault performance + optimization, or guarantee against starvation under continuous competing writes. +- No protection against an administrator independently deleting server history, + pending requests, or recovery state. An ahead-of-history cursor fails rather + than silently accepting a replaced/truncated server database. +- No syncing of empty directories or special filesystem objects. +- No new migration of the history viewer, plugin, CLI, repository-wide architecture + docs, or legacy testing setup. They may still describe the earlier protocol. + +The next integration work is to implement and verify real adapters, update the +excluded callers to v4 and the new notification semantics, and port the behavior +checks into maintained tests. Fresh-state rollout must preserve old data and use +compatible server/client versions together. + +## Validation performed + +These results were obtained at the end of the implementation task, before the +subsequent edits noted above. They are not a claim that every current +working-tree change or every legacy test passes. + +| Check | Result and scope | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `cargo build --manifest-path sync-server/Cargo.toml --quiet` | Passed for the implemented server. | +| `cargo test --manifest-path sync-server/Cargo.toml export_bindings --quiet` | Passed 15 binding-export tests; used to generate the Rust wire types. This is not a behavioral server-suite result. | +| Strict TypeScript check of the production client entrypoint | Passed. Command below. | +| `npm run build -w sync-client` from `frontend` | Passed for both browser and Node production bundles. | +| `git diff --check` | Passed for the implementation diff. | +| Live server CAS/order checks | Passed with an isolated local server and fresh temporary databases. | +| Client reconciliation and interruption checks | Passed against that server using a memory filesystem/persistence harness with injected failures. | + +The TypeScript command was: + +```sh +node frontend/node_modules/typescript/bin/tsc --noEmit --strict \ + --target ESNext --module ESNext --moduleResolution bundler \ + --allowSyntheticDefaultImports --skipLibCheck --types node \ + --typeRoots frontend/node_modules/@types frontend/sync-client/src/index.ts +``` + +Behavior exercised: + +| Scenario | Observed result | +| --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 24 concurrent content creations, then 12 competing file manifest CAS submissions | Exactly one file manifest winner; committed event IDs remained contiguous. | +| WebSocket origin delivery and reconnect replay | Events arrived in increasing order, including events from the connecting device. | +| Duplicate JSON UUID keys and invalid portable file manifests | Rejected without creating events. Cases included duplicate/alias paths, ancestor conflicts, traversal, reserved names, and directory-spelling conflicts. | +| Content and file manifest receipt retry after a server restart and later heads | Returned the original acknowledgement without an additional event. | +| Creation and download; online rename sequence and remote swap | Identities and bytes converged correctly. | +| Concurrent text changes; first-sync same-path text | Both sides' text changes were retained in the checked examples. | +| Concurrent same-path creations | Both UUIDs survived with distinct paths. | +| Lost content acknowledgement followed by client restart | The original request was recovered, with only one accepted content event. | +| Lost file manifest acknowledgement followed by client restart | The original file manifest outcome was recovered without a duplicate mutation. | +| 22 injected filesystem interruption positions during staged swaps | Recovery produced the expected files without an unfinished application journal. | +| 18 injected metadata-save interruption positions during content replacement | Recovery installed the result without merging it a second time. | +| Unexpected destination file and blocking ancestor file | Incoming and unexpected content were both preserved. | +| Binary conflict and remote delete versus unsent local edit | Larger originating mtime won; downloaded mtime was retained; deleted edited bytes remained in recovery. | +| Ignored remote file, then re-enabled syncing | Membership was preserved and the file downloaded once eligible. | +| Save persisted and then reported failure | Reloading durable state allowed retry without restarting the process or duplicating a content mutation. | +| Initial ping offline, then network recovery | The next attempt progressed rather than reusing a permanently rejected ping promise. | +| Move and new file at the old path during a remote fetch | The moved document received the merge and the independent new file remained separate. | +| Notified delete/recreate with identical bytes at the same path | A new UUID was allocated. | +| UTF-8 BOM and CRLF during text merging | Both were preserved in the checked merge. | + +The temporary harnesses were `/tmp/vault-link-engine-check.ts` and +`/tmp/vault-link-server-check.mjs`. They were deliberately not added to the +repository's testing setup. They used a temporary local server on port 18749; +that server was stopped after validation. Their absolute imports and temporary +configuration are development artifacts, not a portable test runner. Additional +server-restart receipt checks were performed during the session. + +The 40 interruption positions are simulated failure points in adapters with the +required atomic semantics. They do not establish durability of real filesystem +adapters, all operating systems, all external-write schedules, or physical storage. + +In the delivered rewrite, the old `put_file_content/tests.rs` file was not ported, +and the rewritten handler no longer included its old test module. Subsequent +edits to those files during this handover are separate work. Removed old helper +modules also contained +legacy unit tests. Client fixtures still target the earlier interfaces. The +webpack loader now checks files reachable from production entrypoints so those +unmigrated fixtures do not block building the library. Full legacy-suite and +repository-wide lint success were not claimed. + +## Complete file inventory + +Every implementation file added, changed, or removed relative to the baseline is +listed below, including regenerated type files and incidental cleanup. Deletions +are listed separately from additions even where Git's similarity detection calls +them renames. Generated binding formatting changes have no additional runtime +meaning unless stated. + +This inventory accounts for **85 implementation paths**. It excludes the preexisting `CLAUDE.md` deletion and includes the old/new sides of Git-detected renames separately. + +### Server + +| File | Change | What changed and why | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [sync-server/Cargo.toml](sync-server/Cargo.toml) | Changed | Declare `sha2` and `unicode-normalization`, remove `sanitize-filename`, and enable ts-rs `no-serde-warnings`. The new mutation code needs request fingerprints and portable-name normalization; server-side filename repair was removed. The preexisting push code already referred to SHA-256; the direct dependency is now declared. | +| [sync-server/Cargo.lock](sync-server/Cargo.lock) | Changed | Record the matching dependency graph: direct SHA-256/normalization dependencies and removal of the filename-sanitizer package. Keeps builds reproducible for the changed file manifest. | +| [sync-server/README.md](sync-server/README.md) | Changed | Replace v3 push/reset instructions with v4 requests, routes, portable-path rules, ordering, recovery, and fresh-state requirements. Documents the actual breaking protocol without suggesting destructive reset commands. | +| [sync-server/src/consts.rs](sync-server/src/consts.rs) | Changed | Raise the supported API from 3 to 4 so incompatible clients fail the version check. | +| [sync-server/src/app_state/database.rs](sync-server/src/app_state/database.rs) | Changed | Wire the consolidated read/write modules, use SQLx `begin_with("BEGIN IMMEDIATE")`, and add explicit SQLite and directory-flush durability settings. Keep commit ordering and persistence in the database boundary. | +| [sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql](sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql) | Changed | Replace the bootstrap schema with events, content-only versions, file manifests, and complete request receipts. Separate membership from bytes and atomically order both. Requires fresh databases; later local edits to this file are outside the earlier validation. | +| [sync-server/src/app_state/database/models.rs](sync-server/src/app_state/database/models.rs) | Changed | Remove path/deletion from content models; add originating mtime, SQLx row decoding, file manifest, event, batch, and vault snapshot types. Flatten wire metadata and use JS-number-compatible exported IDs to express the v4 protocol. | +| [sync-server/src/app_state/database/mutations.rs](sync-server/src/app_state/database/mutations.rs) | Added | Add both CAS transactions, request fingerprint/receipt handling, validation, event allocation, immutable insertions, and post-commit wakeups. Ensures state, order, and retry outcomes commit together. | +| [sync-server/src/app_state/database/queries.rs](sync-server/src/app_state/database/queries.rs) | Added | Add/consolidate current and immutable content reads, current file manifest, consistent bootstrap vault snapshot, and full ordered event replay. Readers no longer infer membership or event history from latest document rows. | +| `sync-server/src/app_state/database/get_document_version.rs` | Removed | Remove the separate old-schema query; immutable content reads are retained in `queries.rs` with the new row shape. | +| `sync-server/src/app_state/database/get_latest_document.rs` | Removed | Remove the separate old-schema query; current-head reads are retained in `queries.rs` and can share the CAS transaction. | +| `sync-server/src/app_state/database/get_latest_document_by_path.rs` | Removed | Remove path-based document lookup because content versions no longer own names. | +| `sync-server/src/app_state/database/get_latest_documents.rs` | Removed | Remove the latest-document listing helper; file manifest membership and a consistent `/vault-snapshot` now define the visible vault. | +| `sync-server/src/app_state/database/get_latest_documents_since.rs` | Removed | Remove head-only catch-up; replay every durable event so intervening namespace changes are not omitted. | +| `sync-server/src/app_state/database/get_max_update_id_in_vault.rs` | Removed | Remove document-table maximum-ID allocation; the common event table allocates IDs for both mutation kinds. | +| `sync-server/src/app_state/database/get_push_acknowledgement.rs` | Removed | Remove the content-specific acknowledgement lookup; shared request receipts now return the original response for either CAS kind. | +| `sync-server/src/app_state/database/insert_document_version.rs` | Removed | Remove the old coupled content/path insertion and direct broadcast. `mutations.rs` now commits state/event/receipt and publishes only a wake hint. | +| [sync-server/src/app_state/websocket/broadcasts.rs](sync-server/src/app_state/websocket/broadcasts.rs) | Changed | Replace wire-event broadcasts with `VaultUpdate` and ephemeral cursor notifications; make channel capacity at least one. Database replay carries correctness when receivers lag or no subscriber exists. | +| [sync-server/src/app_state/websocket/models.rs](sync-server/src/app_state/websocket/models.rs) | Changed | Replace `VaultUpdate` with `VaultEvents(EventBatch)` and remove the origin-filtering wrapper. Durable event batches can represent both mutations without holes for the originating device. | +| [sync-server/src/app_state/websocket/utils.rs](sync-server/src/app_state/websocket/utils.rs) | Changed | Remove `get_unseen_documents`; retain handshake authentication and message sending. Catch-up belongs to the durable event query instead of latest-head selection. | +| [sync-server/src/app_state/cursors.rs](sync-server/src/app_state/cursors.rs) | Changed | Send cursor expiry/update broadcasts through the new ephemeral notification type. Preserve presence behavior independently of persistent event sequencing. | +| [sync-server/src/server.rs](sync-server/src/server.rs) | Changed | Expose request/response modules within the crate for database mutations, add file-manifest/vault-snapshot/events routes, and remove the collection document-list route. Route ownership now reflects the two independent CAS resources. | +| [sync-server/src/server/endpoints.rs](sync-server/src/server/endpoints.rs) | Changed | Register the new file manifest endpoint module and remove the old latest-document-list module so the compiled route set matches v4. | +| `sync-server/src/server/endpoints/get_file_manifest.rs`, `put_file_manifest.rs`, `vault_snapshot.rs`, and `events.rs` | Added | Implement file manifest read/CAS, a consistent vault snapshot, and event replay with cursor checks. Supplies the namespace and recovery endpoints. | +| [sync-server/src/server/endpoints/put_file_content.rs](sync-server/src/server/endpoints/put_file_content.rs) | Changed | Reduce the handler to extraction/auth-context forwarding and content CAS delegation. Remove path/delete decisions and the old v3 test-module inclusion; tests need porting separately. | +| `sync-server/src/server/endpoints/fetch_latest_documents.rs` | Removed | Remove the old collection endpoint; it cannot express file manifest membership plus a consistent event watermark. | +| [sync-server/src/server/endpoints/websocket.rs](sync-server/src/server/endpoints/websocket.rs) | Changed | Use one ordered sender draining database history, replay from the handshake cursor, poll every two seconds, tolerate lag, and process ephemeral cursors in the same connection loop. Notification timing no longer determines event order. | +| [sync-server/src/server/requests.rs](sync-server/src/server/requests.rs) | Changed | Make pushes content-only, add `PushFileManifest`, deny unknown request fields, and reject duplicate parsed UUID keys. Prevent silent interpretation of legacy payloads or ambiguous maps. | +| [sync-server/src/server/responses.rs](sync-server/src/server/responses.rs) | Changed | Add file manifest `Accepted`/`StaleBase` responses and deserialize support for stored acknowledgements. Enables exact receipt replay for both mutation kinds. | +| [sync-server/src/utils/portable_path.rs](sync-server/src/utils/portable_path.rs) | Added | Add namespace-wide portable-path validation and shared comparison conventions. Reject aliases, reserved names, duplicate destinations, and file/ancestor conflicts before canonical acceptance. | +| [sync-server/src/utils.rs](sync-server/src/utils.rs) | Changed | Register portable validation and remove old sanitizer/path-allocation module exports. Keep naming policy on the client. | +| `sync-server/src/utils/dedup_paths.rs` | Removed | Remove the old path-deduplication helper and its embedded legacy tests. Canonical conflicts now require a valid client-submitted file manifest, not a server-generated alternate view. | +| `sync-server/src/utils/find_first_available_path.rs` | Removed | Remove server-side available-name allocation so accepted state equals the submitted file manifest. | +| `sync-server/src/utils/sanitize_path.rs` | Removed | Remove server-side path rewriting and its embedded tests. Invalid paths are rejected; client repair is explicit before CAS. | +| [sync-server/src/errors.rs](sync-server/src/errors.rs) | Changed | Remove an unused `log::error` import. Compile cleanup only; error semantics were not changed here. | + +### Client core and build + +| File | Change | What changed and why | +| -------------------------------------------------------------------------------------------------------------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [frontend/sync-client/README.md](frontend/sync-client/README.md) | Added | Add state/merge policy, notification semantics, recovery layout, adapter requirements, and compatibility boundaries. Gives integrators the contract needed to make the core guarantees meaningful. | +| [frontend/sync-client/src/consts.ts](frontend/sync-client/src/consts.ts) | Changed | Raise the supported API from 2 to 4 so the rewritten client agrees with the server and rejects incompatible versions. | +| [frontend/sync-client/src/index.ts](frontend/sync-client/src/index.ts) | Changed | Export `FileSnapshot` and new file-manifest/event/request types alongside the retained client surface. Consumers can implement the new adapters and use the v4 protocol types. | +| [frontend/sync-client/src/persistence/database.ts](frontend/sync-client/src/persistence/database.ts) | Changed | Replace mutable per-file scheduling records with v4 engine state, pending snapshots/responses, deferred remote heads, and application journals. Save before publishing memory state, reload uncertain outcomes, bind state to a vault, and retain cursor-compatible read-only views. | +| [frontend/sync-client/src/persistence/persistence.ts](frontend/sync-client/src/persistence/persistence.ts) | Changed | Require an atomic durable complete-state save contract. Recovery needs to know that a successful save survives interruption. | +| [frontend/sync-client/src/persistence/settings.ts](frontend/sync-client/src/persistence/settings.ts) | Changed | Persist settings before updating memory/listeners and return copies including the ignore-pattern array. Settings saves are assumed to succeed; caller mutation must not silently change the live configuration. | +| [frontend/sync-client/src/file-operations/filesystem-operations.ts](frontend/sync-client/src/file-operations/filesystem-operations.ts) | Changed | Replace the weaker read/overwrite/text-update interface with coherent snapshots, explicit stat, exclusive durable writes, no-replace durable moves, and directory-only pruning. State the no-link and flush requirements needed for recovery. | +| [frontend/sync-client/src/file-operations/file-operations.ts](frontend/sync-client/src/file-operations/file-operations.ts) | Changed | Replace direct operations and line-ending conversion with journaled staged application, recovery phases, persisted prepared output, race reconciliation, unexpected-occupant preservation, completed-artifact cleanup, and empty-directory pruning. Makes multi-file namespace changes recoverable without permanent per-application archives. | +| `frontend/sync-client/src/file-operations/safe-filesystem-operations.ts` | Removed | Remove the in-memory filesystem safety/locking wrapper. Its guarantees cannot substitute for durable journaling and the stronger adapter contract. | +| [frontend/sync-client/src/services/protocol.ts](frontend/sync-client/src/services/protocol.ts) | Added | Add a single import surface over Rust-generated v4 types plus `FileManifestEntries`. Avoid maintaining a second handwritten protocol definition. | +| [frontend/sync-client/src/services/server-config.ts](frontend/sync-client/src/services/server-config.ts) | Changed | Clear a rejected cached ping promise. An initial offline attempt must be able to fetch configuration again. | +| [frontend/sync-client/src/services/sync-service.ts](frontend/sync-client/src/services/sync-service.ts) | Changed | Replace legacy multipart/create/text/binary/delete methods and retry builders with typed v4 CAS, vault-snapshot, and replay requests; use `Device-Id`, request timeouts, and permanent/auth/transient error distinctions. The persisted actor owns retries. | +| [frontend/sync-client/src/services/websocket-manager.ts](frontend/sync-client/src/services/websocket-manager.ts) | Changed | Parse `vaultEvents` as `EventBatch` and notify the actor. Keep existing transport/cursor support while switching the payload shape. | +| [frontend/sync-client/src/sync-client.ts](frontend/sync-client/src/sync-client.ts) | Changed | Rewire the public client around one durable actor; enforce adapter contracts, serialize persistence, preserve state on reset, restrict vault changes, and keep cursor/history/telemetry integration. Notifications enqueue work and explicit waiting reports completion/failure. | +| [frontend/sync-client/src/sync-operations/content.ts](frontend/sync-client/src/sync-operations/content.ts) | Added | Add stored snapshot conversion and client-only three-way content merging, originating-mtime binary decisions, cursor transformation, BOM handling, and exact-byte/unchanged-side fast paths. Centralizes the content policy used by normal sync and recovery. | +| [frontend/sync-client/src/sync-operations/file-manifest.ts](frontend/sync-client/src/sync-operations/file-manifest.ts) | Added | Add per-UUID three-way merge, absence/deletion handling, portable validation, and deterministic cross-ID conflict allocation, including ancestor and oversized-name cases. Produces a valid namespace before CAS and local application. | +| [frontend/sync-client/src/sync-operations/syncer.ts](frontend/sync-client/src/sync-operations/syncer.ts) | Changed | Replace concurrent per-document scheduling with the recover/bootstrap/retry/scan/replay/merge/push loop, durable requests, complete scans, identity hints, generation guards, deferred downloads, periodic wakeups, and accepted-snapshot base tracking. Makes namespace and content decisions share one consistent state machine. | +| `frontend/sync-client/src/sync-operations/unrestricted-syncer.ts` | Removed | Remove the old second scheduling/operation layer, server-merge response assumptions, and cache-driven update flow. Keeping it beside the new actor would preserve conflicting ownership and v2 behavior. | +| [frontend/sync-client/src/sync-operations/cursor-tracker.ts](frontend/sync-client/src/sync-operations/cursor-tracker.ts) | Changed | Await SHA-256 comparisons, match remote cursors by UUID, display current local paths, and treat missing reads as prior cursor data. Renames must not attach cursors to another document at the old path. | +| `frontend/sync-client/src/utils/find-matching-file.ts` | Removed | Remove the old standalone matching helper. Conservative unique-hash identity inference now lives in the actor with complete scan context. | +| [frontend/sync-client/src/utils/hash.ts](frontend/sync-client/src/utils/hash.ts) | Changed | Replace the 32-bit rolling checksum with async Web Crypto SHA-256 and its known empty digest. Reduce false unchanged/rename decisions from checksum collisions. | +| [frontend/sync-client/src/utils/set-up-telemetry.ts](frontend/sync-client/src/utils/set-up-telemetry.ts) | Changed | Declare the webpack-injected version constant and guard it with a development fallback. Direct source imports used by consumers/validation must not throw before synchronization starts. | +| [frontend/sync-client/webpack.config.js](frontend/sync-client/webpack.config.js) | Changed | Set ts-loader `onlyCompileBundledFiles: true`. Production browser/Node builds typecheck their dependency graph without pulling in unmigrated legacy fixtures; this is not a claim those fixtures pass. | + +### Client generated bindings + +| File | Change | What changed and why | +| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [frontend/sync-client/src/services/types/ClientCursors.ts](frontend/sync-client/src/services/types/ClientCursors.ts) | Changed | Regenerate the existing cursor payload as a type alias with generated array syntax. Keeps generated output current; no intended wire change. | +| [frontend/sync-client/src/services/types/CursorPositionFromClient.ts](frontend/sync-client/src/services/types/CursorPositionFromClient.ts) | Changed | Regenerate the existing cursor payload. Type-alias/array formatting only; the ephemeral wire shape remains the same. | +| [frontend/sync-client/src/services/types/CursorPositionFromServer.ts](frontend/sync-client/src/services/types/CursorPositionFromServer.ts) | Changed | Regenerate the existing cursor response. Formatting/type-alias change only. | +| [frontend/sync-client/src/services/types/CursorSpan.ts](frontend/sync-client/src/services/types/CursorSpan.ts) | Changed | Regenerate the start/end span definition. Formatting/type-alias change only. | +| [frontend/sync-client/src/services/types/DocumentWithCursors.ts](frontend/sync-client/src/services/types/DocumentWithCursors.ts) | Changed | Regenerate the existing document/cursor descriptor from Rust. Preserve its existing wire field names while aligning generated representation. | +| [frontend/sync-client/src/services/types/WebSocketHandshake.ts](frontend/sync-client/src/services/types/WebSocketHandshake.ts) | Changed | Regenerate the token/device/cursor handshake. The replay cursor field is retained; representation changes from interface to type alias. | +| [frontend/sync-client/src/services/types/PingResponse.ts](frontend/sync-client/src/services/types/PingResponse.ts) | Changed | Regenerate current server configuration metadata, including supported API version. No additional ping endpoint behavior is introduced by the generated-file change. | +| [frontend/sync-client/src/services/types/SerializedError.ts](frontend/sync-client/src/services/types/SerializedError.ts) | Changed | Regenerate the existing error shape. Formatting/type-alias/array syntax changes only. | +| [frontend/sync-client/src/services/types/DocumentUpdateResponse.ts](frontend/sync-client/src/services/types/DocumentUpdateResponse.ts) | Changed | Replace obsolete client `FastForwardUpdate`/`MergingUpdate` variants with `Accepted`/`StaleBase`. Align the client with CAS outcomes already introduced server-side and retained in v4. | +| [frontend/sync-client/src/services/types/DocumentVersion.ts](frontend/sync-client/src/services/types/DocumentVersion.ts) | Changed | Regenerate content bytes plus flattened v4 metadata: originating mtime and content size, without path/deletion. Matches the new content-only read/stale response. | +| [frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts](frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts) | Changed | Regenerate v4 content metadata with mtime and without namespace fields. Used by events, acknowledgements, and bootstrap heads. | +| [frontend/sync-client/src/services/types/PushContent.ts](frontend/sync-client/src/services/types/PushContent.ts) | Added | Add generated `Snapshot`/`Diff` transport payloads. No delete operation is allowed in the content resource. | +| [frontend/sync-client/src/services/types/PutFileContent.ts](frontend/sync-client/src/services/types/PutFileContent.ts) | Added | Add generated request ID, nullable content parent, mtime, and content payload. Makes exact content retry data typed. | +| [frontend/sync-client/src/services/types/PushFileManifest.ts](frontend/sync-client/src/services/types/PushFileManifest.ts) | Added | Add generated request ID, file manifest parent, and UUID-to-path map. Makes exact namespace retry data typed. | +| [frontend/sync-client/src/services/types/FileManifest.ts](frontend/sync-client/src/services/types/FileManifest.ts) | Added | Add generated file manifest ID and entries map. Namespace versions are independent of document versions. | +| [frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts](frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts) | Added | Add generated file manifest acceptance/stale variants. The client needs the canonical map when rebasing. | +| [frontend/sync-client/src/services/types/VaultEvent.ts](frontend/sync-client/src/services/types/VaultEvent.ts) | Added | Add generated content-or-file-manifest event union. Both mutation kinds participate in one sequence. | +| [frontend/sync-client/src/services/types/EventRecord.ts](frontend/sync-client/src/services/types/EventRecord.ts) | Added | Add generated event/request IDs and flattened event payload. Supports ordered replay and tracing of accepted operations. | +| [frontend/sync-client/src/services/types/EventBatch.ts](frontend/sync-client/src/services/types/EventBatch.ts) | Added | Add generated events and head watermark. Lets the client check complete, contiguous replay. | +| [frontend/sync-client/src/services/types/VaultSnapshot.ts](frontend/sync-client/src/services/types/VaultSnapshot.ts) | Added | Add the generated consistent file-manifest/content-head/watermark vault snapshot. Makes initial bootstrap independent of lossy notifications. | +| [frontend/sync-client/src/services/types/WebSocketServerMessage.ts](frontend/sync-client/src/services/types/WebSocketServerMessage.ts) | Changed | Replace `vaultUpdate` with `vaultEvents` while retaining cursor positions. Matches the ordered server stream. | +| `frontend/sync-client/src/services/types/CreateDocumentVersion.ts` | Removed | Remove the old optional-server-ID multipart/create request type. V4 creation is client-UUID content CAS followed by file manifest CAS. | +| `frontend/sync-client/src/services/types/DeleteDocumentVersion.ts` | Removed | Remove the old content deletion payload type. Deletion is absence from the submitted file manifest. | +| `frontend/sync-client/src/services/types/UpdateDocumentVersion.ts` | Removed | Remove the legacy content/path update type. V4 uses `PutFileContent` and independent namespace CAS. | +| `frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts` | Removed | Remove the old text-specific path/update request. Text merges locally and uses the same content CAS transport. | +| `frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts` | Removed | Remove the old document-list response type. Membership/bootstrap/replay use file manifests, snapshots, and event batches. | +| `frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts` | Removed | Remove the latest-documents/initial-sync update type. Ordered batches and a separate bootstrap vault snapshot replace its two roles. | + + +## Maintainer reading order + +For the protocol, read [the server README](sync-server/README.md), then server +`requests.rs`, `models.rs`, `mutations.rs`, `queries.rs`, and the WebSocket handler. +For the client, read [the client README](frontend/sync-client/README.md), then +`persistence/database.ts`, `sync-operations/file-manifest.ts`, `sync-operations/syncer.ts`, +and `file-operations/file-operations.ts`. The adapter interface is part of the +correctness argument and should be read before implementing an integration. + +Keep the distinction between a current file manifest version and the event watermark, +and between an accepted sent snapshot and today's disk contents, explicit in any +future change. Those distinctions are central to avoiding skipped namespace +changes and duplicated merges after a lost acknowledgement. diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index 35e63d50..6272040e 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -1,438 +1,97 @@ -# Sync Algorithm - -VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients. - -## Operational Transformation - -Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent. - -### Why OT? - -Traditional conflict resolution approaches: - -- **Last write wins**: Loses data, frustrating for users -- **Manual merging**: Interrupts workflow, requires user intervention -- **Version branching**: Complex, not suitable for real-time sync - -Operational transformation: - -- **Automatic**: No user intervention required -- **Preserves all edits**: No data loss -- **Real-time**: Changes appear immediately -- **Intuitive**: Behaviour matches user expectations - -## The reconcile-text Library - -VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents. - -### Why reconcile-text over CRDTs? - -VaultLink faces a **differential synchronisation** challenge: users edit Obsidian vaults with various editors (Obsidian desktop, Obsidian mobile, Vim, VS Code, or any text editor), often while offline. This means we only observe the **final state** of each document after editing, not the individual keystrokes or operations that produced it. - -**The fundamental problem**: - -- **CRDTs and traditional OT** require capturing every individual operation (each character insertion, deletion, cursor movement) -- **VaultLink's reality**: Users edit files with arbitrary tools, sync happens after the fact -- **What we know**: Parent version and two modified versions -- **What we don't know**: The sequence of operations that created those modifications - -**Why reconcile-text wins for this use case**: - -1. **Works with end states only**: reconcile-text performs conflict-free 3-way merging given just parent, left, and right versions—no operation history needed - -2. **Editor-agnostic**: Users can edit with any tool without requiring VaultLink-specific plugins or operation tracking - -3. **Offline-first**: Edits made while disconnected are merged cleanly when sync resumes, because we're diffing final states rather than replaying operations - -4. **No conflict markers**: Unlike Git merge, produces clean merged output without `<<<<<<<` markers that interrupt note-taking flow - -5. **Human text forgiveness**: For knowledge bases and documentation, a slightly imperfect merge (e.g., minor word order issues) is vastly preferable to manual conflict resolution - -6. **Simpler infrastructure**: No need for complex operation capture, transformation logs, or tombstone management that CRDTs require - -**The trade-off**: - -CRDTs excel when you control the entire editing infrastructure and can capture every operation. reconcile-text excels when you're synchronising independently-edited files—exactly VaultLink's scenario. The merge quality depends on Myers' diff algorithm rather than operation history, which is the correct trade-off for differential sync. - -For note-taking workflows where users value editor freedom and offline editing, this approach provides superior user experience compared to either CRDTs (which would require operation tracking) or Git-style merging (which requires manual conflict resolution). - -[Learn more about reconcile-text →](https://schmelczer.dev/reconcile) - -### How It Works - -Given three versions (parent, left, right), reconcile-text produces a merged result. - -**How reconcile-text works**: - -1. **Tokenisation**: Split text into words (using `BuiltinTokenizer::Word`) -2. **Three-way diff**: Compare parent→left and parent→right changes -3. **Merge**: Combine non-conflicting changes, prefer content preservation for conflicts -4. **Result**: Merged text with both edits applied - -**Example**: - -``` -Parent: "The quick brown fox" -User A: "The quick red fox" (changes "brown" → "red") -User B: "The very quick brown fox" (inserts "very ") - -Merged: "The very quick red fox" (both changes applied) -``` - -**Merge conditions**: Only `.md` and `.txt` files with valid UTF-8 get merged. Binary files or other extensions use last-write-wins. - -### Operation Types - -The algorithm handles these operations: - -- **Insert**: Add text at position -- **Delete**: Remove text from position -- **Retain**: Keep existing text unchanged - -### Transformation Process - -1. **Client A** makes edit and sends to server -2. **Client B** makes concurrent edit and sends to server -3. **Server** receives both edits -4. **Server** transforms operations to account for concurrent changes -5. **Server** applies merged result to database -6. **Server** sends transformed operations to both clients -7. **Clients** apply transformed operations locally - -## Sync State Management - -VaultLink maintains sync state to track which changes have been applied. - -### Version Vectors - -Each document has a version tracked by: - -- **Server version**: Incremented on each change -- **Client cursors**: Track which version each client has seen - -This enables: - -- Efficient syncing (only send changes since last sync) -- Conflict detection (concurrent edits to same version) -- Ordering of operations - -### Cursor Management - -Clients maintain a cursor position: - -```rust -struct Cursor { - vault_id: String, - client_id: String, - last_version: u64, - last_updated: DateTime, -} -``` - -On sync: - -1. Client sends cursor (last seen version) -2. Server returns all changes since that version -3. Client applies changes and updates cursor - -## Conflict Resolution Flow - -### Scenario: Concurrent Edits - -Two users edit the same paragraph simultaneously. - -**Initial state**: - -``` -Version 10: "The quick brown fox jumps over the lazy dog." -``` - -**User A's edit** (version 11): - -``` -"The quick brown fox jumps over the very lazy dog." -``` - -_Inserts "very " at position 40_ - -**User B's edit** (also from version 10): - -``` -"The quick red fox jumps over the lazy dog." -``` - -_Replaces "brown" with "red" at position 10_ - -### Server Processing - -1. **Receive User A's operation**: - - Base: version 10 - - Operation: Insert("very ", position=40) - - Apply to database → version 11 - -2. **Receive User B's operation**: - - Base: version 10 - - Operation: Replace("brown"→"red", position=10) - - **Conflict detected**: Base is version 10, but current is version 11 - -3. **Transform User B's operation**: - - Transform against User A's operation - - Adjust positions/content as needed - - Apply transformed operation → version 12 - -4. **Broadcast updates**: - - Send User A's operation to User B - - Send transformed User B's operation to User A - -### Final Result - -``` -Version 12: "The quick red fox jumps over the very lazy dog." -``` - -Both edits are preserved in the final document. - -## Edge Cases - -### 1. Delete vs Insert Conflict - -**Scenario**: User A deletes a paragraph while User B edits it. - -**Resolution**: - -- OT algorithm prioritizes preservation of content -- Insert operation is transformed to account for deletion -- Typically results in inserted content appearing nearby - -**Example**: - -``` -Base: "Line 1\nLine 2\nLine 3" - -User A: Delete Line 2 → "Line 1\nLine 3" -User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3" - -Result: "Line 1\nLine 2 modified\nLine 3" -``` - -(Insert takes precedence, preserving user content) - -### 2. Overlapping Edits - -**Scenario**: Two users edit overlapping regions. - -**Resolution**: - -- OT splits operations into non-overlapping segments -- Applies each segment independently -- Merges results - -### 3. Delete vs Delete - -**Scenario**: Two users delete overlapping text. - -**Resolution**: - -- Deletes are merged -- Final result has the union of deleted ranges removed - -### 4. Network Partitions - -**Scenario**: Client loses connection, makes edits offline, reconnects. - -**Resolution**: - -1. Client queues edits locally -2. On reconnect, sends all queued operations -3. Server applies OT against all operations that happened during partition -4. Client receives transformed operations and applies - -## Performance Characteristics - -### Time Complexity - -- **Single operation**: O(1) for most operations -- **Transformation**: O(n) where n is operation size -- **Conflict resolution**: O(m × n) where m is number of concurrent operations - -### Space Complexity - -- **Version history**: Grows with number of changes -- **Cursors**: O(clients × vaults) -- **Active operations**: Minimal (processed in real-time) - -### Optimisation - -VaultLink optimises for: - -- Small, frequent edits (typical typing patterns) -- Text documents (not binary files) -- Real-time processing (no batching delay) - -## Limitations - -### Binary and Non-Mergeable Files - -Only **`.md`** and **`.txt`** files get automatic merging. Everything else uses last-write-wins. - -**Binary detection**: - -- Files with NUL bytes (`0x00`) -- Files failing UTF-8 validation - -Even `.md` files are treated as binary if they fail UTF-8 checks. - -**Last-write-wins behaviour**: - -``` -User A uploads image.png → Server version 1 -User B uploads image.png → Server version 2 (A's upload lost) -``` - -**Workaround**: Avoid concurrent edits to non-text files. [See all limitations →](/guide/limitations) - -### Large Documents - -Very large documents (> 1MB) may have: - -- Higher transformation costs -- Slower sync times -- Increased memory usage - -**Workaround**: Split large documents or increase timeout settings. - -### Complex Formatting - -Markdown with complex structures may occasionally produce unexpected results: - -- Nested lists -- Tables -- Code blocks - -**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures. - -## Consistency Guarantees - -### Strong Consistency - -VaultLink provides **strong eventual consistency**: - -- All clients eventually converge to the same state -- Operations applied in causal order -- No data loss under normal operation - -### Ordering Guarantees - -- Operations from the same client are applied in order -- Concurrent operations may be applied in any order -- Final result is independent of operation order (commutative) - -### Durability - -- Operations are written to SQLite before acknowledgment -- SQLite ACID guarantees protect against data loss -- Clients retry failed uploads - -## Comparison with Other Approaches - -### Git-style Merging - -| Aspect | Git Merge | VaultLink OT | -| -------------------------- | ------------ | ----------------------- | -| Real-time | No | Yes | -| Manual conflict resolution | Yes | No | -| Branching | Yes | No | -| Automatic merge | Limited | Always | -| Use case | Code changes | Collaborative documents | - -### CRDTs (Conflict-free Replicated Data Types) - -| Aspect | CRDTs | VaultLink (reconcile-text) | -| ----------------------------- | ------------------------------------ | ------------------------------------------------- | -| **Operation tracking** | Required (every keystroke) | Not required (end states only) | -| **Editor freedom** | Limited (must use CRDT-aware editor) | Unlimited (any text editor works) | -| **Offline editing** | Requires operation log | Works with file comparison | -| **Server required** | No | Yes | -| **Memory overhead** | Higher (tombstones, metadata) | Lower (versions only) | -| **Infrastructure complexity** | Higher | Lower | -| **Best for** | Controlled editing environments | Independent file editing (Obsidian, Vim, VS Code) | - -**Key insight**: CRDTs are superior when you can capture every operation. reconcile-text is superior when users edit files independently with arbitrary tools—exactly VaultLink's scenario. - -### Last Write Wins - -| Aspect | LWW | VaultLink OT | -| --------------- | ---- | ------------ | -| Data loss | Yes | No | -| Simplicity | High | Medium | -| User experience | Poor | Excellent | -| Performance | Best | Good | - -## Algorithm Details - -### Transformation Rules - -When transforming operation `A` against operation `B`: - -1. **Insert vs Insert**: - - If positions equal: Order by client ID - - If different positions: Adjust positions - -2. **Insert vs Delete**: - - If insert in deleted range: Shift insert position - - If insert after delete: Adjust position by deleted length - -3. **Delete vs Delete**: - - If ranges overlap: Merge delete ranges - - If ranges disjoint: Adjust positions - -4. **Retain vs Any**: - - Retain operations don't conflict - - Simply adjust positions - -### Transformation Example - -```rust -// Pseudo-code for transformation -fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) { - match (op_a, op_b) { - (Insert(pos_a, text_a), Insert(pos_b, text_b)) => { - if pos_a < pos_b { - (op_a, Insert(pos_b + text_a.len(), text_b)) - } else if pos_a > pos_b { - (Insert(pos_a + text_b.len(), text_a), op_b) - } else { - // Same position, use client ID to break tie - if client_id_a < client_id_b { - (op_a, Insert(pos_b + text_a.len(), text_b)) - } else { - (Insert(pos_a + text_b.len(), text_a), op_b) - } - } - } - // ... other cases - } -} -``` - -## Best Practices - -### For Smooth Collaboration - -1. **Small edits**: Make small, focused changes for easier merging -2. **Coordinate major changes**: Discuss large refactors with team -3. **Monitor sync status**: Ensure changes are uploaded before signing off -4. **Test conflict resolution**: Verify behaviour matches expectations - -### For Developers - -1. **Text files preferred**: OT works best on text -2. **Limit file sizes**: Keep documents reasonably sized -3. **Binary files**: Use versioning or avoid concurrent edits -4. **Testing**: Test concurrent edit scenarios thoroughly - -## Further Reading - -- [reconcile-text library](https://crates.io/crates/reconcile-text) -- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation) -- [Data flow architecture →](/architecture/data-flow) +# Sync algorithm + +The current API v5 server stores immutable content versions and a separately +versioned file manifest. Clients reconcile snapshots with the current local files. The server does not merge competing edits. + +## Identity and ordering + +Each document has a UUID independent of its path. A file manifest maps live UUIDs +to paths; removing a mapping deletes membership. Updating content never restores +membership. Directories are implicit. + +Every accepted content or manifest update appends an event in the same SQLite +transaction. Event IDs order changes and serve as version IDs. A push names its +parent version and a unique request UUID. The server either accepts that exact +snapshot or returns current metadata as `StaleBase`. Retrying an accepted request +returns its original acknowledgement; changing its payload under the same request +UUID is rejected. + +## Reconciliation + +A new client reads a coherent vault snapshot, applies it, then replays +subsequent events over HTTP. Pages are folded in memory to their final document +heads and manifest before changing local files. Interrupted fetching restarts from +the last incorporated cursor without saved pagination state. This avoids applying +intermediate remote edits that were already reverted. + +For text, the client compares the common base, current local bytes and current +remote bytes using `reconcile-text`. Configured extensions and valid UTF-8 without +NUL bytes determine whether text merging is allowed. Diffs optimize transport; +they do not authorize the server to merge a stale base. Large diff operation sets +are sent as snapshots. + +If only one side changed, that side is retained. Concurrent unmergeable changes +keep the server version. First contact with existing local text has no shared ancestor and uses +an empty parent. Concurrent edits may still require human review. + +Paths use a separate three-way decision: keep a local change when the server path +is unchanged, otherwise prefer the server path if both changed. Portable-name +collisions receive deterministic conflict names. A manifest deletion is not +reversed merely because another client edited the document. Local backup archives +are not created. + +## Filesystem application + +The client applies an in-memory plan directly to the visible namespace, then +saves one metadata snapshot. Swaps and cycles move occupied sources to temporary +conflict names that remain ordinary discoverable files if application stops. +There is no persisted filesystem plan, staged payload, recovery replay or fsync +protocol. An interruption may leave a subset of the operations visible, or partial +file content; the next scan reconciles whatever is present then. + +Create, delete and move notifications persist logical document identities; content +updates only wake the loop to reread current bytes. Namespace changes detected +before mutations invalidate the plan. Notifications during application are incorporated +between files using their document identities; the next scan rereads content. Content is reread +before replacement. Ignored and oversized occupants stay in place. Adapters +use raw filesystem events only to wake a scan. These include engine writes and +atomic editor replacements, so they must not be reported as logical create/delete +notifications. Integrations with reliable user-action events can persist those +explicit notifications to preserve rename identities, as Obsidian does for +`Vault.rename`. Raw watchers infer a move +only when the missing and new files have a unique matching, nonempty content hash. + +Metadata persistence must replace a complete value atomically, leaving a readable +old/new value after interruption. Metadata and user files need not describe the +same moment: scans repair that disagreement. File creation and rename must reject +occupied destinations; scans must be complete and reads coherent. No guarantee is +made that user bytes or rename identities survive power loss. + +## Server restores and failures + +Numeric version equality cannot establish identity after restoring a backup. +Successful HTTP responses include a checkpoint with an event-incarnation token. +The client persists and echoes it; the server rejects a checkpoint from a discarded +history before executing its request. Each new event gets a fresh token even if +its numeric ID or request UUID was used on the discarded branch. + +After a mismatch, the client records reset intent, clears old history-dependent state and bootstraps against the restored +server. Clean files reconcile against the restored snapshot. Unsent edits with an +old base are preserved as separate local documents, with conflict names if needed. +Interrupted recovery saves are replayable. Backups must be restored with the server +stopped. + +WebSocket `vaultChanged` hints wake HTTP replay. The server periodically compares +its history checkpoint to detect missed broadcasts; transport liveness checks +recover disconnected clients. Authentication and permanent errors remain visible; +transient errors retry according to client settings. Observer callback failures +are isolated from the background sync loop. + +API v5 clients and servers must be deployed together. V4 databases migrate without +removing history. CLI and desktop Obsidian share the Node filesystem adapter and +complete-record metadata persistence. Node file moves use exclusive creation +followed by unlink, so interruption may leave two ordinary files. Metadata alone +is flushed before an atomic rename; there is no user-file durability protocol. +Obsidian mobile uses the host storage API. Its exclusive-copy API requires a +scratch input file in the ignored internal directory; leftovers are never replayed +or used for recovery. Mobile metadata persistence uses Obsidian's data store. diff --git a/docs/guide/limitations.md b/docs/guide/limitations.md index 1c514939..87c8e712 100644 --- a/docs/guide/limitations.md +++ b/docs/guide/limitations.md @@ -8,14 +8,8 @@ VaultLink works well for most Obsidian vaults, but has some constraints you shou Only **`.md`** and **`.txt`** files get automatic conflict-free merging. -Other file types (images, PDFs, etc.) use last-write-wins: - -``` -User A updates diagram.png → Server stores version 1 -User B updates diagram.png → Server stores version 2 (overwrites A's changes) -``` - -**Workaround**: Avoid editing the same non-text file simultaneously. +For other file types, a one-sided change is retained. Concurrent unmergeable +changes keep the server's version. The client does not archive displaced bytes. ### Binary Detection @@ -24,7 +18,7 @@ Files are treated as binary if they: - Contain NUL bytes (`0x00`) - Fail UTF-8 validation -Binary files within `.md` or `.txt` extensions still get last-write-wins (no merge). +Binary files within `.md` or `.txt` extensions follow the same unmergeable-content policy. ## Performance Constraints @@ -61,11 +55,11 @@ Rough estimates (varies by vault size and activity): - All versions stored indefinitely (no automatic cleanup) - Each vault is a separate SQLite database -- Deleted files marked as deleted (not purged) +- Deletion removes file-manifest membership; content history remains readable **Growth**: Version history grows with every change. A 10 MB vault with frequent edits might grow to 100+ MB over months. -**Cleanup**: Manual only (see [Advanced Configuration](/config/advanced#version-history-cleanup)). +Immutable merge bases and request receipts must not be independently deleted. There is no automatic safe history-pruning protocol. ### Implications @@ -92,7 +86,7 @@ Result: "The very quick red fox" ← Both changes preserved - Simultaneous edits to the same sentence - Large structural changes (moving sections around) -**Result**: Merged file might need manual cleanup in ~1-5% of concurrent edits. +Merged files may require manual review; there is no measured universal conflict-success rate. ## Scalability @@ -140,6 +134,22 @@ Not currently supported. Running multiple servers requires manual vault partitio - Tokens configured in server config file - No runtime user management +## Restored backups + +Restore server databases while the server is stopped. API v5 clients detect a +changed history even when version numbers are reused. Clean files reconcile with +the restored server; unsent edits are preserved as separate local documents. +Clients and server must be upgraded together. This does not replace independent +backups or protect against arbitrary physical-media corruption. + +## Interrupted sync + +The client has no filesystem journal or power-loss durability protocol. Interrupted +writes may leave missing, partial or moved files. On restart, normal scans reconcile +the current files, including edits made while the client was stopped. Metadata +saves must remain complete and readable, so the client can continue syncing; exact +pre-interruption file contents and identities are not guaranteed. + ## Known Edge Cases ### Simultaneous Deletes and Edits @@ -147,10 +157,10 @@ Not currently supported. Running multiple servers requires manual vault partitio ``` User A deletes note.md User B edits note.md -Result: Edit wins (file recreated with B's content) +Result: Manifest deletion removes the file, including unsent local edits. ``` -Operational transformation prioritises content preservation. +Content updates do not restore deleted membership. There is no local recovery archive. ### Large File Uploads diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md index 6fa2848c..c137bc72 100644 --- a/frontend/deterministic-tests/README.md +++ b/frontend/deterministic-tests/README.md @@ -1,118 +1,134 @@ -# Deterministic Tests +# Sync test harnesses (API v4) -Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case. - -Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios. - -## How it works - -Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one. - -Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process. - -The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit. - -## Step types - -Clients always start with syncing disabled. - -**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited): - -- `create`, `update`, `rename`, `delete` -- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path. - -**Sync control:** - -- `sync` — wait for a specific client or all clients to finish pending operations -- `barrier` — retry until all clients converge to identical file state (60s timeout) -- `enable-sync` / `disable-sync` — simulate going online/offline -- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable -- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync` - -**WebSocket control** (per-client): - -- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client - -**Server control:** - -- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process -- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire. - -**Fault injection** (per-client): - -- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit. -- `wait-for-dropped-create-response` — wait until the armed drop has fired. - -**Assertions:** - -- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback - -## Running +Run the complete harness check from **any working directory**: ```sh -# Build server first -cd sync-server && cargo build --release && cd - - -# Run all tests -cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests - -# Filter by name -npm run test -w deterministic-tests -- --filter=rename - -# Control parallelism (default: number of CPU cores) -npm run test -w deterministic-tests -- -j 4 +E2E_WORKERS=2 /path/to/vault-link/scripts/e2e.sh ``` -## Adding a test +The `E2E_WORKERS` variable is the number of fuzz workers / simultaneous scripted tests. +The runner builds the server and the three in-scope frontend packages, then runs +the harness self-tests, recovery regressions, protocol/real-filesystem tests, +scripted scenarios, and seeded fuzz workers. It does not build the Obsidian plugin +or CLI application. Install frontend dependencies with `npm ci` first. Node's +built-in WebSocket, Rust, and a C compiler (`cc`, for test-only exclusive rename +and OS-lock primitives on macOS/Linux) are required. -1. Create `src/tests/my-scenario.test.ts`: +CI installs those dependencies in `frontend` before invoking the runner. It sets +`E2E_ARTIFACTS=logs/e2e` and uploads that same directory even when a suite fails. +The runner owns all test servers; no separate server or legacy cleanup script is +needed in the workflow. -```typescript -import type { TestDefinition } from "../test-definition"; +Each server gets a fresh temporary database, working directory, and port. Only +owned processes are terminated. There is no `pkill`, fixed database deletion, +or shared server PID file. Failing tests and teardown errors give nonzero exits; +the remaining suites still run. Logs, concrete fuzz traces, results, and failure +snapshots are retained in the printed artifact directory. Snapshots are captured +before teardown so cleanup failures retain the original evidence. -export const myScenarioTest: TestDefinition = { - description: - "Client 0 creates A.md offline. After syncing, both clients should have the file.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "hello" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s) => { - s.assertFileCount(1).assertContent("A.md", "hello"); - } - } - ] -}; +Environment options: + +- `E2E_WORKERS=1`: number of fuzz workers / simultaneous scripted tests (max 32). +- `E2E_SEED=1`: first uint32 seed; workers use successive seeds. +- `E2E_ITERATIONS=25`: generated conflict scenarios per worker. +- `E2E_TIMEOUT_SECONDS=900`: timeout for each build/suite/worker command. +- `E2E_ARTIFACTS=/absolute/path`: optional output directory; defaults to `mkdtemp`. + +## Running individual layers + +From `frontend`: + +```sh +# Rebuild protocol peers and run scripted tests (name filter optional). +npm run test --workspace deterministic-tests -- --filter v4- --concurrency 2 + +# Adapter/oracle/lifecycle self-tests and strict recovery regressions. +npm run test:harness + +# After rebuilding: real worker death, server SIGKILL/restart, CAS/WS ordering, +# and host filesystem contract checks. +npm run test:protocol + +npm run build --workspace sync-client --workspace test-client +node test-client/dist/cli.js --seed 42 --iterations 25 +node test-client/dist/cli.js --replay /path/to/trace-42.json ``` -The `verify` callback receives an `AssertableState` object with chainable assertion methods: +`test:harness` includes strict regressions for **engine** bugs. +A failing default run is not an expected-failure test or a harness skip. See +[`../test-support/README.md`](../test-support/README.md) for their scope. -```typescript -s.assertFileCount(n); // exact file count -s.assertFileExists("path"); // file must exist -s.assertFileNotExists("path"); // file must not exist -s.assertContent("path", "expected"); // exact content match -s.assertContains("path", "a", "b"); // all substrings present in file -s.assertContainsAny("path", "a", "b"); // at least one substring present -s.assertAnyFileContains("text"); // substring present in some file -s.assertNoFileContains("text"); // substring absent from every file -s.assertSubstringCount("path", "x", 3); // substring appears exactly N times -s.assertContentInAtMostOneFile("text"); // no duplicate content -s.ifFileExists("path", (s) => { /* … */ }); // conditional block -s.getContent("path"); // raw content (or "" if missing) -``` +## What the scripted runner checks -2. Register it in `src/test-registry.ts`: +Real `SyncClient` instances use the shared v4 adapter in `../test-support`. +External editor operations and engine filesystem operations are separate: engine +writes never generate self-notifications. Editor identity notifications continue +while transport is disabled. Delayed notifications can be explicitly queued and +flushed. Snapshots and persisted state are deep copies, never shared references. -```typescript -import { myScenarioTest } from "./tests/my-scenario.test"; +Scripted and seeded clients explicitly set `syncIntervalMs: 0`, with assertions +that it stays disabled. Barriers observe convergence; they do not wake the engine. +Network retry timers remain enabled. Separate fake-clock tests in +`test-support/polling.test.ts` exercise optional periodic reconciliation. -const TESTS = { - // ... - "my-scenario": myScenarioTest -}; -``` +A barrier checks byte-for-byte replica equality **and** the server's canonical +UUID/path map, file bytes, independently computed base/observed hashes, content +and manifest CAS parents, event watermark, +contiguous event log, and absence of pending requests. +Internal recovery files are excluded from the user-visible file list, but are +present in failure disk images. Scenario assertions separately check intended +content, document identity, or preservation markers; equality alone is not an +adequate data-loss oracle. + +Clients start disabled. `enable-sync` waits for the initial sync attempt/retries, +except while the server is intentionally paused: that attempt is tracked so a +later script step can resume the server. Thus sequential enables really establish +a winner. To test _independent creates_, first bootstrap both clients empty, then +disable them and create files. Initial adoption of pre-existing files during +bootstrap is a different policy from two already-initialized clients creating +distinct UUIDs at one path. + +## Controls and assertions + +- `create`, `update`, `create-bytes`, `rename`, `delete`: external editor actions. + Editor rename may explicitly replace its target; engine rename never may. +- `sync`, `barrier`, `enable-sync`, `disable-sync`. +- `reset`: restart transports; **not** a crash and not a database reset. +- `pause-websocket`, `resume-websocket`: buffer only WS delivery. HTTP catchup + remains active; this is deliberately not called "disconnect". +- `pause-observation`, `wait-for-observation`, `resume-observation`: explicitly + gate HTTP catchup as well as WS, including replies already in flight. The + checkpoint must be exercised and released, and held reads respect cancellation. +- `delay-notifications`, `flush-notifications`: delay editor notifications without + disabling preservation assertions. +- `drop-response`: `{client, kind: "create" | "content" | "manifest", point: +"before" | "after"}`. After faults fire only on an `Accepted` CAS response. +- `wait-for-response-drop`: explicit checkpoint; every armed fault must fire. + Every interrupted request must be retried with an identical request ID, target + and body before a different CAS is sent. Fault kinds match exactly. +- `rename-next-write`: inject an editor rename after the engine's content write, + before its metadata save. +- `pause-server`, `resume-server`: SIGSTOP/SIGCONT, not a crash. +- `crash-server`, `restart-server`: SIGKILL and restart with the same database/port. + Server-control scenarios have dedicated processes. +- `remember-identity`, `assert-identity`, `assert-files`, `assert-markers`. +- `assert-documents`: a complete expected set of `{key, path, content, conflict?}` + records. Content is text or a byte array. Keys bind UUIDs on the first successful + check and must retain them on later checks; new keys cannot reuse deleted UUIDs. + `conflict: true` resolves the UUID-derived conflict name for `path`. +- `assert-consistent` can provide `verify(state)`: exact text/bytes, UUID lookups, + `assertIdentity`, and `conflictPath(original)` for UUID-derived conflict names. + +Legacy timing scenarios are retained. Prefer explicit fault checkpoints and +barriers for new tests. Only known coherent-scan races and actually injected +transient network errors may retry; they still must converge before the deadline. +Other background errors, unhandled rejections, and cleanup failures fail the run. + +## Reproducibility limits + +Seeds and JSON traces reproduce harness choices and logical actions. They do not +control the OS scheduler, server thread scheduling, or production-generated UUIDs. +The recovery sweep enumerates adapter/save boundaries; worker termination proves +abrupt runtime loss. Neither is a proof covering every real filesystem or hardware +power-loss behavior. Run multiple seeds, repeat timing-sensitive cases, and keep +the scenario's independent oracle when reducing a failing trace. diff --git a/frontend/deterministic-tests/package.json b/frontend/deterministic-tests/package.json index 4bd82c74..4e7ee563 100644 --- a/frontend/deterministic-tests/package.json +++ b/frontend/deterministic-tests/package.json @@ -8,7 +8,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "npm run build && node dist/cli.js" + "test": "cargo build --release --manifest-path ../../sync-server/Cargo.toml && npm run build --workspace sync-client && npm run build && node dist/cli.js" }, "devDependencies": { "commander": "^14.0.2", diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts index 6e15cac0..2c1f495c 100644 --- a/frontend/deterministic-tests/src/cli.ts +++ b/frontend/deterministic-tests/src/cli.ts @@ -9,10 +9,11 @@ import { runWithConcurrency } from "./run-with-concurrency"; import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts"; import * as path from "node:path"; import * as fs from "node:fs"; -import { debugging, Logger } from "sync-client"; +import * as os from "node:os"; +import { Logger } from "sync-client"; const logger = new Logger(); -debugging.logToConsole(logger, { useColors: true }); +logger.onLogEmitted.add((line) => console.log(line.message)); process.on("unhandledRejection", (reason) => { logger.error(`Unhandled Rejection: ${reason}`); @@ -32,6 +33,8 @@ function testUsesPauseServer(test: TestDefinition): boolean { (step) => step.type === "pause-server" || step.type === "resume-server" || + step.type === "crash-server" || + step.type === "restart-server" || step.type === "resume-server-until-history-then-pause" ); } @@ -42,7 +45,7 @@ function testUsesPauseServer(test: TestDefinition): boolean { */ function findProjectRoot(): string { let dir = path.dirname(__filename); - const root = path.parse(dir).root; + const { root } = path.parse(dir); while (dir !== root) { if ( fs.existsSync(path.join(dir, "sync-server")) && @@ -114,11 +117,7 @@ async function runDedicatedServerTest( } return { name, result }; } finally { - try { - await server.stop(); - } catch { - // best-effort cleanup - } + await server.stop(); serverManager.untrack(server); } } @@ -138,6 +137,11 @@ async function main(): Promise { } const { filter, concurrency } = parseArgs(process.argv); + const artifacts = + process.env.E2E_ARTIFACTS ?? + fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-deterministic-")); + fs.mkdirSync(artifacts, { recursive: true }); + logger.info(`Artifacts: ${artifacts}`); const testsToRun: [string, TestDefinition][] = []; for (const [key, test] of Object.entries(TESTS)) { @@ -193,13 +197,7 @@ async function main(): Promise { allResults.push(...results); } finally { - try { - await sharedServer.stop(); - } catch (error) { - logger.warn( - `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}` - ); - } + await sharedServer.stop(); serverManager.untrack(sharedServer); } } @@ -221,6 +219,10 @@ async function main(): Promise { const passed = allResults.filter((r) => r.result.success); const failed = allResults.filter((r) => !r.result.success); + fs.writeFileSync( + path.join(artifacts, "deterministic-results.json"), + JSON.stringify(allResults, null, 2) + ); logger.info( `\n--- Results: ${passed.length}/${allResults.length} passed ---` diff --git a/frontend/deterministic-tests/src/crash-worker.ts b/frontend/deterministic-tests/src/crash-worker.ts new file mode 100644 index 00000000..c015fac6 --- /dev/null +++ b/frontend/deterministic-tests/src/crash-worker.ts @@ -0,0 +1,93 @@ +import { parentPort, workerData } from "node:worker_threads"; +import assert from "node:assert/strict"; +import { SyncClient } from "sync-client"; +import { + MemoryDisk, + MemoryPersistence, + type DiskImage, + type StoredClient +} from "../../test-support/storage"; +import { + classifyRequest, + type RequestRecord, + type RequestKind +} from "../../test-support/network"; + +export interface CrashWorkerInput { + disk: DiskImage; + stored: StoredClient; + stopAt?: RequestKind; + editAfterSync?: { path: string; content: string }; +} +const input = workerData as CrashWorkerInput; +const disk = new MemoryDisk(); +disk.restore(input.disk); +const persistence = new MemoryPersistence(input.stored); +const requests: RequestRecord[] = []; +let editedBase: { documentId: string; vaultUpdateId: number } | undefined; +const snapshot = () => ({ + disk: disk.image(), + stored: persistence.snapshot(), + requests, + editedBase +}); + +async function main() { + const client = await SyncClient.create({ + fs: disk.session(), + persistence, + fetch: async (url, init) => { + const record = classifyRequest(url, init); + if (record) requests.push(record); + const response = await fetch(url, init); + if (record && response.ok && input.stopAt === record.kind) { + const receipt = (await response.clone().json()) as { + type: string; + }; + if (receipt.type === "Accepted") { + parentPort!.postMessage({ + type: "checkpoint", + ...snapshot() + }); + // Parent terminates this worker. No destroy(), no state flush, + // no finally block, and no old runtime survives the restart. + await new Promise(() => {}); + } + } + return response; + } + }); + assert.equal( + client.getSettings().syncIntervalMs, + 0, + "Crash recovery must preserve the disabled polling setting" + ); + await client.start(); + await client.waitUntilFinished(); + if (input.editAfterSync) { + const state = persistence.snapshot().database!; + const documentId = Object.keys(state.local!).find( + (id) => state.local![id] === input.editAfterSync!.path + ); + assert(documentId, "Edit must target an established document"); + editedBase = { + documentId, + vaultUpdateId: state.documents![documentId].base!.vaultUpdateId + }; + await disk.userWrite( + input.editAfterSync.path, + new TextEncoder().encode(input.editAfterSync.content) + ); + await client.syncLocallyUpdatedFile({ + relativePath: input.editAfterSync.path + }); + await client.waitUntilFinished(); + } + await client.destroy(); + parentPort!.postMessage({ type: "finished", ...snapshot() }); + parentPort!.close(); +} +main().catch((error: unknown) => { + parentPort!.postMessage({ type: "error", error: String(error) }); + process.exit(1); +}); diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts index b32b01c2..4ef3a838 100644 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -1,483 +1,336 @@ -import type { - HistoryEntry, - StoredDatabase, - SyncSettings, - RelativePath, - TextWithCursors -} from "sync-client"; import { SyncClient, - SyncResetError, - debugging, LogLevel, - utils + type HistoryEntry, + type SyncSettings } from "sync-client"; +import { MemoryDisk, MemoryPersistence } from "../../test-support/storage"; +import { NetworkFaults, type RequestKind } from "../../test-support/network"; import { assert } from "./utils/assert"; import { sleep } from "./utils/sleep"; import { withTimeout } from "./utils/with-timeout"; -import { - IS_SYNC_ENABLED_BY_DEFAULT, - WAIT_TIMEOUT_MS, - WEBSOCKET_CONNECT_TIMEOUT_MS, - WEBSOCKET_POLL_INTERVAL_MS -} from "./consts"; +import { WAIT_TIMEOUT_MS, WEBSOCKET_CONNECT_TIMEOUT_MS } from "./consts"; import { ManagedWebSocketFactory } from "./managed-websocket"; -export class DeterministicAgent extends debugging.InMemoryFileSystem { - public readonly clientId: number; - private readonly logger: (msg: string) => void; - private client!: SyncClient; - private data: Partial<{ - settings: Partial; - database: Partial; - }> = {}; - private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT; - private readonly syncErrors: Error[] = []; - private readonly pendingSyncOperations = new Set>(); +/** User actions live here; the engine only receives disk.session(). */ +export class DeterministicAgent { + public readonly disk = new MemoryDisk(); + public readonly persistence: MemoryPersistence; + public readonly network = new NetworkFaults(); + private client?: SyncClient; + private readonly errors: Error[] = []; + private readonly pending = new Set>(); private readonly wsFactory = new ManagedWebSocketFactory(); - private nextWriteRename: - | { - oldPath: RelativePath; - newPath: RelativePath; - } - | undefined; - private nextCreateResponseDrop: - | { - dropped: Promise; - resolveDropped: () => void; - } - | undefined; + private nextWriteRename?: { oldPath: string; newPath: string }; + private delayedNotifications = false; + private disposed = false; + private readonly notifications: (() => Promise)[] = []; public constructor( - clientId: number, - initialSettings: Partial, - logger: (msg: string) => void + public readonly clientId: number, + settings: Partial, + private readonly logger: (msg: string) => void ) { - super(); - this.clientId = clientId; - this.logger = logger; - this.data.settings = { ...initialSettings }; + this.persistence = new MemoryPersistence({ + settings: { + networkRetryIntervalMs: 25, + webSocketRetryIntervalMs: 50, + requestTimeoutMs: 5_000, + ...settings, + // Scripted and seeded scenarios must exercise notification delivery. + syncIntervalMs: 0 + } + }); + this.disk.boundary = async (label) => { + const rename = this.nextWriteRename; + // Inject after the content write, before the metadata save. + if (rename && label === `durable:write:${rename.oldPath}`) { + this.nextWriteRename = undefined; + await this.rename(rename.oldPath, rename.newPath); + } + }; } public async init( - fetchImplementation: typeof globalThis.fetch + fetchImplementation: typeof globalThis.fetch = fetch ): Promise { this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: this.wrapFetch(fetchImplementation), + fs: this.disk.session(), + persistence: this.persistence, + fetch: this.network.wrap(fetchImplementation), webSocket: this.wsFactory.constructorFn }); - - this.client.logger.onLogEmitted.add((line) => { - const prefix = `[Client ${this.clientId}]`; - switch (line.level) { - case LogLevel.ERROR: - this.logger(`${prefix} ERROR: ${line.message}`); - break; - case LogLevel.WARNING: - this.logger(`${prefix} WARN: ${line.message}`); - break; - case LogLevel.INFO: - this.logger(`${prefix} INFO: ${line.message}`); - break; - case LogLevel.DEBUG: - this.logger(`${prefix} DEBUG: ${line.message}`); - break; - } - }); - - await this.client.start(); - - const connectionCheck = await this.client.checkConnection(); assert( - connectionCheck.isSuccessful, + this.client.getSettings().syncIntervalMs === 0, + "Correctness scenarios must run with periodic polling disabled" + ); + this.client.logger.onLogEmitted.add((line) => { + this.logger( + `[Client ${this.clientId}] ${line.level}: ${line.message}` + ); + if (this.disposed && line.level === LogLevel.ERROR) + throw new Error( + `Client ${this.clientId} error after cleanup: ${line.message}` + ); + if ( + line.level === LogLevel.ERROR && + !this.isRetryableFailure(line.message) + ) + this.errors.push(new Error(line.message)); + }); + await this.client.start(); + assert( + (await this.client.checkConnection()).isSuccessful, `Client ${this.clientId} connection check failed` ); + } - if (this.isSyncEnabled) { - await this.waitForWebSocket(); + public database() { + return this.persistence.snapshot().database; + } + public files(): Map { + return this.disk.userFiles(); + } + public async listFilesRecursively(): Promise { + return [...this.files().keys()].sort(); + } + public async getFileContent(path: string): Promise { + const bytes = this.files().get(path); + assert(bytes !== undefined, `Missing file: ${path}`); + return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + } + public async read(path: string): Promise { + const snapshot = await this.disk.readSnapshot(path); + assert(snapshot !== undefined, `Missing file: ${path}`); + return snapshot.content; + } + public async write(path: string, content: Uint8Array): Promise { + const created = !(await this.disk.exists(path)); + await this.disk.userWrite(path, content); + this.notify(() => + created + ? this.client!.syncLocallyCreatedFile(path) + : this.client!.syncLocallyUpdatedFile({ relativePath: path }) + ); + } + public async rename(oldPath: string, relativePath: string): Promise { + await this.disk.userRename(oldPath, relativePath); + this.notify(() => + this.client!.syncLocallyUpdatedFile({ oldPath, relativePath }) + ); + } + public async delete(path: string): Promise { + await this.disk.userDelete(path); + this.notify(() => this.client!.syncLocallyDeletedFile(path)); + } + public delayNotifications(): void { + this.delayedNotifications = true; + } + public flushNotifications(): void { + this.delayedNotifications = false; + for (const notify of this.notifications.splice(0)) this.enqueue(notify); + } + private notify(operation: () => Promise): void { + // Offline means transport disabled, not that editor identity events vanish. + if (this.delayedNotifications) this.notifications.push(operation); + else this.enqueue(operation); + } + private enqueue(operation: () => Promise): void { + const promise = operation().catch((error: unknown) => { + if (!this.isRetryableFailure(String(error))) + this.errors.push( + error instanceof Error ? error : new Error(String(error)) + ); + }); + this.pending.add(promise); + void promise.then(() => this.pending.delete(promise)); + } + private async drain(): Promise { + while (this.pending.size) await Promise.all([...this.pending]); + } + private isRetryableFailure(message: string): boolean { + // A coherent scan rejecting an editor race is a required adapter + // behavior. It must subsequently converge; never blanket-ignore errors. + return ( + this.network.isExpectedFailure(message) || + /^(Sync paused for retry: )?Error: File changed during scan: .+$/.test( + message + ) + ); + } + private assertHealthy(): void { + if (this.errors.length) { + const errors = this.errors.splice(0); + throw new AggregateError( + errors, + `Client ${this.clientId}: ${errors.map(String).join("; ")}` + ); } } - - public pauseWebSocket(): void { - this.log("Pausing WebSocket message delivery"); - this.wsFactory.pause(); - } - - public resumeWebSocket(): void { - this.log("Resuming WebSocket message delivery"); - this.wsFactory.resume(); - } - - public dropNextCreateResponse(): void { + public async waitForSync(): Promise { assert( - this.nextCreateResponseDrop === undefined, - `Client ${this.clientId} already has a create response drop armed` - ); - let resolveDropped!: () => void; - const dropped = new Promise((resolve) => { - resolveDropped = resolve; - }); - this.nextCreateResponseDrop = { - dropped, - resolveDropped - }; - this.log("Armed next create response drop"); - } - - public async waitForDroppedCreateResponse(): Promise { - assert( - this.nextCreateResponseDrop !== undefined, - `Client ${this.clientId} has no create response drop armed` + this.client!.getSettings().syncIntervalMs === 0, + "Periodic polling was enabled during a correctness scenario" ); await withTimeout( - this.nextCreateResponseDrop.dropped, + (async () => { + await this.drain(); + const deadline = Date.now() + WAIT_TIMEOUT_MS; + for (;;) { + try { + await this.client!.waitUntilFinished(); + break; + } catch (error) { + if ( + !this.isRetryableFailure(String(error)) || + Date.now() >= deadline + ) + throw error; + await sleep(30); + } + } + this.assertHealthy(); + })(), WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for create response drop` + `Client ${this.clientId} sync timeout` ); - this.log("Create response was dropped after server commit"); } - + public async disableSync(): Promise { + await this.drain(); + await this.client!.setSetting("isSyncEnabled", false); + this.assertHealthy(); + } + public async enableSync(wait = true): Promise { + // setSetting waits for a sync attempt in v4. Track it, but don't block + // the script from resuming a paused server or releasing a checkpoint. + this.enqueue(() => this.client!.setSetting("isSyncEnabled", true)); + const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; + while ( + !this.persistence.snapshot().settings?.isSyncEnabled && + Date.now() < deadline + ) + await sleep(1); + assert( + !!this.persistence.snapshot().settings?.isSyncEnabled, + "Enable did not persist" + ); + if (wait) await this.waitForSync(); + } + public async reset(): Promise { + await this.drain(); + await this.client!.reset(); + } + public pauseWebSocket(): void { + this.wsFactory.pause(); + } + public pauseObservation(): void { + this.pauseWebSocket(); + this.network.pauseObservation(); + } + public resumeObservation(): void { + this.network.resumeObservation(); + this.resumeWebSocket(); + } + public async waitForObservation(): Promise { + await withTimeout( + this.network.waitForObservation(), + WAIT_TIMEOUT_MS, + "HTTP observation checkpoint never reached" + ); + } + public resumeWebSocket(): void { + this.wsFactory.resume(); + } + public dropNextCreateResponse(): void { + this.network.arm("create"); + } + public dropNextResponse( + kind: RequestKind, + point: "before" | "after" = "after" + ): void { + this.network.arm(kind, point); + } + public async waitForDroppedCreateResponse(): Promise { + await withTimeout( + this.network.wait(), + WAIT_TIMEOUT_MS, + "Armed response drop never fired" + ); + } + public renameNextWrite(oldPath: string, newPath: string): void { + assert( + !this.nextWriteRename, + "Previous install/rename hook never fired" + ); + this.nextWriteRename = { oldPath, newPath }; + } public async waitForHistoryEntry( matches: (entry: HistoryEntry) => boolean, onMatch?: (entry: HistoryEntry) => void ): Promise { - const existing = this.client.getHistoryEntries().find(matches); - if (existing !== undefined) { - onMatch?.(existing); - return; - } - - await withTimeout( - new Promise((resolve) => { - const unsubscribe = this.client.onSyncHistoryUpdated.add(() => { - const entry = this.client - .getHistoryEntries() - .find(matches); - if (entry === undefined) { - return; - } - - unsubscribe(); - onMatch?.(entry); - resolve(); - }); - }), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for history entry` - ); - } - - public async waitForSync(): Promise { - this.log("Waiting for sync to complete..."); - // Drain agent-level sync operations first. These are the fire-and-forget - // promises from enqueueSync() that call into the SyncClient's methods. - // Without this, waitUntilFinished() might return before the SyncClient - // has even been told about the operation. - await this.drainPendingSyncOperations(); - await withTimeout( - this.client.waitUntilFinished(), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms` - ); - if (this.syncErrors.length > 0) { - const errors = this.syncErrors.splice(0); - throw new Error( - `Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}` - ); - } - this.log("Sync complete"); - } - - public async reset(): Promise { - this.log("Resetting client (clears tracked state, keeps disk files)"); - await this.drainPendingSyncOperations(); - await this.client.reset(); - if (this.isSyncEnabled) { - await this.waitForWebSocket(); - } - } - - public async disableSync(): Promise { - this.log("Disabling sync"); - // Drain pending enqueued operations before disabling so the SyncClient - // knows about all operations that were enqueued while sync was enabled. - await this.drainPendingSyncOperations(); - await this.client.setSetting("isSyncEnabled", false); - this.isSyncEnabled = false; - // Wait for in-flight operations to drain. Disabling sync triggers - // a reset, which aborts in-flight fetches with SyncResetError. + let unsubscribe = () => {}; try { await withTimeout( - this.client.waitUntilFinished(), + new Promise((resolve) => { + const check = () => { + const entry = + this.client!.getHistoryEntries().find(matches); + if (entry) { + onMatch?.(entry); + resolve(); + } + }; + unsubscribe = this.client!.onSyncHistoryUpdated.add(check); + check(); + }), WAIT_TIMEOUT_MS, - `Client ${this.clientId} disableSync drain timed out` + "History checkpoint not reached" ); - } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log("Disable sync drain interrupted by reset (expected)"); - } else { - throw error; - } + } finally { + unsubscribe(); } } - - public async enableSync(): Promise { - this.log("Enabling sync"); - await this.client.setSetting("isSyncEnabled", true); - this.isSyncEnabled = true; - await this.waitForWebSocket(); - } - - public async getFileContent(path: string): Promise { - const bytes = await this.read(path); - return new TextDecoder().decode(bytes); - } - - public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void { - assert( - this.nextWriteRename === undefined, - `Client ${this.clientId} already has a next-write rename armed` - ); - this.nextWriteRename = { oldPath, newPath }; - this.log(`Armed next write rename: ${oldPath} -> ${newPath}`); - } - public async cleanup(): Promise { - this.log("Cleaning up..."); - // Guard against uninitialized client (init() failed partway). - // The class field uses `!:` so TS thinks this is always defined, - // but at runtime it can be undefined when init() throws partway. - const maybeClient = this.client as SyncClient | undefined; - if (maybeClient === undefined) { - this.log("Client not initialized, nothing to clean up"); - return; + if (!this.client) return; + const errors: unknown[] = []; + this.resumeObservation(); + this.flushNotifications(); + try { + await this.waitForSync(); + } catch (error) { + errors.push(error); + } + try { + this.network.assertConsumed(); + assert(!this.nextWriteRename, "Install/rename hook never fired"); + } catch (error) { + errors.push(error); } try { - await this.drainPendingSyncOperations(); await withTimeout( - this.client.waitUntilFinished(), + this.client.destroy(), WAIT_TIMEOUT_MS, - `Client ${this.clientId} cleanup waitUntilFinished timed out` + "Destroy timed out" + ); + await withTimeout( + this.wsFactory.finish(), + WAIT_TIMEOUT_MS, + "WebSocket callbacks did not finish during cleanup" ); } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Cleanup interrupted by reset (expected): ${error}`); - } else { - this.log(`Cleanup waitUntilFinished failed: ${error}`); - } + errors.push(error); } - // Surface any background sync errors that arrived after the last - // waitForSync (e.g. between the final assert-consistent and here). - // Without this, regressions that fault the engine during the very - // last step of a test would be silently swallowed. - const pendingErrors = this.syncErrors.splice(0); - await this.client.destroy(); - this.log("Cleanup complete"); - if (pendingErrors.length > 0) { - throw new Error( - `Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}` - ); - } - } - - public override async read(path: RelativePath): Promise { - await Promise.resolve(); - return super.read(path); - } - - public override async write( - path: RelativePath, - content: Uint8Array - ): Promise { - await Promise.resolve(); - const isNew = !this.files.has(path); - await super.write(path, content); - - if (this.isSyncEnabled && isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyCreatedFile(path); - }); - } - - const nextWriteRename = this.nextWriteRename; - if ( - nextWriteRename !== undefined && - nextWriteRename.oldPath === path - ) { - this.nextWriteRename = undefined; - await super.rename( - nextWriteRename.oldPath, - nextWriteRename.newPath - ); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath: nextWriteRename.oldPath, - relativePath: nextWriteRename.newPath - }); - }); - } - // The rename consumed `path`. Skip the post-update enqueue below - // — it would send a syncLocallyUpdatedFile for a path that no - // longer exists. - return; - } - - if (!this.isSyncEnabled) { - return; - } - - if (!isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - } - - public override async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const result = await super.atomicUpdateText(path, updater); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - return result; - } - - public override async delete(path: RelativePath): Promise { - await super.delete(path); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyDeletedFile(path); - }); - } - } - - public override async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - await super.rename(oldPath, newPath); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - }); - } - } - - private async waitForWebSocket(): Promise { - const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; - while (!this.client.isWebSocketConnected && Date.now() < deadline) { - await sleep(WEBSOCKET_POLL_INTERVAL_MS); - } - assert( - this.client.isWebSocketConnected, - `Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms` - ); - } - - /** - * Wait until all agent-level enqueued sync operations have completed. - * Uses a loop because completing one operation can trigger new enqueues. - */ - private async drainPendingSyncOperations(): Promise { - while (this.pendingSyncOperations.size > 0) { - await utils.awaitAll([...this.pendingSyncOperations]); - } - } - - private enqueueSync(operation: () => Promise): void { - const promise = this.executeSyncOperation(operation).catch( - (error: unknown) => { - const err = - error instanceof Error ? error : new Error(String(error)); - this.log(`Background sync failed: ${err.message}`); - this.syncErrors.push(err); - } - ); - this.pendingSyncOperations.add(promise); - void promise.finally(() => { - this.pendingSyncOperations.delete(promise); - }); - } - - private async executeSyncOperation( - operation: () => Promise - ): Promise { + this.client = undefined; + this.disposed = true; try { - await operation(); + this.assertHealthy(); } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Sync operation interrupted by reset: ${error}`); - return; - } - if ( - error instanceof Error && - error.message.includes("has been destroyed") - ) { - this.log(`Sync operation interrupted by destroy: ${error}`); - return; - } - - throw error; + errors.push(error); } - } - - private log(message: string): void { - this.logger(`[Client ${this.clientId}] ${message}`); - } - - private wrapFetch( - fetchImplementation: typeof globalThis.fetch - ): typeof globalThis.fetch { - return async (input, init) => { - const response = await fetchImplementation(input, init); - const drop = this.nextCreateResponseDrop; - if ( - drop !== undefined && - DeterministicAgent.isCreateDocumentRequest(input, init) - ) { - this.nextCreateResponseDrop = undefined; - try { - await response.body?.cancel(); - } catch { - // Best-effort — body may already be consumed/closed. - } - drop.resolveDropped(); - throw new SyncResetError(); - } - return response; - }; - } - - private static isCreateDocumentRequest( - input: RequestInfo | URL, - init: RequestInit | undefined - ): boolean { - const method = - init?.method ?? - (typeof Request !== "undefined" && input instanceof Request - ? input.method - : "GET"); - if (method.toUpperCase() !== "POST") { - return false; - } - - const url = - input instanceof URL - ? input - : new URL(typeof input === "string" ? input : input.url); - return /\/documents\/?$/.test(url.pathname); + if (errors.length) + throw new AggregateError( + errors, + `Client ${this.clientId} cleanup failed: ${errors.map(String).join("; ")}` + ); } } diff --git a/frontend/deterministic-tests/src/managed-websocket.ts b/frontend/deterministic-tests/src/managed-websocket.ts index c759891b..7d1159b5 100644 --- a/frontend/deterministic-tests/src/managed-websocket.ts +++ b/frontend/deterministic-tests/src/managed-websocket.ts @@ -209,8 +209,13 @@ export class ManagedWebSocketFactory { // WS before the agent reconnects would silently see the new socket // start un-paused and miss the messages it meant to buffer. private currentlyPaused = false; + private disposed = false; public get constructorFn(): typeof globalThis.WebSocket { + const ensureActive = () => { + if (this.disposed) + throw new Error("WebSocket created after harness cleanup"); + }; const trackInstance = (instance: ManagedWebSocket): void => { this.instances.push(instance); if (this.currentlyPaused) { @@ -222,6 +227,7 @@ export class ManagedWebSocketFactory { url: string | URL, protocols?: string | string[] ) { + ensureActive(); super(url, protocols); trackInstance(this); } @@ -229,6 +235,21 @@ export class ManagedWebSocketFactory { return TrackedManagedWebSocket; } + public async finish(): Promise { + this.disposed = true; + await Promise.all( + this.instances.map((socket) => + socket.readyState === WebSocket.CLOSED + ? Promise.resolve() + : new Promise((resolve) => + socket.addEventListener("close", () => resolve(), { + once: true + }) + ) + ) + ); + } + public pause(): void { this.currentlyPaused = true; for (const ws of this.instances) { diff --git a/frontend/deterministic-tests/src/parse-args.ts b/frontend/deterministic-tests/src/parse-args.ts index 11c56f19..a6968d13 100644 --- a/frontend/deterministic-tests/src/parse-args.ts +++ b/frontend/deterministic-tests/src/parse-args.ts @@ -7,8 +7,8 @@ export interface CliArgs { } function parsePositiveInt(value: string): number { - const n = parseInt(value, 10); - if (isNaN(n) || n <= 0) { + const n = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(n) || n <= 0) { throw new InvalidArgumentError("must be a positive integer"); } return n; diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts index 9cb4cde0..03c3f418 100644 --- a/frontend/deterministic-tests/src/server-control.ts +++ b/frontend/deterministic-tests/src/server-control.ts @@ -20,10 +20,13 @@ export class ServerControl { private _port: number | undefined; private tempDir: string | undefined; private _isPaused = false; + private preserveOnStop = false; + private expectedExit = false; + private unexpectedExit?: Error; public constructor(serverPath: string, configPath: string, logger: Logger) { - this.serverPath = serverPath; - this.baseConfigPath = configPath; + this.serverPath = path.resolve(serverPath); + this.baseConfigPath = path.resolve(configPath); this.logger = logger; } @@ -38,6 +41,15 @@ export class ServerControl { return `http://localhost:${this.port}`; } + public get databaseDirectory(): string { + if (!this.tempDir) throw new Error("Server has no database directory"); + return path.join(this.tempDir, "databases"); + } + + public get isPaused(): boolean { + return this._isPaused; + } + public async start(): Promise { if (this.process !== null) { throw new Error("Server is already running"); @@ -66,14 +78,25 @@ export class ServerControl { } private async startOnce(): Promise { - const reservation = await findFreePort(); - this._port = reservation.port; + const reservation = + this._port === undefined ? await findFreePort() : undefined; + this._port ??= reservation!.port; const tmpBase = os.tmpdir(); - this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-")); - const tempConfigPath = path.join(this.tempDir, "config.yml"); - const dbDir = path.join(this.tempDir, "databases"); - - this.writeConfigFile(tempConfigPath, dbDir); + let tempConfigPath: string; + try { + this.tempDir ??= fs.mkdtempSync( + path.join(tmpBase, "vault-link-test-") + ); + tempConfigPath = path.join(this.tempDir, "config.yml"); + this.writeConfigFile( + tempConfigPath, + path.join(this.tempDir, "databases") + ); + } catch (error) { + reservation?.release(); + this.cleanupTempDir(); + throw error; + } this.logger.info( `Starting server: ${this.serverPath} (port ${this._port})` @@ -81,12 +104,15 @@ export class ServerControl { // Release the port reservation right before spawning to minimize // the TOCTOU window between port discovery and server binding. - reservation.release(); + reservation?.release(); this.process = spawn(this.serverPath, [tempConfigPath], { + cwd: this.tempDir, stdio: ["ignore", "pipe", "pipe"], detached: false }); + this.expectedExit = false; + this.unexpectedExit = undefined; this.process.stdout?.on("data", (data: Buffer) => { this.logger.info(`[SERVER] ${data.toString().trim()}`); @@ -102,6 +128,10 @@ export class ServerControl { const currentProcess = this.process; currentProcess.on("exit", (code, signal) => { + if (!this.expectedExit) + this.unexpectedExit = new Error( + `Owned server exited unexpectedly (code ${code}, signal ${signal})` + ); this.logger.info( `Server exited with code ${code}, signal ${signal}` ); @@ -139,7 +169,9 @@ export class ServerControl { ); } try { - const response = await fetch(pingUrl); + const response = await fetch(pingUrl, { + signal: AbortSignal.timeout(1_000) + }); if (response.ok) { this.logger.info("[SERVER] Ready"); return; @@ -194,11 +226,13 @@ export class ServerControl { public async stop(): Promise { const proc = this.process; if (proc?.pid === undefined) { - this.cleanupTempDir(); + if (!this.preserveOnStop) this.cleanupTempDir(); + if (this.unexpectedExit) throw this.unexpectedExit; return; } + this.expectedExit = true; - // Resume if paused — a SIGSTOP'd process ignores SIGKILL + // SIGKILL terminates even a stopped process; SIGCONT is harmless here. if (this._isPaused) { try { process.kill(proc.pid, "SIGCONT"); @@ -212,7 +246,7 @@ export class ServerControl { // Set up a promise that resolves when the process actually exits. const exitPromise = new Promise((resolve) => { - if (proc.exitCode !== null) { + if (proc.exitCode !== null || proc.signalCode !== null) { resolve(); return; } @@ -229,18 +263,55 @@ export class ServerControl { // Wait for the process to actually exit before cleaning up, // with a 5s safety timeout to avoid hanging forever. - await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]); + let timer: ReturnType | undefined; + try { + await Promise.race([ + exitPromise, + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + "Owned server did not exit after SIGKILL" + ) + ), + STOP_TIMEOUT_MS + ); + }) + ]); + } finally { + clearTimeout(timer); + } this.process = null; this._isPaused = false; - this.cleanupTempDir(); + if (!this.preserveOnStop) this.cleanupTempDir(); + if (this.unexpectedExit) throw this.unexpectedExit; + } + + /** A real process death: retain the exact database and port for restart. */ + public async crash(): Promise { + if (!this.isRunning()) throw new Error("Cannot crash a stopped server"); + this.preserveOnStop = true; + try { + await this.stop(); + } finally { + this.preserveOnStop = false; + } + } + + public async restart(): Promise { + if (!this.tempDir || this.isRunning()) + throw new Error( + "Restart requires a crashed server with retained storage" + ); + await this.startOnce(); } public isRunning(): boolean { const proc = this.process; return ( - proc !== null && - proc.pid !== undefined && + proc?.pid !== undefined && proc.exitCode === null && proc.signalCode === null ); @@ -274,23 +345,27 @@ export class ServerControl { // one `databases_directory_path:` (under `server:` and `database:` // respectively) const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8"); + for (const key of ["port", "databases_directory_path"]) { + if ( + (baseConfig.match(new RegExp(`^ ${key}:`, "gm")) ?? []) + .length !== 1 + ) + throw new Error(`Expected exactly one config key: ${key}`); + } const config = baseConfig .replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`) .replace( /^\s*databases_directory_path:\s*.+/m, - ` databases_directory_path: ${dbDir}` + ` databases_directory_path: ${JSON.stringify(dbDir)}` ); fs.writeFileSync(destPath, config); } private cleanupTempDir(): void { if (this.tempDir !== undefined) { - try { - fs.rmSync(this.tempDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup - } + fs.rmSync(this.tempDir, { recursive: true, force: true }); this.tempDir = undefined; + this._port = undefined; } } } diff --git a/frontend/deterministic-tests/src/server-manager.ts b/frontend/deterministic-tests/src/server-manager.ts index 76c624f7..53862fae 100644 --- a/frontend/deterministic-tests/src/server-manager.ts +++ b/frontend/deterministic-tests/src/server-manager.ts @@ -59,9 +59,8 @@ export class ServerManager { // Last-resort synchronous cleanup. Runs even when the process is // exiting via process.exit() from unhandledRejection / // uncaughtException — paths where async stopAll() cannot complete. - // SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the - // kernel keeps them as zombies holding the test's tmpdir, and the - // next CI run can't reuse the port. + // Kill only the server processes owned by this runner. SIGKILL also + // terminates a SIGSTOP'd process; forceKillSync's SIGCONT is harmless. process.on("exit", () => { for (const server of this.activeServers) { server.forceKillSync(); diff --git a/frontend/deterministic-tests/src/test-definition.ts b/frontend/deterministic-tests/src/test-definition.ts index bd832a50..3e4c5fe0 100644 --- a/frontend/deterministic-tests/src/test-definition.ts +++ b/frontend/deterministic-tests/src/test-definition.ts @@ -3,6 +3,18 @@ import type { AssertableState } from "./utils/assertable-state"; export interface ClientState { files: Map; clientFiles: Map[]; + manifests: Record[]; + canonical: Record; + bytes: Map; +} + +/** A scenario's independent, complete expected state. Keys survive moves; + * a new key must receive a previously unseen UUID, including after deletion. */ +export interface ExpectedDocument { + key: string; + path: string; + conflict?: boolean; + content: string | number[]; } export type TestStep = @@ -31,10 +43,41 @@ export type TestStep = | { type: "assert-consistent"; verify?: (state: AssertableState) => void } | { type: "pause-websocket"; client: number } | { type: "resume-websocket"; client: number } + | { + type: + | "pause-observation" + | "resume-observation" + | "wait-for-observation"; + client: number; + } | { type: "drop-next-create-response"; client: number } | { type: "wait-for-dropped-create-response"; client: number } | { type: "sleep"; ms: number } - | { type: "reset"; client: number }; + | { type: "reset"; client: number } + | { type: "create-bytes"; client: number; path: string; bytes: number[] } + | { + type: "drop-response"; + client: number; + kind: "create" | "content" | "manifest"; + point?: "before" | "after"; + } + | { type: "wait-for-response-drop"; client: number } + | { type: "delay-notifications" | "flush-notifications"; client: number } + | { + type: "remember-identity" | "assert-identity"; + path: string; + key: string; + } + | { + type: "assert-files"; + expected: Record; + absent?: string[]; + count?: number; + } + | { type: "assert-markers"; markers: string[]; removed?: string[] } + | { type: "assert-documents"; expected: ExpectedDocument[] } + | { type: "crash-server" } + | { type: "restart-server" }; export interface TestDefinition { description?: string; @@ -46,4 +89,5 @@ export interface TestResult { success: boolean; error?: string; duration?: number; + diagnostics?: unknown; } diff --git a/frontend/deterministic-tests/src/test-registry.ts b/frontend/deterministic-tests/src/test-registry.ts index 1a07b411..322fe049 100644 --- a/frontend/deterministic-tests/src/test-registry.ts +++ b/frontend/deterministic-tests/src/test-registry.ts @@ -1,4 +1,5 @@ import type { TestDefinition } from "./test-definition"; +import { V4_TESTS } from "./v4-regressions"; import { renameCreateConflictTest } from "./tests/rename-create-conflict.test"; import { renameChainTest } from "./tests/rename-chain.test"; import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test"; @@ -106,6 +107,7 @@ import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-wri import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test"; export const TESTS: Partial> = { + ...V4_TESTS, "rename-create-conflict": renameCreateConflictTest, "rename-chain": renameChainTest, "rename-update-conflict": renameUpdateConflictTest, diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts index 411e9b08..2ea7db3a 100644 --- a/frontend/deterministic-tests/src/test-runner.ts +++ b/frontend/deterministic-tests/src/test-runner.ts @@ -13,6 +13,8 @@ import { IS_SYNC_ENABLED_BY_DEFAULT } from "./consts"; import { randomUUID } from "node:crypto"; +import { assertCanonical } from "../../test-support/canonical"; +import { ContentLedger } from "../../test-support/oracles"; export class TestRunner { private agents: DeterministicAgent[] = []; @@ -20,6 +22,8 @@ export class TestRunner { private readonly token: string; private readonly remoteUri: string; private readonly logger: Logger; + private vaultName = ""; + private readonly identities = new Map(); public constructor( serverControl: ServerControl, @@ -44,6 +48,8 @@ export class TestRunner { } this.logger.info(`Clients: ${test.clients}`); this.logger.info(`Steps: ${test.steps.length}`); + this.identities.clear(); + let diagnostics: TestResult["diagnostics"]; try { assert( @@ -61,6 +67,11 @@ export class TestRunner { await this.executeStep(step); } + assert( + this.serverControl.isRunning(), + "Server died before test completion" + ); + diagnostics = this.captureDiagnostics(); await this.cleanup(); const duration = Date.now() - startTime; @@ -71,25 +82,46 @@ export class TestRunner { duration }; } catch (error) { + diagnostics ??= this.captureDiagnostics(); const duration = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : String(error); this.logger.info(`\n✗ Test failed: ${name}`); this.logger.info(`Error: ${errorMessage}`); - await this.cleanup(); + try { + await this.cleanup(); + } catch (cleanupError) { + return { + success: false, + error: `${errorMessage}\nCleanup: ${String(cleanupError)}`, + duration, + diagnostics + }; + } return { success: false, error: errorMessage, + diagnostics, duration }; } } + private captureDiagnostics(): unknown { + return this.agents.map((agent) => ({ + client: agent.clientId, + database: agent.database(), + disk: agent.disk.image(), + requests: structuredClone(agent.network.requests) + })); + } + private async initializeAgents(count: number): Promise { assert(count > 0, `Client count must be positive, got ${count}`); const vaultName = `test-${randomUUID()}`; + this.vaultName = vaultName; this.logger.info( `Initializing ${count} agents with vault: ${vaultName}` ); @@ -129,6 +161,87 @@ export class TestRunner { private async executeStep(step: TestStep): Promise { switch (step.type) { + case "pause-observation": + this.getAgent(step.client).pauseObservation(); + break; + case "resume-observation": + this.getAgent(step.client).resumeObservation(); + break; + case "wait-for-observation": + await this.getAgent(step.client).waitForObservation(); + break; + case "create-bytes": + await this.getAgent(step.client).write( + step.path, + new Uint8Array(step.bytes) + ); + break; + case "drop-response": + this.getAgent(step.client).dropNextResponse( + step.kind, + step.point + ); + break; + case "wait-for-response-drop": + await this.getAgent(step.client).waitForDroppedCreateResponse(); + break; + case "delay-notifications": + this.getAgent(step.client).delayNotifications(); + break; + case "flush-notifications": + this.getAgent(step.client).flushNotifications(); + break; + case "remember-identity": + case "assert-identity": + await this.assertConsistent((state) => { + if (step.type === "remember-identity") + this.identities.set( + step.key, + state.documentId(step.path) + ); + else { + const id = this.identities.get(step.key); + assert( + id !== undefined, + `Identity ${step.key} was never recorded` + ); + state.assertIdentity(step.path, id); + } + }); + break; + case "assert-files": + await this.assertConsistent((state) => { + for (const [path, value] of Object.entries(step.expected)) + state.assertContent(path, value); + for (const path of step.absent ?? []) + state.assertFileNotExists(path); + if (step.count !== undefined) + state.assertFileCount(step.count); + }); + break; + case "assert-documents": + await this.assertConsistent((state) => + state.assertDocuments(step.expected, this.identities) + ); + break; + case "assert-markers": { + const ledger = new ContentLedger(); + for (const marker of step.markers) ledger.add(marker); + for (const marker of step.removed ?? []) + ledger.removedBy( + marker, + "trace records explicit deletion/overwrite" + ); + for (const agent of this.agents) + ledger.assertPreserved(agent.files()); + break; + } + case "crash-server": + await this.serverControl.crash(); + break; + case "restart-server": + await this.serverControl.restart(); + break; case "create": case "update": await this.getAgent(step.client).write( @@ -170,7 +283,9 @@ export class TestRunner { break; case "enable-sync": - await this.getAgent(step.client).enableSync(); + await this.getAgent(step.client).enableSync( + !this.serverControl.isPaused + ); break; case "pause-server": @@ -190,7 +305,9 @@ export class TestRunner { (entry) => entry.details.type === step.syncType && entry.details.relativePath === step.path, - () => this.serverControl.pause() + () => { + this.serverControl.pause(); + } ); this.serverControl.resume(); await historySeen; @@ -305,6 +422,11 @@ export class TestRunner { // Snapshot all agents' file states upfront to minimize the window // where background sync could mutate state between reads. + const canonical = await assertCanonical( + this.agents, + `${this.remoteUri}/vaults/${encodeURIComponent(this.vaultName)}`, + this.token + ); const clientFiles: Map[] = []; for (const agent of this.agents) { const sortedFiles = (await agent.listFilesRecursively()).sort(); @@ -360,7 +482,12 @@ export class TestRunner { verify( new AssertableState({ files: clientFiles[0], - clientFiles + clientFiles, + manifests: this.agents.map( + (agent) => agent.database()!.local! + ), + canonical: canonical.fileManifest.entries, + bytes: this.agents[0].files() }) ); } catch (error) { @@ -384,10 +511,12 @@ export class TestRunner { } this.logger.info("\nCleaning up agents..."); - for (const agent of this.agents) { + const errors: unknown[] = []; + for (const agent of this.agents.splice(0)) { try { await agent.cleanup(); } catch (error) { + errors.push(error); this.logger.warn( `Agent cleanup error: ${error instanceof Error ? error.message : String(error)}` ); @@ -395,5 +524,10 @@ export class TestRunner { } this.agents = []; this.logger.info("Cleanup complete"); + if (errors.length) + throw new AggregateError( + errors, + `Cleanup failed: ${errors.map(String).join("; ")}` + ); } } diff --git a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts index 467c19f0..f2582964 100644 --- a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts +++ b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts @@ -7,6 +7,12 @@ export const binaryPendingCreateNotDisplacedTest: TestDefinition = { "After syncing, both files should exist on both clients at separate paths.", clients: 2, steps: [ + // Bootstrap both empty clients before making independent documents. + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + { type: "disable-sync", client: 0 }, + { type: "disable-sync", client: 1 }, { type: "create", client: 0, @@ -29,7 +35,7 @@ export const binaryPendingCreateNotDisplacedTest: TestDefinition = { verify: (s: AssertableState): void => { s.assertFileCount(2) .assertFileExists("data.bin") - .assertFileExists("data (1).bin") + .assertFileExists(s.conflictPath("data.bin")) .assertAnyFileContains( "binary data from client 0", "binary data from client 1" diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts index cd8046ce..d76d4053 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts @@ -41,7 +41,7 @@ export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = verify: (state: AssertableState): void => { state .assertFileCount(2) - .assertContains("Y (1).md", "original file X") + .assertContains(state.conflictPath("Y.md"), "original file X") .assertContains("Y.md", "brand new Y content"); } } diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts index 0ac0b721..3ae30504 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts @@ -41,7 +41,7 @@ export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = state .assertFileNotExists("X.md") .assertFileExists("Y.md") - .assertFileExists("Y (1).md") + .assertFileExists(state.conflictPath("Y.md")) .assertAnyFileContains( "original file X", "brand new Y content" diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts index 5337649d..44f3e0a1 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts @@ -52,9 +52,9 @@ export const concurrentRenameFirstWinsTest: TestDefinition = { type: "assert-consistent", verify: (s: AssertableState): void => { s.assertFileNotExists("A.md") - .assertFileCount(2) - .assertContent("B.md", "edit from 0\nline 2\nline 3") - .assertContent("C.md", "line 1\nline 2\nedit from 1"); + .assertFileCount(1) + .assertFileNotExists("C.md") + .assertContent("B.md", "edit from 0\nline 2\nedit from 1"); } } ] diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts index 0b72c0f3..dcd8200d 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts @@ -31,7 +31,7 @@ export const concurrentRenameSameTargetTest: TestDefinition = { .assertFileNotExists("A.md") .assertFileNotExists("B.md") .assertFileExists("C.md") - .assertFileExists("C (1).md") + .assertFileExists(state.conflictPath("C.md")) .assertAnyFileContains("content-a", "content-b"); } } diff --git a/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts b/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts index 782c3cd5..f2ec68f5 100644 --- a/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts +++ b/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts @@ -4,7 +4,7 @@ import type { TestDefinition } from "../test-definition"; export const deleteRecreateDifferentContentTest: TestDefinition = { description: "Client 0 deletes and recreates A.md with new content offline while client 1 edits A.md offline. " + - "Both clients should converge with content from both sides merged.", + "Recreation gets a new UUID; an update to the deleted UUID must not edit the replacement.", clients: 2, steps: [ { @@ -43,11 +43,7 @@ export const deleteRecreateDifferentContentTest: TestDefinition = { { type: "assert-consistent", verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "A.md", - "brand new", - "client 1" - ); + s.assertFileCount(1).assertContent("A.md", "brand new content"); } } ] diff --git a/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts b/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts index 1e9ea8f7..7783daee 100644 --- a/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts +++ b/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts @@ -29,9 +29,8 @@ export const offlineDeleteRemoteRenameTest: TestDefinition = { { type: "assert-consistent", verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md").assertFileNotExists( - "A_renamed.md" - ); + // Both membership/path changed from the base: remote wins. + s.assertFileCount(1).assertFileNotExists("A.md").assertContent("A_renamed.md", "content-a"); } } ] diff --git a/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts b/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts index b951b0be..3e2ebf51 100644 --- a/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts +++ b/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts @@ -26,8 +26,8 @@ export const onlineBothCreateSamePathDeconflictTest: TestDefinition = { type: "assert-consistent", verify: (state: AssertableState): void => { state - .assertFileCount(1) - .assertContains("A.md", "updated-by-0", "from-client-1 "); + .assertFileCount(2) + .assertAnyFileContains("updated-by-0", "from-client-1 "); } } ] diff --git a/frontend/deterministic-tests/src/tests/rename-circular.test.ts b/frontend/deterministic-tests/src/tests/rename-circular.test.ts index 44a65149..c2168f59 100644 --- a/frontend/deterministic-tests/src/tests/rename-circular.test.ts +++ b/frontend/deterministic-tests/src/tests/rename-circular.test.ts @@ -3,7 +3,7 @@ import type { TestDefinition } from "../test-definition"; export const renameCircularTest: TestDefinition = { description: - "Client 0 creates three files, syncs, then goes offline and performs a circular rename via a temp file (A->temp, C->A, B->C, temp->B). After reconnecting, all three contents should exist across three files but paths may be deconflicted.", + "Client 0 creates three files, syncs, then goes offline and performs a circular rename via a temp file (A->temp, C->A, B->C, temp->B). After reconnecting, all three UUIDs and exact contents must follow the requested cycle.", clients: 2, steps: [ { type: "create", client: 0, path: "A.md", content: "content-a" }, @@ -21,6 +21,9 @@ export const renameCircularTest: TestDefinition = { } }, + { type: "remember-identity", key: "A", path: "A.md" }, + { type: "remember-identity", key: "B", path: "B.md" }, + { type: "remember-identity", key: "C", path: "C.md" }, { type: "disable-sync", client: 0 }, { type: "rename", client: 0, oldPath: "A.md", newPath: "temp-a.md" }, { type: "rename", client: 0, oldPath: "C.md", newPath: "A.md" }, @@ -29,15 +32,18 @@ export const renameCircularTest: TestDefinition = { { type: "enable-sync", client: 0 }, { type: "barrier" }, + { type: "assert-identity", key: "A", path: "B.md" }, + { type: "assert-identity", key: "B", path: "C.md" }, + { type: "assert-identity", key: "C", path: "A.md" }, { type: "assert-consistent", verify: (s: AssertableState): void => { s.assertFileNotExists("temp-a.md") .assertFileCount(3) - .assertAnyFileContains("content-c") - .assertAnyFileContains("content-a") - .assertAnyFileContains("content-b"); + .assertContent("A.md", "content-c") + .assertContent("B.md", "content-a") + .assertContent("C.md", "content-b"); } } ] diff --git a/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts b/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts index fc6a00a7..cee99580 100644 --- a/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts +++ b/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts @@ -27,7 +27,7 @@ export const renameCreateConflictTest: TestDefinition = { verify: (s: AssertableState): void => { s.assertFileCount(2) .assertContent("B.md", "hi") - .assertContent("B (1).md", "hi"); + .assertContent(s.conflictPath("B.md"), "hi"); } } ] diff --git a/frontend/deterministic-tests/src/tests/rename-swap.test.ts b/frontend/deterministic-tests/src/tests/rename-swap.test.ts index 9910e8ef..4298d2bc 100644 --- a/frontend/deterministic-tests/src/tests/rename-swap.test.ts +++ b/frontend/deterministic-tests/src/tests/rename-swap.test.ts @@ -5,7 +5,7 @@ export const renameSwapTest: TestDefinition = { description: "Client 0 has A.md and B.md synced. Goes offline and swaps them using " + "a temp file: A.md -> temp.md, B.md -> A.md, temp.md -> B.md. " + - "When Client 0 reconnects, both contents should exist across two files.", + "When Client 0 reconnects, both UUIDs must follow their files to the swapped paths.", clients: 2, steps: [ { type: "create", client: 0, path: "A.md", content: "content-a" }, @@ -23,6 +23,8 @@ export const renameSwapTest: TestDefinition = { } }, + { type: "remember-identity", key: "A", path: "A.md" }, + { type: "remember-identity", key: "B", path: "B.md" }, { type: "disable-sync", client: 0 }, { type: "rename", client: 0, oldPath: "A.md", newPath: "temp.md" }, { type: "rename", client: 0, oldPath: "B.md", newPath: "A.md" }, @@ -30,14 +32,16 @@ export const renameSwapTest: TestDefinition = { { type: "enable-sync", client: 0 }, { type: "barrier" }, + { type: "assert-identity", key: "A", path: "B.md" }, + { type: "assert-identity", key: "B", path: "A.md" }, { type: "assert-consistent", verify: (s: AssertableState): void => { s.assertFileNotExists("temp.md") .assertFileCount(2) - .assertAnyFileContains("content-b") - .assertAnyFileContains("content-a"); + .assertContent("A.md", "content-b") + .assertContent("B.md", "content-a"); } } ] diff --git a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts index 2a3b5de4..e78c2310 100644 --- a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts +++ b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts @@ -13,10 +13,8 @@ export const sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest: TestDef steps: [ { type: "enable-sync", client: 0 }, - // Create a deleted latest version before client 1 joins. - // Catch-up will advance MinCovered with a non-contiguous id, - // keeping client 1's create lastSeen low enough to exercise - // the server's same-doc merge path from the e2e failure. + // Seed a deleted document before client 1 bootstraps. Later creates + // at the reused path must still receive separate identities. { type: "create", client: 0, diff --git a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts index dee3a9ad..cb49d530 100644 --- a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts +++ b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts @@ -1,121 +1,50 @@ -import type { AssertableState } from "../utils/assertable-state"; import type { TestDefinition } from "../test-definition"; +// Historical registry name retained so old reproduction commands still work. export const sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest: TestDefinition = { description: - "Client B creates X with content C2; the server commits and " + - "broadcasts. Client A's WS is paused so the RemoteCreate buffers. " + - "Server is then paused so A's about-to-POST LocalCreate will " + - "hang. A creates X with content C1: file lands on disk, " + - "LocalCreate enqueues, drain starts the POST, the POST stalls " + - "at the paused server. A's WS is resumed: the buffered " + - "RemoteCreate for doc-X is delivered to A and enqueues behind " + - "the in-flight LocalCreate. Per the lazy-paths model, when " + - "the RemoteCreate is processed it observes that path X is " + - "occupied locally by A's pending-create bytes, so it tracks " + - "doc-X with `localPath = undefined` / `remoteRelativePath = " + - "X` and does NOT fetch content. The server is then resumed: " + - "A's LocalCreate POST returns. The server, finding X already " + - "taken by doc-X, replies with doc-X's existing documentId " + - "(typically a MergingUpdate carrying the merged bytes). A's " + - "processCreate handler detects that response.documentId " + - "matches the no-localPath record built from the RemoteCreate " + - "and collapses the two: it sets localPath = X on that " + - "record, writes the merged bytes, and resolves the pending " + - "create promise. Final state: exactly one file at X on both " + - "clients, both pointing at doc-X's documentId, content " + - "carrying both contributions, and no conflict-- " + - "stash anywhere.", + "Two initialized clients create distinct UUIDs at X.md while remote observation is held. V4 must preserve both identities/content after catchup; the server never merges creates by path.", clients: 2, steps: [ { type: "enable-sync", client: 0 }, { type: "enable-sync", client: 1 }, { type: "barrier" }, - - // Buffer broadcasts to client 0 (A) so client 1's create - // doesn't reach A's WS handler until we say so. - { type: "pause-websocket", client: 0 }, - - // Client 1 (B) commits doc-X at path X with content C2. - // The server commits, broadcasts (broadcast queued at A's - // paused WS). + // Freeze HTTP catchup as well as WS. Pausing only WS is insufficient. + { type: "pause-observation", client: 0 }, { type: "create", client: 1, path: "X.md", - content: "from-client-1 " + content: "from-client-1" }, { type: "sync", client: 1 }, - - // Pause the server so A's upcoming LocalCreate POST hangs. - // This holds A's drain on the in-flight POST while we - // release the WS so the RemoteCreate enqueues behind it. { type: "pause-server" }, - - // Client 0 (A) creates X locally with content C1. The - // file lands on A's disk; LocalCreate enqueues; drain - // starts the POST; POST stalls at the paused server. { type: "create", client: 0, path: "X.md", - content: "from-client-0 " + content: "from-client-0" }, - - // Release A's WS. The buffered RemoteCreate for doc-X is - // delivered to A and enqueues behind the in-flight - // LocalCreate. Whichever of (RemoteCreate processed first - // → no-localPath record, then LocalCreate POST returns - // with merging response that collapses) or (LocalCreate - // POST returns first with merging response that creates - // the canonical record, then RemoteCreate finds the doc - // already tracked by id and no-ops) actually plays out - // depends on the fine-grained interleaving the runtime - // produces, but both paths are required to converge to - // the same single-record same-docId state. - { type: "resume-websocket", client: 0 }, - - // Resume the server: A's LocalCreate POST completes. - // Server returns doc-X's existing documentId (MergingUpdate - // with merged content). processCreate runs the collapse - // path. + { type: "wait-for-observation", client: 0 }, + { type: "resume-observation", client: 0 }, { type: "resume-server" }, - { type: "barrier" }, - { type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(1); - state.assertFileExists("X.md"); - // Server-side merge of the two text creates must - // carry both contributions through to the - // converged file. - state.assertContains( - "X.md", - "from-client-0", - "from-client-1" + verify: (state) => { + state + .assertFileCount(2) + .assertContent("X.md", "from-client-1"); + state.assertContent( + state.conflictPath("X.md"), + "from-client-0" ); - // The lazy-paths collapse path must not leave a - // conflict-- stash on either client. - for (const path of state.files.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a converged client: ${path}` - ); - } - } - for (const perClient of state.clientFiles) { - for (const path of perClient.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a per-client view: ${path}` - ); - } - } - } } + }, + { + type: "assert-markers", + markers: ["from-client-0", "from-client-1"] } ] }; diff --git a/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts b/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts index ac8ed3ed..e3504098 100644 --- a/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts +++ b/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts @@ -33,11 +33,8 @@ export const selfMergePendingRenameAliasesSecondCreateTest: TestDefinition = { { type: "enable-sync", client: 1 }, { type: "barrier" }, - // Pause client 0's WS so its MinCovered watermark stays at 0 - // through the whole bug sequence. The merge condition the - // server is going to fire is `creation_vuid > last_seen`; with - // a non-zero gap the same-device second create gets merged - // into the same-device first create. + // Delay socket notifications while client 0 continues local work. + // HTTP catchup must preserve the two generations at the reused path. { type: "pause-websocket", client: 0 }, // Client 1 commits a doc to push the server's vuid above 0. diff --git a/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts b/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts index 70a2fc8c..e783cc4a 100644 --- a/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts +++ b/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts @@ -1,7 +1,7 @@ import type { AssertableState } from "../utils/assertable-state"; import type { TestDefinition } from "../test-definition"; -export const updateDoesNotSurvivesRemoteDeleteTest: TestDefinition = { +export const updateDoesNotSurviveRemoteDeleteTest: TestDefinition = { description: "Client 0 deletes a file while client 1 edits it offline. Client 0 syncs the delete first, then client 1 reconnects. Deletes always win.", clients: 2, diff --git a/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts b/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts index ef6cd771..58bebb44 100644 --- a/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts +++ b/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts @@ -8,6 +8,9 @@ export const userParenthesizedFileNotDeletedTest: TestDefinition = { clients: 2, steps: [ { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + { type: "disable-sync", client: 1 }, { type: "create", @@ -40,7 +43,8 @@ export const userParenthesizedFileNotDeletedTest: TestDefinition = { .assertFileCount(3) .assertFileExists("Chapter.bin") .assertFileExists("Chapter (1).bin") - .assertFileExists("Chapter (2).bin"); + .assertContent("Chapter (1).bin", "chapter one notes") + .assertContent(state.conflictPath("Chapter.bin"), "chapter one notes"); } } ] diff --git a/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts b/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts index 063faff4..9615b5aa 100644 --- a/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts +++ b/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts @@ -28,7 +28,7 @@ export const watermarkAdvancesOnSkipTest: TestDefinition = { { type: "assert-consistent", verify: (s: AssertableState): void => { - s.assertFileCount(1).assertFileExists("doc.md"); + s.assertFileCount(2).assertFileExists("doc.md").assertAnyFileContains("from client 0", "from client 1"); } } ] diff --git a/frontend/deterministic-tests/src/utils/assertable-state.ts b/frontend/deterministic-tests/src/utils/assertable-state.ts index 7c6f192c..c208fa13 100644 --- a/frontend/deterministic-tests/src/utils/assertable-state.ts +++ b/frontend/deterministic-tests/src/utils/assertable-state.ts @@ -1,12 +1,103 @@ -import type { ClientState } from "../test-definition"; +import assert from "node:assert/strict"; +import type { ClientState, ExpectedDocument } from "../test-definition"; export class AssertableState { public readonly files: Map; public readonly clientFiles: Map[]; + public readonly manifests: Record[]; + public readonly canonical: Record; + public readonly bytes: Map; public constructor(state: ClientState) { this.files = state.files; this.clientFiles = state.clientFiles; + this.manifests = state.manifests; + this.canonical = state.canonical; + this.bytes = state.bytes; + } + + public documentId(path: string): string { + const id = Object.keys(this.canonical).find( + (id) => this.canonical[id] === path + ); + if (!id) throw new Error(`No document identity for ${path}`); + return id; + } + + public assertDocuments( + expected: readonly ExpectedDocument[], + identities: Map + ): this { + this.assertFileCount(expected.length); + assert.equal( + new Set(expected.map((doc) => doc.key)).size, + expected.length, + "Duplicate expected identity key" + ); + const next = new Map(identities); + const paths = new Set(); + for (const doc of expected) { + const path = doc.conflict ? this.conflictPath(doc.path) : doc.path; + assert( + !paths.has(path), + `Two expected documents resolved to ${path}` + ); + paths.add(path); + this.assertBytes( + path, + typeof doc.content === "string" + ? new TextEncoder().encode(doc.content) + : new Uint8Array(doc.content) + ); + const id = this.documentId(path); + const recorded = next.get(doc.key); + if (recorded !== undefined) this.assertIdentity(path, recorded); + else { + assert( + ![...next.values()].includes(id), + `New document ${doc.key} reused an existing or deleted UUID` + ); + next.set(doc.key, id); + } + } + // A failed check must not establish a partial identity oracle. + for (const [key, id] of next) identities.set(key, id); + return this; + } + + public conflictPath(original: string): string { + const dot = original.lastIndexOf("."); + const split = + dot > original.lastIndexOf("/") + 1 ? dot : original.length; + const matches = Object.entries(this.canonical).filter( + ([id, path]) => + path === + `${original.slice(0, split)} (conflict ${id})${original.slice(split)}` + ); + if (matches.length !== 1) + throw new Error( + `Expected one UUID-derived conflict path for ${original}, found ${matches.length}` + ); + return matches[0][1]; + } + + public assertIdentity(path: string, id: string): this { + if (this.documentId(path) !== id) + throw new Error( + `Document identity changed at ${path}: expected ${id}, got ${this.documentId(path)}` + ); + return this; + } + + public assertBytes(path: string, expected: Uint8Array): this { + const actual = this.bytes.get(path); + if ( + !actual || + actual.length !== expected.length || + actual.some((byte, i) => byte !== expected[i]) + ) + throw new Error(`Byte mismatch at ${path}`); + return this; } public assertFileCount(expected: number): this { diff --git a/frontend/deterministic-tests/src/v4-regressions.ts b/frontend/deterministic-tests/src/v4-regressions.ts new file mode 100644 index 00000000..75841fc4 --- /dev/null +++ b/frontend/deterministic-tests/src/v4-regressions.ts @@ -0,0 +1,253 @@ +import type { TestDefinition, TestStep } from "./test-definition"; + +const online: TestStep[] = [ + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" } +]; +const offline: TestStep[] = [ + { type: "disable-sync", client: 0 }, + { type: "disable-sync", client: 1 } +]; +export const V4_TESTS: Record = { + "v4-text-byte-and-component-boundaries": { + clients: 2, + steps: [ + ...online, + { type: "create", client: 0, path: "empty.md", content: "" }, + { + type: "create", + client: 0, + path: "bom.md", + content: "\ufefffirst\r\nlast" + }, + { + type: "create", + client: 0, + path: `${"a".repeat(252)}.md`, + content: "255 ASCII bytes" + }, + { + type: "create", + client: 0, + path: `${"é".repeat(126)}.md`, + content: "255 UTF-8 bytes" + }, + { type: "barrier" }, + { + type: "assert-files", + count: 4, + expected: { + "empty.md": "", + "bom.md": "\ufefffirst\r\nlast", + [`${"a".repeat(252)}.md`]: "255 ASCII bytes", + [`${"é".repeat(126)}.md`]: "255 UTF-8 bytes" + } + } + ] + }, + "v4-binary-bytes-and-independent-identities": { + clients: 2, + steps: [ + ...online, + ...offline, + { + type: "create-bytes", + client: 0, + path: "nested/blob.bin", + bytes: [0, 255, 254, 128, 13, 10] + }, + { + type: "create-bytes", + client: 1, + path: "nested/blob.bin", + bytes: [0, 254, 255, 129, 10, 13] + }, + ...online, + { + type: "assert-consistent", + verify: (state) => { + state + .assertFileCount(2) + .assertBytes( + "nested/blob.bin", + new Uint8Array([0, 255, 254, 128, 13, 10]) + ); + state.assertBytes( + state.conflictPath("nested/blob.bin"), + new Uint8Array([0, 254, 255, 129, 10, 13]) + ); + } + } + ] + }, + "v4-three-file-cycle-preserves-identities": { + clients: 2, + steps: [ + ...online, + { type: "create", client: 0, path: "a.md", content: "A" }, + { type: "create", client: 0, path: "b.md", content: "B" }, + { type: "create", client: 0, path: "c.md", content: "C" }, + { type: "barrier" }, + { type: "remember-identity", key: "a", path: "a.md" }, + { type: "remember-identity", key: "b", path: "b.md" }, + { type: "remember-identity", key: "c", path: "c.md" }, + { type: "disable-sync", client: 0 }, + { type: "delay-notifications", client: 0 }, + { type: "rename", client: 0, oldPath: "a.md", newPath: "temp.md" }, + { type: "rename", client: 0, oldPath: "b.md", newPath: "a.md" }, + { type: "rename", client: 0, oldPath: "c.md", newPath: "b.md" }, + { type: "rename", client: 0, oldPath: "temp.md", newPath: "c.md" }, + { + type: "update", + client: 0, + path: "c.md", + content: "A edited offline" + }, + { type: "flush-notifications", client: 0 }, + { type: "enable-sync", client: 0 }, + { type: "barrier" }, + { type: "assert-identity", key: "a", path: "c.md" }, + { type: "assert-identity", key: "b", path: "a.md" }, + { type: "assert-identity", key: "c", path: "b.md" }, + { + type: "assert-files", + count: 3, + expected: { + "a.md": "B", + "b.md": "C", + "c.md": "A edited offline" + }, + absent: ["temp.md"] + } + ] + }, + "v4-case-alias-and-file-directory-conflicts": { + clients: 2, + steps: [ + ...online, + ...offline, + { + type: "create", + client: 0, + path: "folder/note.md", + content: "nested file" + }, + { + type: "create", + client: 0, + path: "README.md", + content: "uppercase file" + }, + { + type: "create", + client: 1, + path: "folder", + content: "blocking file" + }, + { + type: "create", + client: 1, + path: "readme.md", + content: "lowercase file" + }, + ...online, + { + type: "assert-consistent", + verify: (state) => { + state + .assertFileCount(4) + .assertContent("folder/note.md", "nested file") + .assertContent("README.md", "uppercase file"); + state.assertAnyFileContains( + "blocking file", + "lowercase file" + ); + } + }, + { + type: "assert-markers", + markers: [ + "nested file", + "blocking file", + "uppercase file", + "lowercase file" + ] + } + ] + }, + "v4-server-sigkill-retains-acknowledged-state": { + clients: 2, + steps: [ + ...online, + { + type: "create", + client: 0, + path: "a.md", + content: "durable bytes" + }, + { type: "barrier" }, + { type: "remember-identity", key: "a", path: "a.md" }, + ...offline, + { type: "crash-server" }, + { type: "restart-server" }, + ...online, + { type: "assert-identity", key: "a", path: "a.md" }, + { + type: "assert-files", + count: 1, + expected: { "a.md": "durable bytes" } + } + ] + } +}; + +for (const kind of ["create", "content", "manifest"] as const) { + for (const point of ["before", "after"] as const) { + V4_TESTS[`v4-${kind}-${point}-commit-network-loss`] = { + clients: 2, + steps: [ + ...online, + ...(kind === "content" + ? ([ + { + type: "create", + client: 0, + path: "file.md", + content: "old bytes" + }, + { type: "barrier" }, + { + type: "remember-identity", + key: "updated-file", + path: "file.md" + } + ] as TestStep[]) + : []), + { type: "drop-response", client: 0, kind, point }, + { + type: kind === "content" ? "update" : "create", + client: 0, + path: "file.md", + content: "must survive retry" + }, + { type: "wait-for-response-drop", client: 0 }, + { type: "barrier" }, + ...(kind === "content" + ? [ + { + type: "assert-identity", + key: "updated-file", + path: "file.md" + } as TestStep + ] + : []), + { + type: "assert-files", + count: 1, + expected: { "file.md": "must survive retry" } + } + ] + }; + } +} diff --git a/frontend/deterministic-tests/tsconfig.json b/frontend/deterministic-tests/tsconfig.json index 7558871d..47e92325 100644 --- a/frontend/deterministic-tests/tsconfig.json +++ b/frontend/deterministic-tests/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "baseUrl": ".", "strict": true, + "skipLibCheck": true, "target": "ES2022", "module": "CommonJS", "esModuleInterop": true, diff --git a/frontend/deterministic-tests/webpack.config.js b/frontend/deterministic-tests/webpack.config.js index 6aee1547..12c1653f 100644 --- a/frontend/deterministic-tests/webpack.config.js +++ b/frontend/deterministic-tests/webpack.config.js @@ -2,8 +2,9 @@ const path = require("path"); const webpack = require("webpack"); module.exports = { - entry: "./src/cli.ts", + entry: { cli: "./src/cli.ts", "crash-worker": "./src/crash-worker.ts" }, target: "node", + externals: { "sync-client": "commonjs sync-client" }, mode: "production", optimization: { minimize: false @@ -21,7 +22,7 @@ module.exports = { }, output: { globalObject: "this", - filename: "cli.js", + filename: "[name].js", path: path.resolve(__dirname, "dist") }, plugins: [ diff --git a/frontend/local-client-cli/README.md b/frontend/local-client-cli/README.md index e91322f9..9c519a68 100644 --- a/frontend/local-client-cli/README.md +++ b/frontend/local-client-cli/README.md @@ -62,7 +62,6 @@ vaultlink \ | `--ignore-pattern ` | - | Glob pattern to ignore (repeatable) | | `--websocket-retry-interval-ms ` | `3500` | WebSocket reconnection interval | | `--log-level ` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR | -| `--line-endings ` | `auto` | Line ending style: auto, lf, crlf | | `-q, --quiet` | - | Suppress startup banner for non-interactive use | | `-h, --help` | - | Show help | | `-V, --version` | - | Show version | @@ -96,12 +95,7 @@ vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ --log-level DEBUG --quiet ``` -Force LF line endings (useful for cross-platform vaults): - -```bash -vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ - --line-endings lf -``` +File bytes, including line endings, are preserved on every platform. ## Docker Deployment @@ -211,10 +205,21 @@ docker build -f local-client-cli/Dockerfile -t vault-link-cli:test . 1. Creates `.vaultlink` directory for sync metadata 2. Performs initial sync of local files to server -3. Watches filesystem for changes using Node's `fs.watch` +3. Uses filesystem events to wake a fresh scan, including atomic editor saves 4. Syncs changes bidirectionally in real-time 5. Handles graceful shutdown on SIGINT/SIGTERM +The complete client record (database, settings, queued logical notifications and +server history checkpoint) is saved atomically in `.vaultlink/sync-data.json`. +Existing database-only files migrate on the next save. Invalid or unreadable +metadata is reported instead of silently starting with empty state. + +File moves use an exclusive copy followed by deletion of the source. An +interruption may leave both files; the next scan reconciles the visible files. +Raw watcher events do not distinguish user edits from sync writes. A rename +preserves identity when the scan finds a unique matching nonempty content hash; +a simultaneous rename and edit may instead appear as deletion and creation. + ## License MIT diff --git a/frontend/local-client-cli/src/args.ts b/frontend/local-client-cli/src/args.ts index 34d839b1..cb6ce421 100644 --- a/frontend/local-client-cli/src/args.ts +++ b/frontend/local-client-cli/src/args.ts @@ -2,9 +2,6 @@ import { Command, Option } from "commander"; import packageJson from "../package.json"; import { LogLevel } from "sync-client"; -export const LINE_ENDING_MODES = ["auto", "lf", "crlf"] as const; -export type LineEndingMode = (typeof LINE_ENDING_MODES)[number]; - interface CliArgs { remoteUri: string; token: string; @@ -17,7 +14,6 @@ interface CliArgs { health?: string; enableTelemetry?: boolean; quiet: boolean; - lineEndings: LineEndingMode; } const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"]; @@ -131,15 +127,6 @@ export function parseArgs(argv: string[]): CliArgs { "[OPTIONAL] Suppress startup banner for non-interactive use" ).env("VAULTLINK_QUIET") ) - .addOption( - new Option( - "--line-endings ", - "[OPTIONAL] Line ending style: auto (platform default), lf, crlf" - ) - .default("auto") - .choices([...LINE_ENDING_MODES]) - .env("VAULTLINK_LINE_ENDINGS") - ) .addHelpText( "after", ` @@ -173,7 +160,6 @@ Environment variables: const health = opts.health as string | undefined; const enableTelemetry = opts.enableTelemetry as boolean | undefined; const quiet = (opts.quiet as boolean | undefined) ?? false; - const lineEndingsStr = (opts.lineEndings as string | undefined) ?? "auto"; /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */ const requiredLocalPath = requireOption(localPath, "localPath"); @@ -203,15 +189,6 @@ Environment variables: } const logLevel = logLevelUpper; - const isLineEndingMode = (value: string): value is LineEndingMode => - (LINE_ENDING_MODES as readonly string[]).includes(value); - if (!isLineEndingMode(lineEndingsStr)) { - throw new Error( - `Invalid line endings mode '${lineEndingsStr}'. Valid values are: ${LINE_ENDING_MODES.join(", ")}` - ); - } - const lineEndings = lineEndingsStr; - return { localPath: requiredLocalPath, remoteUri: requiredRemoteUri, @@ -223,7 +200,6 @@ Environment variables: logLevel, health, enableTelemetry, - quiet, - lineEndings + quiet }; } diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index 31c81d5c..5f2a8c9f 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -1,19 +1,18 @@ import * as path from "path"; import * as fs from "fs/promises"; import * as fsSync from "fs"; -import type { NetworkConnectionStatus } from "sync-client"; +import type { NetworkConnectionStatus, Logger } from "sync-client"; import { SyncClient, DEFAULT_SETTINGS, - Logger, LogLevel, LogLine, - type SyncSettings, - type StoredDatabase + type SyncSettings } from "sync-client"; -import { parseArgs, type LineEndingMode } from "./args"; +import { parseArgs } from "./args"; import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem"; import { FileWatcher } from "./file-watcher"; +import { ClientPersistence } from "./client-persistence"; import { formatLogLine } from "./logger-formatter"; import packageJson from "../package.json"; @@ -41,7 +40,6 @@ const LOG_LEVEL_ORDER = { function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { return (logLine: LogLine): void => { if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) { - // eslint-disable-next-line no-console console.log(formatLogLine(logLine)); } }; @@ -50,17 +48,6 @@ function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { const HEALTH_CHECK_INTERVAL_MS = 30 * 1000; const PROGRESS_LOG_INTERVAL_MS = 2000; -function resolveLineEndings(mode: LineEndingMode): string { - switch (mode) { - case "lf": - return "\n"; - case "crlf": - return "\r\n"; - case "auto": - return process.platform === "win32" ? "\r\n" : "\n"; - } -} - async function main(): Promise { const args = parseArgs(process.argv); const absolutePath = path.resolve(args.localPath); @@ -96,12 +83,6 @@ async function main(): Promise { emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`); emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`); emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`); - if (args.lineEndings !== "auto") { - emitBoot( - LogLevel.INFO, - `Line endings: ${args.lineEndings.toUpperCase()}` - ); - } } const dataDir = path.join(absolutePath, VAULTLINK_DIR); @@ -133,33 +114,7 @@ async function main(): Promise { const client = await SyncClient.create({ fs: fileSystem, - persistence: { - load: async () => { - let database: Partial | undefined = undefined; - try { - const content = await fs.readFile(dataFile, "utf-8"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - database = JSON.parse(content) as Partial; - } catch { - emitBoot( - LogLevel.WARNING, - `Cannot read data file at ${dataFile}` - ); - } - - return { - settings, - database - }; - }, - save: async ({ database: persistedDatabase }) => { - await fs.writeFile( - dataFile, - JSON.stringify(persistedDatabase, null, 2) - ); - } - }, - nativeLineEndings: resolveLineEndings(args.lineEndings) + persistence: new ClientPersistence(dataFile, settings) }); if (args.health !== undefined) { @@ -232,7 +187,6 @@ async function main(): Promise { client.logger.info(`${signal} received, shutting down gracefully`); fileWatcher.stop(); - await client.waitUntilFinished(); await client.destroy(); if (totalSyncOps > 0) { @@ -279,7 +233,6 @@ async function main(): Promise { } main().catch((error: unknown) => { - // eslint-disable-next-line no-console console.error( `Unexpected error: ${error instanceof Error ? error.message : String(error)}` ); diff --git a/frontend/local-client-cli/src/client-persistence.test.ts b/frontend/local-client-cli/src/client-persistence.test.ts new file mode 100644 index 00000000..b400d50f --- /dev/null +++ b/frontend/local-client-cli/src/client-persistence.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import fs from "node:fs/promises"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { ClientPersistence } from "./client-persistence"; +import type { StoredClient } from "sync-client"; + +test("CLI persistence retains the whole record and migrates database-only saves", async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-metadata-") + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const file = path.join(directory, "state.json"); + const store = new ClientPersistence(file, { token: "current-token" }); + assert.deepEqual(await store.load(), { + settings: { token: "current-token" } + }); + const record: StoredClient = { + settings: { token: "old-token", vaultName: "vault" }, + database: { lastSeenUpdateId: 12 }, + localChanges: [ + { type: "delete", path: "removed.md", changeId: "queued" } + ], + localChangesVaultKey: "vault", + historyCheckpoint: { vaultKey: "vault", checkpoint: "12:incarnation" } + }; + await store.save(record); + assert.deepEqual(await store.load(), { + ...record, + settings: { ...record.settings, token: "current-token" } + }); + const legacy = { + fileManifest: { fileManifestId: 0, entries: {} }, + lastSeenUpdateId: 0 + }; + await fs.writeFile(file, JSON.stringify(legacy)); + assert.deepEqual((await store.load()).database, legacy); + await fs.writeFile(file, "{truncated"); + await assert.rejects(store.load(), SyntaxError); + await fs.rm(file); + await fs.mkdir(file); + await assert.rejects(store.load(), /EISDIR/); +}); + +for (const afterReplacement of [false, true]) { + test(`failed metadata save ${afterReplacement ? "after" : "before"} replacement leaves a complete record`, async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-save-") + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const store = new ClientPersistence( + path.join(directory, "state.json"), + {} + ); + const old: StoredClient = { + historyCheckpoint: { vaultKey: "v", checkpoint: "1:old" } + }; + const next: StoredClient = { + ...old, + localChanges: [{ type: "delete", path: "note.md", changeId: "new" }] + }; + await store.save(old); + const rename = fs.rename; + t.mock.method( + fs, + "rename", + async (...args: Parameters) => { + if (afterReplacement) await rename(...args); + throw new Error("interrupted replacement"); + } + ); + await assert.rejects(store.save(next), /interrupted replacement/); + assert.deepEqual(await store.load(), { + ...(afterReplacement ? next : old), + settings: {} + }); + assert.deepEqual(await fs.readdir(directory), ["state.json"]); + }); +} diff --git a/frontend/local-client-cli/src/client-persistence.ts b/frontend/local-client-cli/src/client-persistence.ts new file mode 100644 index 00000000..ce421654 --- /dev/null +++ b/frontend/local-client-cli/src/client-persistence.ts @@ -0,0 +1,67 @@ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { randomUUID } from "node:crypto"; +import type { + PersistenceProvider, + StoredClient, + StoredDatabase, + SyncSettings +} from "sync-client"; + +/** Only client metadata needs an atomic replacement; user files do not. */ +export class ClientPersistence implements PersistenceProvider { + public constructor( + private readonly file: string, + private readonly settings: Partial + ) {} + + public async load(): Promise { + const content = await fs + .readFile(this.file, "utf8") + .catch((error: unknown) => { + if ( + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) + return undefined; + throw error; + }); + if (content === undefined) return { settings: this.settings }; + const parsed: unknown = JSON.parse(content); + if ( + parsed === null || + typeof parsed !== "object" || + Array.isArray(parsed) + ) + throw new Error("Invalid client metadata"); + // The sync engine validates and migrates the persisted fields. + const data = parsed as StoredClient & Partial; + // Earlier CLI versions saved the database alone. + const stored: StoredClient = + "fileManifest" in data ? { database: data } : data; + return { + ...stored, + settings: { ...stored.settings, ...this.settings } + }; + } + + public async save(data: StoredClient): Promise { + await fs.mkdir(path.dirname(this.file), { recursive: true }); + const temporary = `${this.file}.${randomUUID()}.tmp`; + try { + const handle = await fs.open(temporary, "wx", 0o600); + try { + await handle.writeFile(JSON.stringify(data)); + // Flush the complete value before exposing its name. A crash can + // keep the previous value, but must not expose truncated JSON. + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, this.file); + } finally { + await fs.rm(temporary, { force: true }); + } + } +} diff --git a/frontend/local-client-cli/src/file-watcher.ts b/frontend/local-client-cli/src/file-watcher.ts index c273a412..541192f7 100644 --- a/frontend/local-client-cli/src/file-watcher.ts +++ b/frontend/local-client-cli/src/file-watcher.ts @@ -1,93 +1,54 @@ import Watcher from "watcher"; -import * as path from "path"; -import type { SyncClient, RelativePath } from "sync-client"; +import * as path from "node:path"; +import type { SyncClient } from "sync-client"; import { toUnixPath, matchesGlob } from "./path-utils"; +/** OS events include sync writes and editor temporary-file replacements. They + * wake reconciliation; only a fresh scan determines the current namespace. */ export class FileWatcher { private watcher: Watcher | undefined; - private isRunning = false; - private readonly ignorePatterns: string[]; public constructor( private readonly basePath: string, private readonly client: SyncClient, - ignorePatterns: string[] = [] - ) { - this.ignorePatterns = ignorePatterns; - } + private readonly ignorePatterns: string[] = [] + ) {} public start(): void { - if (this.isRunning) { - return; - } - - this.isRunning = true; - + if (this.watcher) return; this.watcher = new Watcher(this.basePath, { recursive: true, - renameDetection: true, - renameTimeout: 125, + renameDetection: false, ignoreInitial: true, - ignore: (filePath: string): boolean => this.shouldIgnore(filePath) + ignore: (filePath: string): boolean => + this.ignorePatterns.some((pattern) => + matchesGlob(this.toRelativePath(filePath), pattern) + ) }); - - this.watcher.on("add", (filePath: string) => { - this.handleCreate(this.toRelativePath(filePath)); + this.watcher.on("all", (_event: string, filePath: string) => { + void this.client + .syncLocallyUpdatedFile({ + relativePath: this.toRelativePath(filePath) + }) + .catch((error: unknown) => { + this.client.logger.error( + `File notification failed: ${error}` + ); + }); }); - - this.watcher.on("change", (filePath: string) => { - this.handleChange(this.toRelativePath(filePath)); + this.watcher.on("error", (error: Error) => { + this.client.logger.error(`File watcher failed: ${error.message}`); }); - - this.watcher.on("unlink", (filePath: string) => { - this.handleDelete(this.toRelativePath(filePath)); - }); - - this.watcher.on("rename", (oldPath: string, newPath: string) => { - this.handleRename( - this.toRelativePath(oldPath), - this.toRelativePath(newPath) - ); - }); - this.client.logger.info("File watcher started"); } public stop(): void { - if (this.watcher !== undefined) { - this.watcher.close(); - this.watcher = undefined; - } - this.isRunning = false; + this.watcher?.close(); + this.watcher = undefined; this.client.logger.info("File watcher stopped"); } - private shouldIgnore(filePath: string): boolean { - const rel = toUnixPath(path.relative(this.basePath, filePath)); - return this.ignorePatterns.some((pattern) => matchesGlob(rel, pattern)); - } - - private handleCreate(relativePath: RelativePath): void { - this.client.syncLocallyCreatedFile(relativePath); - } - - private handleChange(relativePath: RelativePath): void { - this.client.syncLocallyUpdatedFile({ relativePath }); - } - - private handleDelete(relativePath: RelativePath): void { - this.client.syncLocallyDeletedFile(relativePath); - } - - private handleRename(oldPath: RelativePath, newPath: RelativePath): void { - this.client.logger.info(`File renamed: ${oldPath} -> ${newPath}`); - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - } - - private toRelativePath(absolutePath: string): RelativePath { + private toRelativePath(absolutePath: string): string { return toUnixPath(path.relative(this.basePath, absolutePath)); } } diff --git a/frontend/local-client-cli/src/healthcheck.ts b/frontend/local-client-cli/src/healthcheck.ts index d7211c88..b2c296a0 100644 --- a/frontend/local-client-cli/src/healthcheck.ts +++ b/frontend/local-client-cli/src/healthcheck.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -/* eslint-disable no-console */ + /** * Healthcheck script for Docker container diff --git a/frontend/local-client-cli/src/node-filesystem.test.ts b/frontend/local-client-cli/src/node-filesystem.test.ts index 85fb7a0f..3cab1147 100644 --- a/frontend/local-client-cli/src/node-filesystem.test.ts +++ b/frontend/local-client-cli/src/node-filesystem.test.ts @@ -11,10 +11,13 @@ test("NodeFileSystemOperations - read and write files", async () => { try { const content = new TextEncoder().encode("Hello, world!"); - await fsOps.write("test.txt", content); + await fsOps.write("test.txt", { content }); - const readContent = await fsOps.read("test.txt"); - assert.equal(new TextDecoder().decode(readContent), "Hello, world!"); + const readContent = await fsOps.readSnapshot("test.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "Hello, world!" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -27,10 +30,13 @@ test("NodeFileSystemOperations - create nested directories with forward slashes" try { const content = new TextEncoder().encode("Nested file"); // Always use forward slashes in API - await fsOps.write("dir1/dir2/test.txt", content); + await fsOps.write("dir1/dir2/test.txt", { content }); - const readContent = await fsOps.read("dir1/dir2/test.txt"); - assert.equal(new TextDecoder().decode(readContent), "Nested file"); + const readContent = await fsOps.readSnapshot("dir1/dir2/test.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "Nested file" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -43,7 +49,9 @@ test("NodeFileSystemOperations - exists with forward slashes", async () => { try { assert.equal(await fsOps.exists("test.txt"), false); - await fsOps.write("test.txt", new TextEncoder().encode("test")); + await fsOps.write("test.txt", { + content: new TextEncoder().encode("test") + }); assert.equal(await fsOps.exists("test.txt"), true); } finally { @@ -51,16 +59,20 @@ test("NodeFileSystemOperations - exists with forward slashes", async () => { } }); -test("NodeFileSystemOperations - delete with forward slashes", async () => { +test("NodeFileSystemOperations - deleteFile with forward slashes", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); try { - await fsOps.write("test.txt", new TextEncoder().encode("test")); + await fsOps.write("test.txt", { + content: new TextEncoder().encode("test") + }); assert.equal(await fsOps.exists("test.txt"), true); - await fsOps.delete("test.txt"); + await fsOps.deleteFile("test.txt"); assert.equal(await fsOps.exists("test.txt"), false); + + await fsOps.deleteFile("test.txt"); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -72,15 +84,18 @@ test("NodeFileSystemOperations - rename with forward slashes", async () => { try { const content = new TextEncoder().encode("test content"); - await fsOps.write("old.txt", content); + await fsOps.write("old.txt", { content }); await fsOps.rename("old.txt", "new.txt"); assert.equal(await fsOps.exists("old.txt"), false); assert.equal(await fsOps.exists("new.txt"), true); - const readContent = await fsOps.read("new.txt"); - assert.equal(new TextDecoder().decode(readContent), "test content"); + const readContent = await fsOps.readSnapshot("new.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "test content" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -92,7 +107,7 @@ test("NodeFileSystemOperations - rename to nested path with forward slashes", as try { const content = new TextEncoder().encode("test content"); - await fsOps.write("old.txt", content); + await fsOps.write("old.txt", { content }); await fsOps.rename("old.txt", "dir1/dir2/new.txt"); @@ -103,42 +118,21 @@ test("NodeFileSystemOperations - rename to nested path with forward slashes", as } }); -test("NodeFileSystemOperations - getFileSize", async () => { +test("NodeFileSystemOperations - stat", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); try { const content = new TextEncoder().encode("Hello!"); - await fsOps.write("test.txt", content); + await fsOps.write("test.txt", { content }); - const size = await fsOps.getFileSize("test.txt"); + const size = (await fsOps.stat("test.txt"))!.size; assert.equal(size, content.length); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); -test("NodeFileSystemOperations - atomicUpdateText", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); - const fsOps = new NodeFileSystemOperations(tempDir); - - try { - await fsOps.write("test.txt", new TextEncoder().encode("Hello")); - - const result = await fsOps.atomicUpdateText("test.txt", (current) => ({ - text: current.text + " World", - cursors: [] - })); - - assert.equal(result, "Hello World"); - - const content = await fsOps.read("test.txt"); - assert.equal(new TextDecoder().decode(content), "Hello World"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } -}); - test("NodeFileSystemOperations - handles paths with forward slashes on all platforms", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); @@ -148,15 +142,51 @@ test("NodeFileSystemOperations - handles paths with forward slashes on all platf const testPath = "deep/nested/directory/file.txt"; const content = new TextEncoder().encode("test"); - await fsOps.write(testPath, content); + await fsOps.write(testPath, { content }); assert.equal(await fsOps.exists(testPath), true); - const readContent = await fsOps.read(testPath); - assert.equal(new TextDecoder().decode(readContent), "test"); + const readContent = await fsOps.readSnapshot(testPath); + assert.equal(new TextDecoder().decode(readContent!.content), "test"); - await fsOps.delete(testPath); + await fsOps.deleteFile(testPath); assert.equal(await fsOps.exists(testPath), false); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test("NodeFileSystemOperations - delete prunes empty directory trees", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); + const fsOps = new NodeFileSystemOperations(tempDir); + + try { + await fs.mkdir(path.join(tempDir, "parent", "child"), { + recursive: true + }); + + await fsOps.delete("parent"); + assert.equal(await fsOps.exists("parent"), false); + + await fsOps.delete("parent"); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test("NodeFileSystemOperations - delete does not remove files", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); + const fsOps = new NodeFileSystemOperations(tempDir); + + try { + await fsOps.write("parent/file.txt", { + content: new TextEncoder().encode("preserve me") + }); + + await assert.rejects(async () => fsOps.delete("parent")); + assert.equal(await fsOps.exists("parent/file.txt"), true); + await assert.rejects(async () => fsOps.delete("parent/file.txt")); + assert.equal(await fsOps.exists("parent/file.txt"), true); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index ba95ab6a..dadc8c5f 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -1,201 +1,188 @@ -import * as fs from "fs/promises"; -import type { Dirent } from "fs"; -import * as path from "path"; -import { randomUUID } from "crypto"; -import type { - FileSystemOperations, - RelativePath, - TextWithCursors -} from "sync-client"; -import { toUnixPath } from "./path-utils"; +import assert from "node:assert/strict"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { constants, type Stats } from "node:fs"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +function hasCode(error: unknown, ...codes: string[]): boolean { + return ( + error instanceof Error && + "code" in error && + typeof error.code === "string" && + codes.includes(error.code) + ); +} -// VaultLink's per-vault metadata directory. Holds the persisted sync database -// and the tmp files atomicWrite renames into place; the matching `${VAULTLINK_DIR}/**` -// ignore pattern keeps everything in here invisible to the file watcher. export const VAULTLINK_DIR = ".vaultlink"; +/** Preflights reject unsafe paths and links. This is not a sandbox against a + * hostile process concurrently replacing ancestor directories. */ export class NodeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly basePath: string) { } - - public async listFilesRecursively( - directory: RelativePath | undefined - ): Promise { - const files: RelativePath[] = []; - await this.walkDirectory(directory ?? "", files); - return files; + public constructor(private readonly root: string) {} + public async stat( + relative: string + ): ReturnType { + const info = await this.info(await this.resolve(relative)); + return info + ? { + kind: info.isFile() + ? ("file" as const) + : ("directory" as const), + size: info.size + } + : undefined; } - - public async read(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); + public async exists(relative: string): Promise { + return (await this.stat(relative)) !== undefined; + } + public async readSnapshot( + relative: string + ): Promise { + const absolute = await this.resolve(relative); + const handle = await fs + .open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW) + .catch((error: unknown) => { + if (hasCode(error, "ENOENT", "ENOTDIR")) return undefined; + throw error; + }); + if (!handle) return undefined; try { - return await fs.readFile(fullPath); - } catch (error) { - throw new Error( - `Failed to read file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` + const before = await handle.stat({ bigint: true }); + assert( + before.isFile() && before.nlink === 1n, + "Not an independent regular file" ); - } - } - - public async write( - relativePath: RelativePath, - content: Uint8Array - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - const dir = path.dirname(fullPath); - - try { - await fs.mkdir(dir, { recursive: true }); - await this.atomicWrite(fullPath, content); - } catch (error) { - throw new Error( - `Failed to write file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` + const content = new Uint8Array(await handle.readFile()); + const after = await handle.stat({ bigint: true }); + const atPath = await fs.lstat(absolute, { bigint: true }); + assert( + before.ino === after.ino && + after.ino === atPath.ino && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs, + "File changed during read" ); + return { content }; + } finally { + await handle.close(); } } - - public async atomicUpdateText( - relativePath: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - - try { - const currentContent = await fs.readFile(fullPath, "utf-8"); - const result = updater({ text: currentContent, cursors: [] }); - await this.atomicWrite(fullPath, result.text, "utf-8"); - return result.text; - } catch (error) { - throw new Error( - `Failed to atomically update file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); + public async listFilesRecursively(root = ""): Promise { + const result: string[] = []; + for (const name of await fs.readdir(await this.resolve(root))) { + const relative = root ? `${root}/${name}` : name; + const entry = await this.stat(relative); + assert(entry, `File changed during scan: ${relative}`); + if (entry.kind === "directory") + result.push(...(await this.listFilesRecursively(relative))); + else result.push(relative); } + return result.sort(); } - - public async getFileSize(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - const stats = await fs.stat(fullPath); - return stats.size; - } catch (error) { - throw new Error( - `Failed to get file size for ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async exists(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.access(fullPath); - return true; - } catch { - return false; - } - } - - public async createDirectory(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.mkdir(fullPath, { recursive: false }); - } catch (error) { - throw new Error( - `Failed to create directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async delete(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.unlink(fullPath); - } catch (error) { - throw new Error( - `Failed to delete file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - const oldFullPath = path.join(this.basePath, oldPath); - const newFullPath = path.join(this.basePath, newPath); - const newDir = path.dirname(newFullPath); - - try { - await fs.mkdir(newDir, { recursive: true }); - await fs.rename(oldFullPath, newFullPath); - } catch (error) { - throw new Error( - `Failed to rename file from ${oldFullPath} to ${newFullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - private async atomicWrite( - fullPath: string, - content: Uint8Array | string, - encoding?: BufferEncoding - ): Promise { - const tmpDir = path.join(this.basePath, VAULTLINK_DIR); - await fs.mkdir(tmpDir, { recursive: true }); - const tmpPath = path.join(tmpDir, `atomic-write-${randomUUID()}.tmp`); - try { - await fs.writeFile(tmpPath, content, encoding); - const fd = await fs.open(tmpPath, "r"); + public async createDirectory(relative: string): Promise { + await this.resolve(relative); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + const next = path.join(current, component); try { - await fd.datasync(); - } finally { - await fd.close(); + await fs.mkdir(next); + } catch (error) { + if (!hasCode(error, "EEXIST")) throw error; } - await fs.rename(tmpPath, fullPath); - await this.syncDirectory(path.dirname(fullPath)); + assert((await fs.lstat(next)).isDirectory()); + current = next; + } + } + public async write( + relative: string, + snapshot: FileSnapshot + ): Promise { + const destination = await this.resolve(relative); + await this.createDirectory( + path.posix.dirname(relative) === "." + ? "" + : path.posix.dirname(relative) + ); + await fs.writeFile(destination, snapshot.content, { + flag: "wx", + mode: 0o600 + }); + } + public async rename(from: string, to: string): Promise { + const snapshot = await this.readSnapshot(from); + assert(snapshot, `Missing source: ${from}`); + // Node's rename overwrites destinations on POSIX. Exclusive copy followed + // by unlink leaves ordinary discoverable files if interrupted. + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + assert( + current && + Buffer.from(current.content).equals( + Buffer.from(snapshot.content) + ), + `Source changed during move: ${from}` + ); + await this.deleteFile(from); + } + public async delete(relative: string): Promise { + assert(relative, "Cannot prune root"); + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isDirectory(), "Cannot delete regular file"); + for (const child of await fs.readdir(absolute)) + await this.delete(`${relative}/${child}`); + await fs.rmdir(absolute); + } + public async deleteFile(relative: string): Promise { + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); + await fs.unlink(absolute).catch((error: unknown) => { + if (!hasCode(error, "ENOENT")) throw error; + }); + } + private async resolve(relative: string): Promise { + assert( + relative === "" || + relative + .split("/") + .every( + (p) => + p !== "" && + p !== "." && + p !== ".." && + !p.includes("\\") && + !p.includes("\0") && + (process.platform !== "win32" || !p.includes(":")) + ), + "Unsafe path" + ); + assert( + (await fs.lstat(this.root)).isDirectory(), + "Vault root must be a directory, not a symlink" + ); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + current = path.join(current, component); + const entry = await this.info(current); + if (entry) + assert( + entry.isDirectory() || + (entry.isFile() && entry.nlink === 1), + "Symlink, hardlink or special file" + ); + } + return current; + } + private async info(absolute: string): Promise { + try { + return await fs.lstat(absolute); } catch (error) { - await fs.unlink(tmpPath).catch(() => undefined); + if (hasCode(error, "ENOENT", "ENOTDIR")) return undefined; throw error; } } - - // Make the rename durable by fsync'ing the destination's parent directory. - // Skipped on Windows: fsync on a directory handle isn't supported there - private async syncDirectory(dir: string): Promise { - if (process.platform === "win32") { - return; - } - const fd = await fs.open(dir, "r"); - try { - await fd.sync(); - } finally { - await fd.close(); - } - } - - private async walkDirectory( - relativePath: string, - files: RelativePath[] - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - let entries: Dirent[] = []; - - try { - entries = await fs.readdir(fullPath, { withFileTypes: true }); - } catch (error) { - throw new Error( - `Failed to read directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - - for (const entry of entries) { - const entryName = entry.name; - const entryRelativePath = path.join(relativePath, entryName); - - if (entry.isDirectory()) { - await this.walkDirectory(entryRelativePath, files); - } else if (entry.isFile()) { - // Always return forward slashes - files.push(toUnixPath(entryRelativePath)); - } - } - } } diff --git a/frontend/local-client-cli/webpack.config.js b/frontend/local-client-cli/webpack.config.js index 9226b9dc..d96048e0 100644 --- a/frontend/local-client-cli/webpack.config.js +++ b/frontend/local-client-cli/webpack.config.js @@ -7,6 +7,7 @@ module.exports = { healthcheck: "./src/healthcheck.ts" }, target: "node", + externals: { bufferutil: "bufferutil", "utf-8-validate": "utf-8-validate" }, mode: "production", optimization: { minimize: false diff --git a/frontend/obsidian-plugin/README.md b/frontend/obsidian-plugin/README.md index 68e10a83..de3f3027 100644 --- a/frontend/obsidian-plugin/README.md +++ b/frontend/obsidian-plugin/README.md @@ -1,63 +1,33 @@ -# Obsidian Sample Plugin +# VaultLink Obsidian plugin -This is a sample plugin for Obsidian (https://obsidian.md). +Syncs an Obsidian vault with the VaultLink server, including active editor text, +selections, remote cursors, history and connection status. Use an API v5 server. +The vault configuration directory, Git metadata and trash are ignored by default. -This project uses TypeScript to provide type checking and documentation. -The repo depends on the latest plugin API (obsidian.d.ts) in TypeScript Definition format, which contains TSDoc comments describing what it does. +From `frontend`: -**Note:** The Obsidian API is still in early alpha and is subject to change at any time! +```sh +npm run build -w sync-client -w vault-link-obsidian-plugin +npm run test -w vault-link-obsidian-plugin +``` -This sample plugin demonstrates some of the basic functionality the plugin API can do. +The production bundle is in `obsidian-plugin/dist`: `main.js`, `manifest.json` +and `styles.css`. Copy those files into the vault's plugin directory to install. -- Adds a ribbon icon, which shows a Notice when clicked. -- Adds a command "Open Sample Modal" which opens a Modal. -- Adds a plugin setting tab to the settings page. -- Registers a global click event and output 'click' to the console. -- Registers a global interval which logs 'setInterval' to the console. +Desktop Obsidian shares the CLI's Node filesystem and atomic metadata adapters. +The complete client record remains in the plugin's `data.json`; it includes +settings, sync state, queued logical notifications and the server history +checkpoint. Mobile uses Obsidian's storage and plugin data APIs. Because mobile +writes overwrite existing files, exclusive creation uses a disposable input for +`DataAdapter.copy` in `.vault-link-sync`. This is not a filesystem journal or a +recovery payload. -## First time developing plugins? +Raw create/modify/delete events wake a fresh scan, including events caused by +sync and editor replacements. Obsidian's logical rename event preserves file +and directory move identities. Editor snapshots include unsaved text and +selections; content application updates both disk and the active editor. -Quick starting guide for new plugin devs: - -- Check if [someone already developed a plugin for what you want](https://obsidian.md/plugins)! There might be an existing plugin similar enough that you can partner up with. -- Make a copy of this repo as a template with the "Use this template" button (login to GitHub if you don't see it). -- Clone your repo to a local development folder. For convenience, you can place this folder in your `.obsidian/plugins/your-plugin-name` folder. -- Install NodeJS, then run `npm i` in the command line under your repo folder. -- Run `npm run dev` to compile your plugin from `main.ts` to `main.js`. -- Make changes to `main.ts` (or create new `.ts` files). Those changes should be automatically compiled into `main.js`. -- Reload Obsidian to load the new version of your plugin. -- Enable plugin in settings window. -- For updates to the Obsidian API run `npm update` in the command line under your repo folder. - -## Releasing new releases - -- Update your `manifest.json` with your new version number, such as `1.0.1`, and the minimum Obsidian version required for your latest release. -- Update your `versions.json` file with `"new-plugin-version": "minimum-obsidian-version"` so older versions of Obsidian can download an older version of your plugin that's compatible. -- Create new GitHub release using your new version number as the "Tag version". Use the exact version number, don't include a prefix `v`. See here for an example: https://github.com/obsidianmd/obsidian-sample-plugin/releases -- Upload the files `manifest.json`, `main.js`, `styles.css` as binary attachments. Note: The manifest.json file must be in two places, first the root path of your repository and also in the release. -- Publish the release. - -> You can simplify the version bump process by running `npm version patch`, `npm version minor` or `npm version major` after updating `minAppVersion` manually in `manifest.json`. -> The command will bump version in `manifest.json` and `package.json`, and add the entry for the new version to `versions.json` - -## Adding your plugin to the community plugin list - -- Check the [plugin guidelines](https://docs.obsidian.md/Plugins/Releasing/Plugin+guidelines). -- Publish an initial version. -- Make sure you have a `README.md` file in the root of your repo. -- Make a pull request at https://github.com/obsidianmd/obsidian-releases to add your plugin. - -## How to use - -- Clone this repo. -- Make sure your NodeJS is at least v16 (`node --version`). -- `npm i` or `yarn` to install dependencies. -- `npm run dev` to start compilation in watch mode. - -## Manually installing the plugin - -- Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`. - -## API Documentation - -See https://github.com/obsidianmd/obsidian-api +Adapter tests cover unsaved text, cursor restoration, exclusive creation and +storage failures. The repository E2E runner builds the plugin and tests its +filesystem adapter against the server alongside real CLI processes. These tests +do not launch Obsidian itself. diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index d24e537b..f8d510eb 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "echo \"no tests defined\" && exit 0", + "test": "tsx --test 'src/**/*.test.ts'", "version": "node version-bump.mjs" }, "keywords": [], diff --git a/frontend/obsidian-plugin/src/mobile-file-system.ts b/frontend/obsidian-plugin/src/mobile-file-system.ts new file mode 100644 index 00000000..43869bcd --- /dev/null +++ b/frontend/obsidian-plugin/src/mobile-file-system.ts @@ -0,0 +1,130 @@ +import type { DataAdapter } from "obsidian"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +/** Mobile storage has no Node filesystem. DataAdapter.copy provides its + * documented exclusive-create primitive; writeBinary overwrites existing files. */ +export class MobileFileSystemOperations implements FileSystemOperations { + public constructor(private readonly adapter: DataAdapter) {} + + public async stat(path: string): ReturnType { + const entry = await this.adapter.stat(this.check(path)); + return entry + ? { + kind: + entry.type === "file" + ? ("file" as const) + : ("directory" as const), + size: entry.size + } + : undefined; + } + public async exists(path: string): Promise { + return (await this.stat(path)) !== undefined; + } + public async listFilesRecursively(root = ""): Promise { + const entries = await this.adapter.list(this.check(root)); + const files = [...entries.files]; + for (const folder of entries.folders) + files.push(...(await this.listFilesRecursively(folder))); + return files; + } + public async readSnapshot(path: string): Promise { + this.check(path); + const before = await this.adapter.stat(path); + if (!before) return undefined; + if (before.type !== "file") throw new Error(`Not a file: ${path}`); + const content = new Uint8Array(await this.adapter.readBinary(path)); + const after = await this.adapter.stat(path); + if ( + !after || + before.size !== after.size || + before.mtime !== after.mtime || + before.ctime !== after.ctime + ) + throw new Error(`File changed during read: ${path}`); + return { content }; + } + public async createDirectory(path: string): Promise { + this.check(path); + let current = ""; + for (const part of path.split("/").filter(Boolean)) { + current = current ? `${current}/${part}` : part; + if (!(await this.exists(current))) { + try { + await this.adapter.mkdir(current); + } catch (error) { + if ((await this.stat(current))?.kind !== "directory") + throw error; + } + } + if ((await this.stat(current))?.kind !== "directory") + throw new Error(`Not a directory: ${current}`); + } + } + public async write(path: string, snapshot: FileSnapshot): Promise { + this.check(path); + await this.createDirectory(path.split("/").slice(0, -1).join("/")); + await this.createDirectory(".vault-link-sync"); + const temporary = `.vault-link-sync/${crypto.randomUUID()}.tmp`; + try { + await this.adapter.writeBinary( + temporary, + new Uint8Array(snapshot.content).buffer + ); + await this.adapter.copy(temporary, path); + } finally { + if (await this.adapter.exists(temporary)) + await this.adapter.remove(temporary); + } + } + public async rename(from: string, to: string): Promise { + this.check(from); + this.check(to); + const snapshot = await this.readSnapshot(from); + if (!snapshot) throw new Error(`Missing source: ${from}`); + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + if ( + !current || + current.content.length !== snapshot.content.length || + current.content.some((byte, i) => byte !== snapshot.content[i]) + ) + throw new Error(`Source changed during move: ${from}`); + await this.deleteFile(from); + } + public async deleteFile(path: string): Promise { + const entry = await this.stat(path); + if (!entry) return; + if (entry.kind !== "file") throw new Error(`Not a file: ${path}`); + await this.adapter.remove(path); + } + public async delete(path: string): Promise { + if (!path) throw new Error("Cannot prune root"); + const entry = await this.stat(path); + if (!entry) return; + if (entry.kind !== "directory") + throw new Error(`Not a directory: ${path}`); + const children = await this.adapter.list(path); + if (children.files.length) + throw new Error(`Directory contains files: ${path}`); + for (const folder of children.folders) await this.delete(folder); + await this.adapter.rmdir(path, false); + } + private check(path: string): string { + if ( + path !== "" && + path + .split("/") + .some( + (part) => + !part || + part === "." || + part === ".." || + part.includes("\\") || + part.includes("\0") + ) + ) + throw new Error(`Unsafe path: ${path}`); + return path; + } +} diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.test.ts b/frontend/obsidian-plugin/src/obsidian-file-system.test.ts new file mode 100644 index 00000000..5faf5330 --- /dev/null +++ b/frontend/obsidian-plugin/src/obsidian-file-system.test.ts @@ -0,0 +1,167 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as fs from "node:fs/promises"; +import { constants } from "node:fs"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import type { DataAdapter, Editor, EditorSelection } from "obsidian"; +import { NodeFileSystemOperations } from "../../local-client-cli/src/node-filesystem"; +import { ObsidianFileSystemOperations } from "./obsidian-file-system"; +import { MobileFileSystemOperations } from "./mobile-file-system"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const text = (content: Uint8Array) => new TextDecoder().decode(content); + +test("Obsidian snapshots include unsaved editor text and restore merged selections", async (t) => { + const root = await fs.mkdtemp(path.join(tmpdir(), "vaultlink-editor-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const disk = new NodeFileSystemOperations(root); + await disk.write("note.md", { content: bytes("saved") }); + let value = "unsaved\ntext"; + let selections: EditorSelection[] = [ + { anchor: { line: 0, ch: 2 }, head: { line: 1, ch: 3 } } + ]; + const editor = { + getValue: () => value, + listSelections: () => selections, + setValue: (next: string) => { + value = next; + }, + setSelections: (next: EditorSelection[]) => { + selections = next; + } + } as Editor; + const adapter = new ObsidianFileSystemOperations(disk, () => ({ + path: "note.md", + editor + })); + const snapshot = await adapter.readSnapshot("note.md"); + assert.equal(text(snapshot!.content), value); + assert.deepEqual(snapshot!.cursors, [ + { id: 0, position: 2 }, + { id: 1, position: 11 } + ]); + assert.equal((await adapter.stat("note.md"))!.size, bytes(value).length); + await assert.rejects( + adapter.write("note.md", { content: bytes("collision") }) + ); + assert.equal(value, "unsaved\ntext"); + await adapter.deleteFile("note.md"); + await adapter.write("note.md", { + content: bytes("new\ntext"), + cursors: [ + { id: 1, position: 7 }, + { id: 0, position: 1 } + ] + }); + assert.equal(value, "new\ntext"); + assert.equal(text((await disk.readSnapshot("note.md"))!.content), value); + assert.deepEqual(selections, [ + { anchor: { line: 0, ch: 1 }, head: { line: 1, ch: 3 } } + ]); + value = "edited before rename"; + selections = [{ anchor: { line: 0, ch: 1 }, head: { line: 0, ch: 4 } }]; + await adapter.rename("note.md", "moved.md"); + assert.equal(text((await disk.readSnapshot("moved.md"))!.content), value); + assert.equal(await adapter.readSnapshot("note.md"), undefined); +}); + +// A storage host with Obsidian's documented semantics: writes replace, copies +// reject existing destinations. No runtime Obsidian module is needed in Node. +async function mobile(root: string) { + const disk = new NodeFileSystemOperations(root); + const host = { + stat: async (p: string) => { + const entry = await disk.stat(p); + return entry + ? { + type: entry.kind === "file" ? "file" : "folder", + size: entry.size, + mtime: 1, + ctime: 1 + } + : null; + }, + exists: (p: string) => disk.exists(p), + readBinary: async (p: string) => + new Uint8Array((await disk.readSnapshot(p))!.content).buffer, + writeBinary: (p: string, content: ArrayBuffer) => + fs.writeFile(path.join(root, p), new Uint8Array(content)), + copy: (from: string, to: string) => + fs.copyFile( + path.join(root, from), + path.join(root, to), + constants.COPYFILE_EXCL + ), + mkdir: (p: string) => disk.createDirectory(p), + remove: (p: string) => disk.deleteFile(p), + rmdir: (p: string) => fs.rmdir(path.join(root, p)), + list: async (p: string) => { + const entries = await fs.readdir(path.join(root, p), { + withFileTypes: true + }); + return { + files: entries + .filter((e) => e.isFile()) + .map((e) => (p ? `${p}/${e.name}` : e.name)), + folders: entries + .filter((e) => e.isDirectory()) + .map((e) => (p ? `${p}/${e.name}` : e.name)) + }; + } + } as unknown as DataAdapter; + return { adapter: new MobileFileSystemOperations(host), host }; +} + +test("mobile writes use exclusive copies and never overwrite a competing writer", async (t) => { + const root = await fs.mkdtemp(path.join(tmpdir(), "vaultlink-mobile-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { adapter } = await mobile(root); + await adapter.createDirectory("folder/nested"); + const writes = await Promise.allSettled([ + adapter.write("folder/note.md", { content: bytes("first") }), + adapter.write("folder/note.md", { content: bytes("second") }) + ]); + assert.equal(writes.filter((w) => w.status === "fulfilled").length, 1); + const original = (await adapter.readSnapshot("folder/note.md"))!; + await adapter.write("destination.md", { content: bytes("occupied") }); + await assert.rejects(adapter.rename("folder/note.md", "destination.md")); + assert.deepEqual(await adapter.readSnapshot("folder/note.md"), original); + assert.equal( + text((await adapter.readSnapshot("destination.md"))!.content), + "occupied" + ); + await assert.rejects(adapter.delete("folder")); + await adapter.rename("folder/note.md", "moved.md"); + assert.deepEqual(await adapter.readSnapshot("moved.md"), original); + await adapter.delete("folder"); + assert.equal(await adapter.exists("folder"), false); + assert.deepEqual(await fs.readdir(path.join(root, ".vault-link-sync")), []); + await assert.rejects( + adapter.write("../escape", { content: bytes("bad") }), + /Unsafe/ + ); +}); + +test("mobile scans propagate storage errors and snapshots reject read races", async (t) => { + const root = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-mobile-read-") + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { adapter, host } = await mobile(root); + await adapter.write("note.md", { content: bytes("before") }); + const read = host.readBinary.bind(host); + t.mock.method(host, "readBinary", async (p: string) => { + const content = await read(p); + await fs.writeFile(path.join(root, p), "changed length"); + return content; + }); + await assert.rejects( + adapter.readSnapshot("note.md"), + /changed during read/ + ); + t.mock.method(host, "list", async () => { + throw new Error("storage unavailable"); + }); + await assert.rejects(adapter.listFilesRecursively(), /storage unavailable/); +}); diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index ceb8bc2a..2788ebff 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -1,176 +1,105 @@ -import type { Stat, Vault, Workspace } from "obsidian"; -import { MarkdownView, normalizePath } from "obsidian"; -import type { CursorPosition, TextWithCursors } from "sync-client"; +import type { Editor } from "obsidian"; import { utils, type FileSystemOperations, - type RelativePath + type FileSnapshot } from "sync-client"; import { getSelectionsFromEditor } from "./views/cursors/get-selections-from-editor"; +/** Editor contents and selections are one synchronous snapshot. Disk namespace + * operations remain the responsibility of the platform adapter. */ export class ObsidianFileSystemOperations implements FileSystemOperations { public constructor( - private readonly vault: Vault, - private readonly workspace: Workspace + private readonly disk: FileSystemOperations, + private readonly activeEditor: () => + | { path: string; editor: Editor } + | undefined ) {} - public async listFilesRecursively( - root: RelativePath | undefined - ): Promise { - // Let's implement this by hand because vault.adapter.listAllFiles doesn't always return all files. - const allFiles = []; - const remainingFolders = [root ?? this.vault.getRoot().path]; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - const folder = remainingFolders.pop(); - if (folder == undefined) { - break; - } - - // This would be a very bad idea to sync as it would mess with - // the integrity of the sync database. - if (folder.endsWith(".obsidian/plugins/vault-link/data.json")) { - continue; - } - - const files = await this.vault.adapter.list(normalizePath(folder)); - allFiles.push(...files.files); - remainingFolders.push(...files.folders); - } - - return allFiles; + public async listFilesRecursively(root?: string): Promise { + return this.disk.listFilesRecursively(root); + } + public async exists(path: string): Promise { + return this.disk.exists(path); + } + public async createDirectory(path: string): Promise { + return this.disk.createDirectory(path); + } + public async delete(path: string): Promise { + return this.disk.delete(path); + } + public async deleteFile(path: string): Promise { + return this.disk.deleteFile(path); + } + public async rename(from: string, to: string): Promise { + const snapshot = await this.readSnapshot(from); + if (!snapshot) throw new Error(`Missing source: ${from}`); + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + if ( + !current || + current.content.length !== snapshot.content.length || + current.content.some((byte, i) => byte !== snapshot.content[i]) + ) + throw new Error(`Source changed during move: ${from}`); + await this.deleteFile(from); } - public async read(path: RelativePath): Promise { - path = normalizePath(path); - const view = this.workspace.getActiveViewOfType(MarkdownView); - if (view?.file?.path === path) { - return new TextEncoder().encode(view.editor.getValue()); - } - - return new Uint8Array(await this.vault.adapter.readBinary(path)); + public async stat(path: string): ReturnType { + const entry = await this.disk.stat(path); + const view = this.activeEditor(); + if (entry?.kind === "file" && view?.path === path) + return { + ...entry, + size: new TextEncoder().encode(view.editor.getValue()) + .byteLength + }; + return entry; } - public async write(path: RelativePath, content: Uint8Array): Promise { - path = normalizePath(path); - - const view = this.workspace.getActiveViewOfType(MarkdownView); - if (view?.file?.path === path) { - const position = view.editor.getCursor(); - view.editor.setValue(new TextDecoder().decode(content)); - view.editor.setCursor(position); - return; + public async readSnapshot(path: string): Promise { + // Validate the disk path even when an editor is open. + const entry = await this.disk.stat(path); + if (!entry) return undefined; + const view = this.activeEditor(); + if (entry.kind === "file" && view?.path === path) { + return { + content: new TextEncoder().encode(view.editor.getValue()), + cursors: getSelectionsFromEditor(view.editor).flatMap( + ({ id, start, end }) => [ + { id: 2 * id, position: start }, + { id: 2 * id + 1, position: end } + ] + ) + }; } + return this.disk.readSnapshot(path); + } - return this.vault.adapter.writeBinary( - path, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - content.buffer as ArrayBuffer + public async write(path: string, snapshot: FileSnapshot): Promise { + await this.disk.write(path, snapshot); + const view = this.activeEditor(); + if (view?.path !== path) return; + const text = new TextDecoder().decode(snapshot.content); + view.editor.setValue(text); + const cursors = [...(snapshot.cursors ?? [])].sort( + (a, b) => a.id - b.id ); - } - - public async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - path = normalizePath(path); - - const view = this.workspace.getActiveViewOfType(MarkdownView); - - if (view?.file?.path === path) { - const text = view.editor.getValue(); - - const cursors: CursorPosition[] = getSelectionsFromEditor( - view.editor - ).flatMap(({ id, start: anchor, end: head }) => [ - { - id: 2 * id, - position: anchor - }, - { - id: 2 * id + 1, - position: head - } - ]); - - const result = updater({ + const selections = []; + for (let i = 0; i + 1 < cursors.length; i += 2) { + const anchor = utils.positionToLineAndColumn( text, - cursors + cursors[i].position + ); + const head = utils.positionToLineAndColumn( + text, + cursors[i + 1].position + ); + selections.push({ + anchor: { line: anchor.line, ch: anchor.column }, + head: { line: head.line, ch: head.column } }); - - if (result.text === text) { - return text; - } - - view.editor.setValue(result.text); - - const selections = []; - for (let i = 0; i < result.cursors.length / 2; i++) { - const from = result.cursors[2 * i]; - const to = result.cursors[2 * i + 1]; - const { line: fromLine, column: fromColumn } = - utils.positionToLineAndColumn(result.text, from.position); - - const { line: toLine, column: toColumn } = - utils.positionToLineAndColumn(result.text, to.position); - - selections.push({ - anchor: { line: fromLine, ch: fromColumn }, - head: { line: toLine, ch: toColumn } - }); - } - view.editor.setSelections(selections); - - return result.text; } - - return this.vault.adapter.process( - path, - (text) => - updater({ - text, - cursors: [] - }).text - ); - } - - public async getFileSize(path: RelativePath): Promise { - return (await this.statFile(path)).size; - } - - public async getModificationTime(path: RelativePath): Promise { - return new Date((await this.statFile(path)).mtime); - } - - public async exists(path: RelativePath): Promise { - return this.vault.adapter.exists(normalizePath(path)); - } - - public async createDirectory(path: RelativePath): Promise { - return this.vault.adapter.mkdir(normalizePath(path)); - } - - public async delete(path: RelativePath): Promise { - if (!(await this.vault.adapter.trashSystem(normalizePath(path)))) { - return this.vault.adapter.remove(normalizePath(path)); - } - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - return this.vault.adapter.rename(oldPath, newPath); - } - - private async statFile(path: string): Promise { - const file = await this.vault.adapter.stat(normalizePath(path)); - - if (!file) { - throw new Error(`File not found: ${path}`); - } - - return file; + if (selections.length) view.editor.setSelections(selections); } } diff --git a/frontend/obsidian-plugin/src/vault-link-plugin.ts b/frontend/obsidian-plugin/src/vault-link-plugin.ts index e222796b..de4326f2 100644 --- a/frontend/obsidian-plugin/src/vault-link-plugin.ts +++ b/frontend/obsidian-plugin/src/vault-link-plugin.ts @@ -1,11 +1,16 @@ import type { - MarkdownView, Editor, MarkdownFileInfo, TAbstractFile, WorkspaceLeaf } from "obsidian"; -import { Notice, Platform, Plugin, TFile } from "obsidian"; +import { + Notice, + Plugin, + TFile, + MarkdownView, + FileSystemAdapter +} from "obsidian"; import "../manifest.json"; import { HistoryView } from "./views/history/history-view"; import { StatusBar } from "./views/status-bar/status-bar"; @@ -14,11 +19,14 @@ import { StatusDescription } from "./views/status-description/status-description import { SyncClient, rateLimit, - DEFAULT_SETTINGS, Logger, - debugging + debugging, + type FileSystemOperations, + type PersistenceProvider, + type StoredClient } from "sync-client"; import { ObsidianFileSystemOperations } from "./obsidian-file-system"; +import { MobileFileSystemOperations } from "./mobile-file-system"; import { SyncSettingsTab } from "./views/settings/settings-tab"; import { EditorStatusDisplayManager } from "./views/editor-status-display-manager/editor-status-display-manager"; import { remoteCursorsTheme } from "./views/cursors/remote-cursor-theme"; @@ -38,7 +46,7 @@ export default class VaultLinkPlugin extends Plugin { () => Promise >(); - private readonly syncClient: SyncClient | undefined; + private syncClient: SyncClient | undefined; private settingsTab: SyncSettingsTab | undefined; public async onload(): Promise { @@ -53,14 +61,19 @@ export default class VaultLinkPlugin extends Plugin { // eslint-disable-next-line (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = this; - const client = await this.createSyncClient(); + this.register(() => { + // eslint-disable-next-line + (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; + }); + const client = (this.syncClient = await this.createSyncClient()); this.registerObsidianExtensions(client); this.registerEditorEvents(client); this.register(async () => { - await client.waitUntilFinished(); + this.syncClient = undefined; + this.rateLimitedUpdatesPerFile.clear(); await client.destroy(); }); @@ -116,23 +129,56 @@ export default class VaultLinkPlugin extends Plugin { } private async createSyncClient(): Promise { - DEFAULT_SETTINGS.ignorePatterns.push( - ".obsidian/**", - ".git/**", - ".trash/**", - "**/.DS_Store" - ); - - const client = await SyncClient.create({ - fs: new ObsidianFileSystemOperations( - this.app.vault, - this.app.workspace - ), - persistence: { + const { vault, workspace } = this.app; + let disk: FileSystemOperations; + let persistence: PersistenceProvider; + if (vault.adapter instanceof FileSystemAdapter) { + // Keep Node modules unevaluated on mobile. + const { NodeFileSystemOperations } = + require("../../local-client-cli/src/node-filesystem") as typeof import("../../local-client-cli/src/node-filesystem"); + const { ClientPersistence } = + require("../../local-client-cli/src/client-persistence") as typeof import("../../local-client-cli/src/client-persistence"); + const root = vault.adapter.getBasePath(); + disk = new NodeFileSystemOperations(root); + persistence = new ClientPersistence( + `${root}/${this.manifest.dir ?? `${vault.configDir}/plugins/${this.manifest.id}`}/data.json`, + {} + ); + } else { + disk = new MobileFileSystemOperations(vault.adapter); + persistence = { load: this.loadData.bind(this), save: this.saveData.bind(this) + }; + } + const client = await SyncClient.create({ + fs: new ObsidianFileSystemOperations(disk, () => { + const view = workspace.getActiveViewOfType(MarkdownView); + return view?.file + ? { path: view.file.path, editor: view.editor } + : undefined; + }), + persistence: { + load: async () => { + const stored = await persistence.load(); + return { + ...stored, + settings: { + ...stored?.settings, + ignorePatterns: [ + ...new Set([ + ...(stored?.settings?.ignorePatterns ?? []), + `${vault.configDir}/**`, + ".git/**", + ".trash/**", + "**/.DS_Store" + ]) + ] + } + }; + }, + save: (stored) => persistence.save(stored) }, - nativeLineEndings: Platform.isWin ? "\r\n" : "\n", ...(IS_DEBUG_BUILD ? { fetch: debugging.slowFetchFactory(1), @@ -142,7 +188,7 @@ export default class VaultLinkPlugin extends Plugin { }); if (IS_DEBUG_BUILD) { - debugging.logToConsole(client.logger); + debugging.logToConsole(client); } return client; @@ -196,11 +242,6 @@ export default class VaultLinkPlugin extends Plugin { this.register(() => { editorStatusDisplayManager.dispose(); }); - - this.register(() => { - // eslint-disable-next-line - (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; - }); } private addRibbonIcons(): void { @@ -233,7 +274,7 @@ export default class VaultLinkPlugin extends Plugin { ), this.app.vault.on("create", (file: TAbstractFile) => { if (file instanceof TFile) { - client.syncLocallyCreatedFile(file.path); + void this.rateLimitedUpdate(file.path, client); } }), this.app.vault.on("modify", async (file: TAbstractFile) => { @@ -242,17 +283,17 @@ export default class VaultLinkPlugin extends Plugin { } }), this.app.vault.on("delete", (file: TAbstractFile) => { - client.syncLocallyDeletedFile(file.path); + void this.rateLimitedUpdate(file.path, client); }), + // Vault.rename is a logical move (including folders). Engine file + // moves use copy/unlink, so they cannot generate this event. this.app.vault.on( "rename", (file: TAbstractFile, oldPath: string) => { - if (file instanceof TFile) { - client.syncLocallyUpdatedFile({ - oldPath, - relativePath: file.path - }); - } + void client.syncLocallyUpdatedFile({ + oldPath, + relativePath: file.path + }); } ) ].forEach((event) => { @@ -268,7 +309,8 @@ export default class VaultLinkPlugin extends Plugin { this.rateLimitedUpdatesPerFile.set( path, rateLimit(async () => { - client.syncLocallyUpdatedFile({ + if (this.syncClient !== client) return; + await client.syncLocallyUpdatedFile({ relativePath: path }); }, MIN_WAIT_BETWEEN_UPDATES_IN_MS) diff --git a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts index 409402a4..bfebfbf4 100644 --- a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts +++ b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts @@ -36,7 +36,7 @@ export function renderCursorsInFileExplorer( (parent) => { cursors.forEach((cursor) => { cursor.documentsWithCursors.forEach((document) => { - if (document.relativePath.startsWith(key)) { + if (document.relative_path.startsWith(key)) { parent.appendChild( createSpan({ text: cursor.userName, diff --git a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts index 4200a72a..d6650dcb 100644 --- a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts +++ b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts @@ -61,7 +61,7 @@ export class RemoteCursorsPluginValue implements PluginValue { return clientCursors.flatMap((cursor) => cursor.cursors.map((span) => ({ name: client.userName, - path: cursor.relativePath, + path: cursor.relative_path, deviceId: client.deviceId, isOutdated: client.isOutdated, span: { ...span } diff --git a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts index 5a5823c2..3f7ed7ca 100644 --- a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts @@ -266,8 +266,9 @@ export class SyncSettingsTab extends PluginSettingTab { new Notice("Checking connection to the server..."); new Notice( - (await this.syncClient.checkConnection()) - .serverMessage + ( + await this.syncClient.checkConnection() + ).serverMessage ); await this.statusDescription.updateConnectionState(); } else { @@ -434,6 +435,42 @@ export class SyncSettingsTab extends PluginSettingTab { containerEl.createEl("h3", { text: "Advanced" }); + new Setting(containerEl) + .setName("Request timeout (ms)") + .setDesc( + "The maximum time to wait for a server request, in milliseconds." + ) + .addText((input) => + input + .setValue( + this.syncClient + .getSettings() + .requestTimeoutMs.toString() + ) + .onChange(async (value) => { + if (value === "") { + return; + } + let parsedValue = Number(value); + if ( + !Number.isSafeInteger(parsedValue) || + parsedValue <= 0 + ) { + parsedValue = + this.syncClient.getSettings().requestTimeoutMs; + } + + if (value !== parsedValue.toString()) { + input.setValue(parsedValue.toString()); + } + + return this.syncClient.setSetting( + "requestTimeoutMs", + parsedValue + ); + }) + ); + new Setting(containerEl) .setName("Network retry interval (ms)") .setDesc( @@ -467,6 +504,47 @@ export class SyncSettingsTab extends PluginSettingTab { ); }) ); + + new Setting(containerEl) + .setName("Sync interval (ms)") + .setDesc( + "How often to reconcile when no changes are detected. Leave blank to rely on change notifications." + ) + .addText((input) => + input + .setPlaceholder("Disabled") + .setValue( + this.syncClient + .getSettings() + .syncIntervalMs?.toString() ?? "" + ) + .onChange(async (value) => { + if (value === "") { + return this.syncClient.setSetting( + "syncIntervalMs", + undefined + ); + } + + const parsedValue = Number(value); + if ( + !Number.isSafeInteger(parsedValue) || + parsedValue <= 0 + ) { + input.setValue( + this.syncClient + .getSettings() + .syncIntervalMs?.toString() ?? "" + ); + return; + } + + return this.syncClient.setSetting( + "syncIntervalMs", + parsedValue + ); + }) + ); } private setStatusDescriptionSubscription( diff --git a/frontend/obsidian-plugin/src/views/status-description/status-description.ts b/frontend/obsidian-plugin/src/views/status-description/status-description.ts index 6d8d74fe..53fea486 100644 --- a/frontend/obsidian-plugin/src/views/status-description/status-description.ts +++ b/frontend/obsidian-plugin/src/views/status-description/status-description.ts @@ -88,7 +88,7 @@ export class StatusDescription { text: ` and has indexed approximately ` }); container.createSpan({ - text: `${this.syncClient.syncedDocumentCount}`, + text: `${this.syncClient.documentCount}`, cls: "number" }); container.createSpan({ diff --git a/frontend/obsidian-plugin/tsconfig.json b/frontend/obsidian-plugin/tsconfig.json index 7ec2a9cd..dcad6394 100644 --- a/frontend/obsidian-plugin/tsconfig.json +++ b/frontend/obsidian-plugin/tsconfig.json @@ -4,6 +4,7 @@ "module": "ESNext", "target": "ES2023", "strict": true, + "skipLibCheck": true, "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, "lib": ["DOM", "ES2024"] diff --git a/frontend/obsidian-plugin/webpack.config.js b/frontend/obsidian-plugin/webpack.config.js index 12844fd7..0d287eb2 100644 --- a/frontend/obsidian-plugin/webpack.config.js +++ b/frontend/obsidian-plugin/webpack.config.js @@ -12,6 +12,11 @@ module.exports = (env, argv) => ({ ignored: "**/node_modules" }, externals: { + "node:assert/strict": "commonjs node:assert/strict", + "node:fs/promises": "commonjs node:fs/promises", + "node:fs": "commonjs node:fs", + "node:path": "commonjs node:path", + "node:crypto": "commonjs node:crypto", obsidian: "commonjs obsidian", electron: "commonjs electron", "@codemirror/autocomplete": "commonjs @codemirror/autocomplete", diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4d8218ba..5aa0bee9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,6 +9,7 @@ "sync-client", "obsidian-plugin", "test-client", + "deterministic-tests", "local-client-cli" ], "devDependencies": { @@ -21,26 +22,102 @@ "typescript-eslint": "8.41.0" } }, + "deterministic-tests": { + "version": "0.14.0", + "bin": { + "deterministic-tests": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^25.0.2", + "commander": "^14.0.2", + "sync-client": "file:../sync-client", + "ts-loader": "^9.5.4", + "tslib": "2.8.1", + "typescript": "5.9.3", + "webpack": "^5.103.0", + "webpack-cli": "^6.0.1" + } + }, + "deterministic-tests/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "deterministic-tests/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "deterministic-tests/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "local-client-cli": { "version": "0.14.0", - "dependencies": { - "commander": "^14.0.2", - "watcher": "^2.3.1" - }, "bin": { "vaultlink": "dist/cli.js" }, "devDependencies": { - "@types/node": "^24.8.1", + "@types/node": "^25.0.2", + "commander": "^14.0.2", "sync-client": "file:../sync-client", - "ts-loader": "^9.5.2", + "ts-loader": "^9.5.4", "tslib": "2.8.1", - "tsx": "^4.20.6", - "typescript": "5.8.3", - "webpack": "^5.99.9", + "tsx": "^4.21.0", + "typescript": "5.9.3", + "watcher": "^2.3.1", + "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } }, + "local-client-cli/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "local-client-cli/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "local-client-cli/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "node_modules/@codemirror/state": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz", @@ -52,20 +129,6 @@ "@marijn/find-cluster-break": "^1.0.0" } }, - "node_modules/@codemirror/view": { - "version": "6.38.1", - "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.1.tgz", - "integrity": "sha512-RmTOkE7hRU3OVREqFVITWHz6ocgBjv08GoePscAakgVQfciA3SGCEk7mb9IzwW61cKKmlTpHXG6DUE5Ubx+MGQ==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@codemirror/state": "^6.5.0", - "crelt": "^1.0.6", - "style-mod": "^4.1.0", - "w3c-keyname": "^2.2.4" - } - }, "node_modules/@discoveryjs/json-ext": { "version": "0.6.3", "dev": true, @@ -75,9 +138,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", - "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", "cpu": [ "ppc64" ], @@ -92,9 +155,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", - "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", "cpu": [ "arm" ], @@ -109,9 +172,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", - "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", "cpu": [ "arm64" ], @@ -126,9 +189,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", - "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", "cpu": [ "x64" ], @@ -143,9 +206,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", - "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", "cpu": [ "arm64" ], @@ -160,9 +223,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", - "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", "cpu": [ "x64" ], @@ -177,9 +240,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", - "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", "cpu": [ "arm64" ], @@ -194,9 +257,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", - "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", "cpu": [ "x64" ], @@ -211,9 +274,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", - "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", "cpu": [ "arm" ], @@ -228,9 +291,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", - "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", "cpu": [ "arm64" ], @@ -245,9 +308,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", - "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", "cpu": [ "ia32" ], @@ -262,9 +325,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", - "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", "cpu": [ "loong64" ], @@ -279,9 +342,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", - "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", "cpu": [ "mips64el" ], @@ -296,9 +359,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", - "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", "cpu": [ "ppc64" ], @@ -313,9 +376,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", - "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", "cpu": [ "riscv64" ], @@ -330,9 +393,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", - "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", "cpu": [ "s390x" ], @@ -347,9 +410,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", - "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", "cpu": [ "x64" ], @@ -364,9 +427,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", - "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", "cpu": [ "arm64" ], @@ -381,9 +444,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", - "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", "cpu": [ "x64" ], @@ -398,9 +461,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", - "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", "cpu": [ "arm64" ], @@ -415,9 +478,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", - "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", "cpu": [ "x64" ], @@ -432,9 +495,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", - "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", "cpu": [ "arm64" ], @@ -449,9 +512,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", - "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", "cpu": [ "x64" ], @@ -466,9 +529,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", - "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", "cpu": [ "arm64" ], @@ -483,9 +546,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", - "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", "cpu": [ "ia32" ], @@ -500,9 +563,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", - "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", "cpu": [ "x64" ], @@ -723,6 +786,8 @@ }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", "engines": { @@ -730,7 +795,9 @@ } }, "node_modules/@jridgewell/source-map": { - "version": "0.3.6", + "version": "0.3.11", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.11.tgz", + "integrity": "sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==", "dev": true, "license": "MIT", "dependencies": { @@ -739,14 +806,16 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.0", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.30", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", - "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, "license": "MIT", "dependencies": { @@ -759,7 +828,8 @@ "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz", "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -961,26 +1031,10 @@ "@types/tern": "*" } }, - "node_modules/@types/eslint": { - "version": "9.6.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/eslint-scope": { - "version": "3.7.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/eslint": "*", - "@types/estree": "*" - } - }, "node_modules/@types/estree": { - "version": "1.0.7", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "dev": true, "license": "MIT" }, @@ -1013,7 +1067,6 @@ "integrity": "sha512-8fz6oa6wEKZrhXWro/S3n2eRJqlRcIa6SlDh59FXJ5Wp5XRZ8B9ixpJDcjadHq47hMx0u+HW6SNa6LjJQ6NLtw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.41.0", @@ -1052,7 +1105,6 @@ "integrity": "sha512-gTtSdWX9xiMPA/7MV9STjJOOYtWwIJIYxkQxnSV1U3xcE+mnJSH3f6zI0RYP+ew66WSlZ5ed+h0VCxsvdC1jJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.41.0", "@typescript-eslint/types": "8.41.0", @@ -1453,7 +1505,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1485,7 +1536,6 @@ "version": "6.12.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1763,6 +1813,19 @@ "dev": true, "license": "MIT" }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.25", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.25.tgz", + "integrity": "sha512-gMmEShwwq7FJqMwvfRwvCl00v4kN+KOfJqXn+f4nrufak5gNHJOksd/60Dvjuz7sI8Y5WiSFBa8FEYr+zoyqCw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/big.js": { "version": "5.2.2", "dev": true, @@ -1804,7 +1867,9 @@ } }, "node_modules/browserslist": { - "version": "4.24.4", + "version": "4.29.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", + "integrity": "sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==", "dev": true, "funding": [ { @@ -1821,12 +1886,12 @@ } ], "license": "MIT", - "peer": true, "dependencies": { - "caniuse-lite": "^1.0.30001688", - "electron-to-chromium": "^1.5.73", - "node-releases": "^2.0.19", - "update-browserslist-db": "^1.1.1" + "baseline-browser-mapping": "^2.11.23", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.427", + "node-releases": "^2.0.55", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -1860,6 +1925,8 @@ }, "node_modules/buffer-from": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true, "license": "MIT" }, @@ -1983,7 +2050,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001707", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -2038,14 +2107,16 @@ } }, "node_modules/chokidar": { - "version": "4.0.3", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", "dev": true, "license": "MIT", "dependencies": { - "readdirp": "^4.0.1" + "readdirp": "^5.0.0" }, "engines": { - "node": ">= 14.16.0" + "node": ">= 20.19.0" }, "funding": { "url": "https://paulmillr.com/funding/" @@ -2232,6 +2303,7 @@ "version": "14.0.2", "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=20" @@ -2286,7 +2358,8 @@ "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -2442,10 +2515,15 @@ "node": ">=0.10" } }, + "node_modules/deterministic-tests": { + "resolved": "deterministic-tests", + "link": true + }, "node_modules/dettle": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/dettle/-/dettle-1.0.5.tgz", "integrity": "sha512-ZVyjhAJ7sCe1PNXEGveObOH9AC8QvMga3HJIghHawtG7mE4K5pW9nz/vDGAr/U7a3LWgdOzEE7ac9MURnyfaTA==", + "dev": true, "license": "MIT" }, "node_modules/dunder-proto": { @@ -2771,7 +2849,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.127", + "version": "1.5.433", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.433.tgz", + "integrity": "sha512-5lCAbyZBjtmUt/RAGHRqrL2q0oEFRThDAsZHHDn9XHa89Qw7gMYOeSicBTy+AHfvo0r6vwsZvqNJTQIQy1BLzA==", "dev": true, "license": "ISC" }, @@ -2889,12 +2969,14 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.18.1", + "version": "5.25.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.25.1.tgz", + "integrity": "sha512-nGXts5znJzmWPu+mIE9izCOzdg63oJca2mDzGWWTth7sr4aCToKcoyFVBQwN75Ij5Pf6p510EwkTqViTRzDV+w==", "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" + "tapable": "^2.3.3" }, "engines": { "node": ">=10.13.0" @@ -2928,7 +3010,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.6.0", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, "license": "MIT" }, @@ -2944,9 +3028,9 @@ } }, "node_modules/esbuild": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", - "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -2957,32 +3041,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.9", - "@esbuild/android-arm": "0.25.9", - "@esbuild/android-arm64": "0.25.9", - "@esbuild/android-x64": "0.25.9", - "@esbuild/darwin-arm64": "0.25.9", - "@esbuild/darwin-x64": "0.25.9", - "@esbuild/freebsd-arm64": "0.25.9", - "@esbuild/freebsd-x64": "0.25.9", - "@esbuild/linux-arm": "0.25.9", - "@esbuild/linux-arm64": "0.25.9", - "@esbuild/linux-ia32": "0.25.9", - "@esbuild/linux-loong64": "0.25.9", - "@esbuild/linux-mips64el": "0.25.9", - "@esbuild/linux-ppc64": "0.25.9", - "@esbuild/linux-riscv64": "0.25.9", - "@esbuild/linux-s390x": "0.25.9", - "@esbuild/linux-x64": "0.25.9", - "@esbuild/netbsd-arm64": "0.25.9", - "@esbuild/netbsd-x64": "0.25.9", - "@esbuild/openbsd-arm64": "0.25.9", - "@esbuild/openbsd-x64": "0.25.9", - "@esbuild/openharmony-arm64": "0.25.9", - "@esbuild/sunos-x64": "0.25.9", - "@esbuild/win32-arm64": "0.25.9", - "@esbuild/win32-ia32": "0.25.9", - "@esbuild/win32-x64": "0.25.9" + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" } }, "node_modules/escalade": { @@ -3010,7 +3094,6 @@ "integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3179,11 +3262,6 @@ "node": ">=0.10.0" } }, - "node_modules/eventemitter3": { - "version": "5.0.1", - "dev": true, - "license": "MIT" - }, "node_modules/events": { "version": "3.3.0", "dev": true, @@ -3555,7 +3633,9 @@ } }, "node_modules/fs-extra": { - "version": "11.3.0", + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", "dev": true, "license": "MIT", "dependencies": { @@ -3667,19 +3747,6 @@ "node": ">=6" } }, - "node_modules/get-tsconfig": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz", - "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -3759,11 +3826,6 @@ "node": ">=0.10.0" } }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "dev": true, - "license": "BSD-2-Clause" - }, "node_modules/globals": { "version": "14.0.0", "dev": true, @@ -4240,7 +4302,9 @@ } }, "node_modules/immutable": { - "version": "5.1.1", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.9.tgz", + "integrity": "sha512-m8nVez3rwrgmWxtLMt1ZYXB2Lv7OKYn/disyxAlSDYAlKSlFoPPfIAmAM/M5xqL4m4C/wAPw7S2/CNaUii1Hxg==", "dev": true, "license": "MIT" }, @@ -4548,6 +4612,21 @@ "node": ">=0.10.0" } }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -4566,11 +4645,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "dev": true, - "license": "MIT" - }, "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, @@ -4694,14 +4768,6 @@ "iconv-lite": "^0.4.15" } }, - "node_modules/loader-runner": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.11.5" - } - }, "node_modules/loader-utils": { "version": "2.0.4", "dev": true, @@ -4812,6 +4878,8 @@ }, "node_modules/merge-stream": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true, "license": "MIT" }, @@ -4838,24 +4906,15 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -4867,7 +4926,9 @@ } }, "node_modules/mini-css-extract-plugin": { - "version": "2.9.2", + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.10.2.tgz", + "integrity": "sha512-AOSS0IdEB95ayVkxn5oGzNQwqAi2J0Jb/kKm43t7H73s8+f5873g0yuj0PNvK4dO75mu5DHg4nlgp4k6Kga8eg==", "dev": true, "license": "MIT", "dependencies": { @@ -4889,7 +4950,6 @@ "version": "8.17.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4956,6 +5016,154 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minimizer-webpack-plugin": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.11.0.tgz", + "integrity": "sha512-2JVy2DY+iBtfRyrQB5cRM7lfurnJIi6jFTgLip/7FxDu0WxswrIRzr4fBySQ0X7HqM7BcwE28zm+HfQ5V6y4Cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.3", + "terser": "^5.51.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, + "@napi-rs/image": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "html-minifier-terser": { + "optional": true + }, + "imagemin": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, + "sharp": { + "optional": true + }, + "svgo": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimizer-webpack-plugin/node_modules/schema-utils": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -5038,22 +5246,16 @@ "license": "MIT", "optional": true }, - "node_modules/node-gyp-build": { - "version": "4.8.4", + "node_modules/node-releases": { + "version": "2.0.56", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.56.tgz", + "integrity": "sha512-x0InOIyzgdk+eyaWaRJFH5snEtiImgBgblZ2CyPrLmqqcuMQkEvcDPHbzqbD8eDsSeJbVOjn+crzyzHaM4D+/A==", "dev": true, "license": "MIT", - "optional": true, - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" + "engines": { + "node": ">=18" } }, - "node_modules/node-releases": { - "version": "2.0.19", - "dev": true, - "license": "MIT" - }, "node_modules/normalize-path": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", @@ -5170,21 +5372,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obsidian": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.10.2.tgz", - "integrity": "sha512-bX03YCHf06OTzI/D+QK71ajCPCmwr/cjxzlVXjQa10DjK5iHRWhtJJpp83arSCyayFMp23u+UHcY7hxcEx2Mvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/codemirror": "5.60.8", - "moment": "2.29.4" - }, - "peerDependencies": { - "@codemirror/state": "6.5.0", - "@codemirror/view": "6.38.1" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5329,32 +5516,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-queue": { - "version": "8.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^5.0.1", - "p-timeout": "^6.1.2" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-timeout": { - "version": "6.1.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-try": { "version": "2.2.0", "dev": true, @@ -5531,7 +5692,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.8", "picocolors": "^1.1.1", @@ -5648,6 +5808,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/promise-make-counter/-/promise-make-counter-1.0.2.tgz", "integrity": "sha512-FJAxTBWQuQoAs4ZOYuKX1FHXxEgKLEzBxUvwr4RoOglkTpOjWuM+RXsK3M9q5lMa8kjqctUrhwYeZFT4ygsnag==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-naked": "^3.0.2" @@ -5657,6 +5818,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/promise-make-naked/-/promise-make-naked-3.0.2.tgz", "integrity": "sha512-B+b+kQ1YrYS7zO7P7bQcoqqMUizP06BOyNSBEnB5VJKDSWo8fsVuDkfSmwdjF0JsRtaNh83so5MMFJ95soH5jg==", + "dev": true, "license": "MIT" }, "node_modules/pseudomap": { @@ -5743,14 +5905,6 @@ ], "license": "MIT" }, - "node_modules/randombytes": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -5775,11 +5929,13 @@ "license": "MIT" }, "node_modules/readdirp": { - "version": "4.1.2", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", + "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", "dev": true, "license": "MIT", "engines": { - "node": ">= 14.18.0" + "node": ">= 20.19.0" }, "funding": { "type": "individual", @@ -5944,16 +6100,6 @@ "node": ">= 0.10" } }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, "node_modules/resolve-url-loader": { "version": "5.0.0", "dev": true, @@ -6039,22 +6185,21 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.91.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.91.0.tgz", - "integrity": "sha512-aFOZHGf+ur+bp1bCHZ+u8otKGh77ZtmFyXDo4tlYvT7PWql41Kwd8wdkPqhhT+h2879IVblcHFglIMofsFd1EA==", + "version": "1.104.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.104.1.tgz", + "integrity": "sha512-yDA+1aIG3EHgN4V/BvuhCvu61FF6hEd4e+9DxikUm9U0CAGuvdIZ/UYy7qbOxjhbbWQraoyLlMuzdRGOSV5Bmw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "chokidar": "^4.0.0", - "immutable": "^5.0.2", + "chokidar": "^5.0.0", + "immutable": "^5.1.5", "source-map-js": ">=0.6.2 <2.0.0" }, "bin": { "sass": "sass.js" }, "engines": { - "node": ">=14.0.0" + "node": ">=20.19.0" }, "optionalDependencies": { "@parcel/watcher": "^2.4.1" @@ -6129,14 +6274,6 @@ "node": ">=10" } }, - "node_modules/serialize-javascript": { - "version": "6.0.2", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" - } - }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -6326,6 +6463,17 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -6415,14 +6563,16 @@ "node_modules/stubborn-fs": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", - "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==", + "dev": true }, "node_modules/style-mod": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/supports-color": { "version": "8.1.1", @@ -6454,11 +6604,17 @@ "link": true }, "node_modules/tapable": { - "version": "2.2.1", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true, "license": "MIT", "engines": { "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/term-size": { @@ -6552,12 +6708,14 @@ } }, "node_modules/terser": { - "version": "5.39.0", + "version": "5.51.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.51.2.tgz", + "integrity": "sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "@jridgewell/source-map": "^0.3.3", - "acorn": "^8.8.2", + "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, @@ -6569,14 +6727,15 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.14", + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", - "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "engines": { @@ -6590,22 +6749,50 @@ "webpack": "^5.1.0" }, "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, "@swc/core": { "optional": true }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, "esbuild": { "optional": true }, + "html-minifier-terser": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, "uglify-js": { "optional": true } } }, "node_modules/terser-webpack-plugin/node_modules/ajv": { - "version": "8.17.1", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6617,8 +6804,28 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/terser-webpack-plugin/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/terser-webpack-plugin/node_modules/ajv-keywords": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "dev": true, "license": "MIT", "dependencies": { @@ -6628,32 +6835,23 @@ "ajv": "^8.8.2" } }, - "node_modules/terser-webpack-plugin/node_modules/jest-worker": { - "version": "27.5.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, "node_modules/terser-webpack-plugin/node_modules/json-schema-traverse": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, "node_modules/terser-webpack-plugin/node_modules/schema-utils": { - "version": "4.3.0", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "ajv-keywords": "^5.1.0" }, "engines": { @@ -6671,15 +6869,6 @@ "dev": true, "license": "MIT" }, - "node_modules/terser/node_modules/source-map-support": { - "version": "0.5.21", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "node_modules/test-client": { "resolved": "test-client", "link": true @@ -6720,6 +6909,7 @@ "version": "2.7.4", "resolved": "https://registry.npmjs.org/tiny-readdir/-/tiny-readdir-2.7.4.tgz", "integrity": "sha512-721U+zsYwDirjr8IM6jqpesD/McpZooeFi3Zc6mcjy1pse2C+v19eHPFRqz4chGXZFw7C3KITDjAtHETc2wj7Q==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-counter": "^1.0.2" @@ -6795,22 +6985,41 @@ } }, "node_modules/ts-loader": { - "version": "9.5.2", + "version": "9.6.2", + "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.6.2.tgz", + "integrity": "sha512-R4iuczmtgxvtuI556s+hTZ6/7Ee03VCAk/l/M8LY1OAsUgB7YydsCxkgq9D9pKRaD7GJqUi2u8fp9zZP/ufjKA==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.0", - "enhanced-resolve": "^5.0.0", - "micromatch": "^4.0.0", - "semver": "^7.3.4", + "picomatch": "^4.0.0", "source-map": "^0.7.4" }, "engines": { "node": ">=12.0.0" }, "peerDependencies": { + "loader-utils": "*", "typescript": "*", - "webpack": "^5.0.0" + "webpack": "^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "loader-utils": { + "optional": true + } + } + }, + "node_modules/ts-loader/node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/ts-loader/node_modules/source-map": { @@ -6827,14 +7036,13 @@ "license": "0BSD" }, "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" + "esbuild": "~0.28.0" }, "bin": { "tsx": "dist/cli.mjs" @@ -6861,7 +7069,6 @@ "version": "5.8.3", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6931,7 +7138,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.3", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -7088,12 +7297,14 @@ "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/watcher": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/watcher/-/watcher-2.3.1.tgz", "integrity": "sha512-d3yl+ey35h05r5EFP0TafE2jsmQUJ9cc2aernRVyAkZiu0J3+3TbNugNcqdUJDoWOfL2p+bNsN427stsBC/HnA==", + "dev": true, "dependencies": { "dettle": "^1.0.2", "stubborn-fs": "^1.2.5", @@ -7101,11 +7312,12 @@ } }, "node_modules/watchpack": { - "version": "2.4.2", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", + "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", "dev": true, "license": "MIT", "dependencies": { - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" }, "engines": { @@ -7113,35 +7325,29 @@ } }, "node_modules/webpack": { - "version": "5.99.9", + "version": "5.111.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.111.1.tgz", + "integrity": "sha512-cNypaz0RP+S4cvQVi1M/3bRUkvNP0xZpL0TjFx+c6jg64VbxD+2weWDgY9UnjRI6dhZgtaj8DU76GGFcJ79xPQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@types/eslint-scope": "^3.7.7", - "@types/estree": "^1.0.6", + "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", - "acorn": "^8.14.0", - "browserslist": "^4.24.0", + "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.1", - "es-module-lexer": "^1.2.1", - "eslint-scope": "5.1.1", + "enhanced-resolve": "^5.25.0", + "es-module-lexer": "^2.1.0", "events": "^3.2.0", - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.2.0", - "mime-types": "^2.1.27", - "neo-async": "^2.6.2", - "schema-utils": "^4.3.2", - "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.3.11", - "watchpack": "^2.4.1", - "webpack-sources": "^3.2.3" + "mime-db": "^1.54.0", + "minimizer-webpack-plugin": "^5.7.0", + "schema-utils": "^4.5.0", + "tapable": "^2.3.0", + "watchpack": "^2.5.2", + "webpack-sources": "^3.5.1" }, "bin": { "webpack": "bin/webpack.js" @@ -7163,7 +7369,6 @@ "version": "6.0.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -7228,7 +7433,9 @@ } }, "node_modules/webpack-sources": { - "version": "3.2.3", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.5.1.tgz", + "integrity": "sha512-jyuiGJdtvY434z5bUZrjz67v76/ePNvFZTp9Mdz29IlH4+GPsgyGjiv0fKI+M7BdkU6ADjulUcKAd3tUK3WlEw==", "dev": true, "license": "MIT", "engines": { @@ -7236,10 +7443,11 @@ } }, "node_modules/webpack/node_modules/ajv": { - "version": "8.17.1", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -7251,8 +7459,28 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/webpack/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/webpack/node_modules/ajv-keywords": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "dev": true, "license": "MIT", "dependencies": { @@ -7262,39 +7490,23 @@ "ajv": "^8.8.2" } }, - "node_modules/webpack/node_modules/eslint-scope": { - "version": "5.1.1", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/webpack/node_modules/estraverse": { - "version": "4.3.0", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, "node_modules/webpack/node_modules/json-schema-traverse": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, "node_modules/webpack/node_modules/schema-utils": { - "version": "4.3.2", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "ajv-keywords": "^5.1.0" }, "engines": { @@ -7467,28 +7679,95 @@ "version": "0.14.0", "license": "MIT", "devDependencies": { - "@types/node": "^24.8.1", + "@types/node": "^25.0.2", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", - "fs-extra": "^11.3.0", - "mini-css-extract-plugin": "^2.9.2", - "obsidian": "1.10.2", - "reconcile-text": "^0.8.0", + "fs-extra": "^11.3.2", + "mini-css-extract-plugin": "^2.9.4", + "obsidian": "1.11.0", + "reconcile-text": "^0.11.0", "resolve-url-loader": "^5.0.0", - "sass": "^1.91.0", + "sass": "^1.96.0", "sass-loader": "^16.0.6", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.14", - "ts-loader": "^9.5.2", + "terser-webpack-plugin": "^5.3.16", + "ts-loader": "^9.5.4", "tslib": "2.8.1", - "tsx": "^4.20.6", - "typescript": "5.8.3", + "tsx": "^4.21.0", + "typescript": "5.9.3", "url": "^0.11.4", - "webpack": "^5.99.9", + "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } }, + "obsidian-plugin/node_modules/@codemirror/view": { + "version": "6.38.6", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.6.tgz", + "integrity": "sha512-qiS0z1bKs5WOvHIAC0Cybmv4AJSkAXgX5aD6Mqd2epSLlVJsQl8NG23jCVouIgkh4All/mrbdsf2UOLFnJw0tw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@codemirror/state": "^6.5.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "obsidian-plugin/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "obsidian-plugin/node_modules/obsidian": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.11.0.tgz", + "integrity": "sha512-lVqN9AmDWHzhNATi2tDnjqVgI6WUYKeT+lIsAycAyLt4XCC6zRsWzb+tFCiB7Rn3PpttefjoovilhYwvS4Iqxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/codemirror": "5.60.8", + "moment": "2.29.4" + }, + "peerDependencies": { + "@codemirror/state": "6.5.0", + "@codemirror/view": "6.38.6" + } + }, + "obsidian-plugin/node_modules/reconcile-text": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/reconcile-text/-/reconcile-text-0.11.0.tgz", + "integrity": "sha512-a3sy3obazoc1BMEHx6IQn8ESZKnakVWZuRLi7OSEB56E8evRtrXBMj7yuo10fMoG4JkcZC6tokOfzpwZAKX+PQ==", + "dev": true, + "license": "MIT" + }, + "obsidian-plugin/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "obsidian-plugin/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "sync-client": { "version": "0.14.0", "devDependencies": { @@ -7496,7 +7775,6 @@ "@types/node": "^24.8.1", "byte-base64": "^1.1.0", "minimatch": "^10.0.1", - "p-queue": "^8.1.0", "reconcile-text": "^0.8.0", "ts-loader": "^9.5.2", "tslib": "2.8.1", diff --git a/frontend/package.json b/frontend/package.json index 0dd9057d..17a03fb3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,9 @@ "build": "npm run build --workspaces", "dev": "concurrently --kill-others \"npm run dev -w sync-client\" \"npm run dev -w obsidian-plugin\"", "test": "npm run test --workspaces", + "check:harness": "tsc --noEmit -p test-support/tsconfig.json && tsc --noEmit -p deterministic-tests/tsconfig.json && tsc --noEmit -p test-client/tsconfig.json", + "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/file-application.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", + "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/restart-reconciliation.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts' 'test-support/production-adapters.test.ts'", "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md new file mode 100644 index 00000000..2373c040 --- /dev/null +++ b/frontend/sync-client/README.md @@ -0,0 +1,123 @@ +# Sync client (API v5) + +The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket +`vaultChanged` hints wake it; complete disk scans and the durable server event log are the +sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a +positive interval to repair missed notifications. Unset or `0` disables polling. +Create, delete and move notifications persist logical identity changes before +resolving, including while offline. Shutdown waits for those saves. Content-update +notifications only wake reconciliation; scans read the latest bytes, including after +restart. Notifications describe logical external changes: adapters suppress +self-generated create/delete/move notifications, and report atomic editor saves +as content updates. This distinguishes a deliberate delete/recreate from an edit +when the host supplies reliable logical events. Raw OS watchers instead send +content-update hints for every event and let scans infer the current namespace. +Use `waitUntilFinished()` to await the active attempt; errors leave +pending requests saved and are reported to the caller. Transient failures retry; +authentication and protocol errors wait for an explicit wake or settings change. +Pausing or destroying the client aborts its HTTP session, including body reads; +resuming starts a fresh session. Connection checks can run while sync is disabled. + +## State and policy + +The persisted state contains the incorporated server file manifest, the current +local identity map, and the exact pending submission. A received server file +manifest becomes the merge base when its application metadata is saved. Successful +submissions advance the base to the submitted content snapshot, preserving later +local edits. Unknown outcomes are retried unchanged before incorporating newer +remote state. HTTP is the sole source of remote events; WebSocket messages only +wake the loop. Contiguous event pages are folded in memory into their final manifest +and content heads before merging. An interrupted fetch restarts from the last +incorporated cursor; pagination progress is not persisted. Intermediate changes +that the server has already reverted must not overwrite local edits. + +A permanent upload rejection is recorded in metadata. An unchanged rejected payload is skipped so other files can sync; the +failure remains visible in history and `waitUntilFinished()`. Editing it, resetting +the client, or changing settings permits a fresh attempt. Earlier attempts that +committed despite a lost reply are incorporated through normal event replay. + +For each document ID (absence means deletion), use the server path if local is +unchanged, use local if the server is unchanged, and use the server when both +changed. Then resolve cross-document path conflicts: a retained server path wins, +UUID ordering breaks other ties, and displaced documents receive deterministic +`(conflict UUID)` names. This also handles file/ancestor and portable-name aliases. +Nonportable local names are repaired client-side before a file manifest is published. +Allocated filename components fit within 255 UTF-8 bytes, including conflict suffixes. +The protocol does not impose a universal total-path byte limit; filesystem-specific +limits surface from the adapter instead of silently flattening a name. Directories +are implicit; empty directories are not synced. + +- Content is uploaded before new membership; the client chooses and persists UUIDs. +- Text uses three-way `reconcile`. First contact at a matching path adopts the + remote ID and reconciles differing text with an empty parent. +- Concurrent binary/unmergeable edits keep the server version. If only one side + changed, keep that side. No local backup archive is created. +- Deletion follows the file manifest decision, including when the file has local edits. +- Notified moves preserve IDs. Offline inference requires a unique nonempty hash + match on both sides; uncertain cases become delete/create. +- Ignored and oversized files preserve their disk paths and remote membership. + The original manifest base is retained while excluded, so becoming eligible + resumes reconciliation without losing local renames. Not-yet-downloaded files + are not mistaken for local deletions. +- Bytes and line endings are preserved. File modification times are not tracked + or synchronized. Optional editor cursors are repositioned when text is reconciled. + +## Restored server histories + +Successful HTTP responses carry an `X-Vault-Link-History` checkpoint. The client +persists it before consuming the response and supplies it on subsequent reads and +writes. WebSocket events wake HTTP replay, which validates that checkpoint. A +restored database cannot silently reuse a version number: a missing or different +event incarnation returns a history mismatch before executing the request. + +On mismatch the client scans local work and records a reset before clearing the +old checkpoint. It clears old requests, receipts, cursors, bases and diff-cache +entries. Clean files reconcile against the restored snapshot. Unsent edits with +an old base become separate local documents; path conflicts preserve them under +conflict names instead of applying a diff to an unrelated restored version. +Recovery resumes after interrupted state saves. Existing API v4 client state +without a checkpoint takes this same conservative bootstrap on upgrade. + +Status observers cannot stop the background engine by throwing or rejecting. +Their failures are reported, and remaining observers continue receiving updates. + +## Required adapter contract + +`FileSystemOperations` and `PersistenceProvider` define the adapter operations. +The CLI and desktop Obsidian share a Node filesystem adapter and atomic full-record +metadata persistence. Mobile Obsidian uses its host storage APIs. The E2E runner +builds and tests both integrations, including actual CLI processes. +The sync client does not coordinate separate processes accessing the same vault. + +The filesystem adapter must provide complete scans, coherent snapshots, recursive +directory creation, exclusive file creation, no-replace renames, and deletion of +individual regular files. Directory pruning uses only `rmdir`, never recursive +file removal. Reject links, special files and paths escaping the vault. An unreadable +file or directory must fail the scan rather than look like a deletion. Optional +editor cursor metadata is applied with file content. + +User-file writes have no power-loss durability requirement. The engine has no +filesystem journal, recovery staging area, replay, backup archive, or flush protocol. +Mobile storage needs a disposable scratch file to use its exclusive-copy primitive; +that file is not used for recovery. Swaps and +cycles temporarily move occupied files to visible conflict names; a fresh scan +can discover them if application stops partway through. Partial writes and edits +made before, during or after an interruption are treated as current disk content. +The next sync reconciles that content; it does not finish a saved filesystem plan. +Power loss may lose bytes or rename identity and may cause a merge to run again. + +Metadata is the one atomic boundary: persistence `save` replaces a complete value +and an interrupted/failed save leaves an old or new complete value, never a torn +object. A failed save is reloaded before retrying. No ordering between metadata and +user-file durability is required. Settings and engine saves are serialized. +Keep metadata outside the user-file namespace or in `.vault-link-sync`, which is +excluded from scans. Changing a client's vault once it holds identities, offline +notifications, pending requests or bootstrap state requires a separate state store. + +Filesystem application saves metadata once when it finishes. Namespace changes detected +before mutations invalidate the plan. Changes during application are incorporated +between files using their document identities; the next scan rereads content. Content is reread before +replacement. Unexpected eligible occupants receive visible conflict names; +ignored and oversized occupants stay in place. External writers can still change +files between observations, so these checks do not promise transactional file +contents or perfect identity reconstruction. There is no automated v2/v3 state migration. diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index aa369fa7..bdc094cb 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -15,7 +15,6 @@ "devDependencies": { "byte-base64": "^1.1.0", "minimatch": "^10.0.1", - "p-queue": "^8.1.0", "reconcile-text": "^0.8.0", "uuid": "^13.0.0", "@types/node": "^24.8.1", diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index da70ba47..9cdf64d2 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -1,6 +1,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; -export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; -export const SUPPORTED_API_VERSION = 2; +export const SUPPORTED_API_VERSION = 5; export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; diff --git a/frontend/sync-client/src/errors/errors.ts b/frontend/sync-client/src/errors/errors.ts new file mode 100644 index 00000000..6026d692 --- /dev/null +++ b/frontend/sync-client/src/errors/errors.ts @@ -0,0 +1,37 @@ +export class SyncResetError extends Error { + public constructor() { + super("SyncClient has been reset, cleaning up"); + this.name = "SyncResetError"; + } +} + +export class ServerVersionMismatchError extends Error { + public constructor(message: string) { + super(message); + this.name = "ServerVersionMismatchError"; + } +} + +export class AuthenticationError extends Error { + public constructor(message: string) { + super(message); + this.name = "AuthenticationError"; + } +} + +export class FileNotFoundError extends Error { + public constructor( + message: string, + public readonly filePath: string + ) { + super(message); + this.name = "FileNotFoundError"; + } +} + +export class PermanentSyncError extends Error { } + +export class LocalChangesDuringReconciliation extends Error { } + +/** The server rejected a checkpoint from a discarded/restored history. */ +export class ServerHistoryChangedError extends Error { } diff --git a/frontend/sync-client/src/file-operations/file-not-found-error.ts b/frontend/sync-client/src/file-operations/file-not-found-error.ts deleted file mode 100644 index b8acd265..00000000 --- a/frontend/sync-client/src/file-operations/file-not-found-error.ts +++ /dev/null @@ -1,9 +0,0 @@ -export class FileNotFoundError extends Error { - public constructor( - message: string, - public readonly filePath: string - ) { - super(message); - this.name = "FileNotFoundError"; - } -} diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts deleted file mode 100644 index 998e47ec..00000000 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ /dev/null @@ -1,235 +0,0 @@ -import { describe, it } from "node:test"; -import type { - Database, - DocumentRecord, - RelativePath -} from "../persistence/database"; -import { FileOperations } from "./file-operations"; -import { Logger } from "../tracing/logger"; -import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; -import type { FileSystemOperations } from "./filesystem-operations"; -import type { TextWithCursors } from "reconcile-text"; -import type { ServerConfig, ServerConfigData } from "../services/server-config"; - -class MockServerConfig implements Pick { - public async getConfig(): Promise { - return { - mergeableFileExtensions: ["md", "txt"], - supportedApiVersion: 1, - isAuthenticated: true - }; - } -} - -class MockDatabase implements Partial { - public getLatestDocumentByRelativePath( - _find: RelativePath - ): DocumentRecord | undefined { - // no-op - return undefined; - } - - public move( - _oldRelativePath: RelativePath, - _newRelativePath: RelativePath - ): void { - // no-op - } -} - -class FakeFileSystemOperations implements FileSystemOperations { - public readonly names = new Set(); - - public async listFilesRecursively( - _root: RelativePath | undefined - ): Promise { - return ["file.md"]; - } - public async read(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async write( - path: RelativePath, - _content: Uint8Array - ): Promise { - this.names.add(path); - } - public async atomicUpdateText( - _path: RelativePath, - _updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - throw new Error("Method not implemented."); - } - public async getFileSize(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async getModificationTime(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async exists(path: RelativePath): Promise { - return this.names.has(path); - } - public async createDirectory(_path: RelativePath): Promise { - // this is called but irrelevant for this mock - } - public async delete(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - this.names.delete(oldPath); - this.names.add(newPath); - } -} - -describe("File operations", () => { - it("should deconflict renames", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("a", new Uint8Array()); - assertSetContainsExactly(fileSystemOperations.names, "a"); - await fileOperations.move("a", "b"); - assertSetContainsExactly(fileSystemOperations.names, "b"); - - await fileOperations.create("c", new Uint8Array()); - assertSetContainsExactly(fileSystemOperations.names, "b", "c"); - - await fileOperations.move("c", "b"); - assertSetContainsExactly(fileSystemOperations.names, "b", "b (1)"); - - await fileOperations.create("c", new Uint8Array()); - await fileOperations.move("c", "b"); - assertSetContainsExactly( - fileSystemOperations.names, - "b", - "b (1)", - "b (2)" - ); - }); - - it("should deconflict renames with file extension", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("b.md", new Uint8Array()); - await fileOperations.create("c.md", new Uint8Array()); - await fileOperations.move("c.md", "b.md"); - assertSetContainsExactly( - fileSystemOperations.names, - "b.md", - "b (1).md" - ); - - await fileOperations.create("d.md", new Uint8Array()); - await fileOperations.move("d.md", "b.md"); - assertSetContainsExactly( - fileSystemOperations.names, - "b.md", - "b (1).md", - "b (2).md" - ); - - await fileOperations.create("file-23.md", new Uint8Array()); - await fileOperations.create("file-23 (1).md", new Uint8Array()); - await fileOperations.move("file-23.md", "file-23 (1).md"); - assertSetContainsExactly( - fileSystemOperations.names, - "b.md", - "b (1).md", - "b (2).md", - "file-23 (1).md", - "file-23 (2).md" - ); - }); - - it("should deconflict renames with paths", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("a/b.c/d", new Uint8Array()); - await fileOperations.create("a/b.c/e", new Uint8Array()); - await fileOperations.move("a/b.c/d", "a/b.c/e"); - assertSetContainsExactly( - fileSystemOperations.names, - "a/b.c/e", - "a/b.c/e (1)" - ); - }); - - it("should continue deconfliction from existing number in filename", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("document (5).md", new Uint8Array()); - await fileOperations.create("other.md", new Uint8Array()); - - await fileOperations.move("other.md", "document (5).md"); - assertSetContainsExactly( - fileSystemOperations.names, - "document (5).md", - "document (6).md" - ); - - await fileOperations.create("another.md", new Uint8Array()); - await fileOperations.move("another.md", "document (5).md"); - assertSetContainsExactly( - fileSystemOperations.names, - "document (5).md", - "document (6).md", - "document (7).md" - ); - }); - - it("should handle dotfiles correctly", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create(".gitignore", new Uint8Array()); - await fileOperations.create("temp", new Uint8Array()); - await fileOperations.move("temp", ".gitignore"); - assertSetContainsExactly( - fileSystemOperations.names, - ".gitignore", - ".gitignore (1)" - ); - - await fileOperations.create(".config.json", new Uint8Array()); - await fileOperations.create("temp2", new Uint8Array()); - await fileOperations.move("temp2", ".config.json"); - assertSetContainsExactly( - fileSystemOperations.names, - ".gitignore", - ".gitignore (1)", - ".config.json", - ".config (1).json" - ); - }); -}); diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 2864bd20..49c91dc4 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,291 +1,445 @@ -import type { Logger } from "../tracing/logger"; +import { v4 as uuid } from "uuid"; import type { FileSystemOperations } from "./filesystem-operations"; -import type { Database, RelativePath } from "../persistence/database"; -import { SafeFileSystemOperations } from "./safe-filesystem-operations"; -import type { TextWithCursors } from "reconcile-text"; -import { reconcile } from "reconcile-text"; -import { isFileTypeMergable } from "../utils/is-file-type-mergable"; -import { isBinary } from "../utils/is-binary"; +import type { + Database, + EngineState, + StoredSnapshot +} from "../persistence/database"; +import { allocatePortablePath, isInternalPath } from "../utils/portable-path"; +import { + mergeContent, + fromStoredSnapshot, + toStoredSnapshot +} from "../sync-operations/content"; import type { ServerConfig } from "../services/server-config"; +import { + applyLocalChange, + unappliedChanges, + type LocalChange +} from "../sync-operations/local-changes"; +import { + FileNotFoundError, + LocalChangesDuringReconciliation +} from "../errors/errors"; +export interface FileWrite { + expected?: StoredSnapshot; + replacement: StoredSnapshot; +} + +/** Files and metadata are independent observations, not a filesystem transaction. + * Interrupted applications are reconciled by the next ordinary scan. */ export class FileOperations { - private static readonly PARENTHESES_REGEX = / \((?\d+)\)$/; - private readonly fs: SafeFileSystemOperations; + private physical?: EngineState; public constructor( - private readonly logger: Logger, + public readonly fs: FileSystemOperations, private readonly database: Database, - fs: FileSystemOperations, private readonly serverConfig: ServerConfig, - private readonly nativeLineEndings = "\n" - ) { - this.fs = new SafeFileSystemOperations(fs, logger); + private readonly filesApplied: ( + paths: readonly string[] + ) => void = () => undefined, + private readonly localChanges: { + entries: () => readonly LocalChange[]; + flush: () => Promise; + } = { entries: () => [], flush: async () => undefined }, + private readonly protectedFile: ( + path: string, + size: number + ) => boolean = () => false + ) {} + + public async read(path: string): Promise { + const snapshot = await this.fs.readSnapshot(path); + if (!snapshot) + throw new FileNotFoundError(`File not found: ${path}`, path); + return snapshot.content; } - private static getParentDirAndFile( - path: RelativePath - ): [RelativePath, RelativePath] { - const pathParts = path.split("/"); - const fileName = pathParts.pop(); - if (fileName == null || fileName === "") { - throw new Error(`Path '${path}' cannot be empty`); - } - - return [pathParts.join("/"), fileName]; - } - - public async listFilesRecursively( - root: RelativePath | undefined = undefined - ): Promise { - return this.fs.listFilesRecursively(root); - } - - public async read(path: RelativePath): Promise { - return this.fromNativeLineEndings(await this.fs.read(path)); - } - - /** - * Create a file at the specified path. - * - * If a file with the same name already exists, it is moved before creating the new one. - * Parent directories are created if necessary. - */ - public async create( - path: RelativePath, - newContent: Uint8Array - ): Promise { - await this.ensureClearPath(path); - return this.fs.write(path, this.toNativeLineEndings(newContent)); - } - - public async ensureClearPath(path: RelativePath): Promise { - if (await this.fs.exists(path)) { - const deconflictedPath = await this.deconflictPath(path); - try { - this.logger.debug( - `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` - ); - - this.database.move(path, deconflictedPath); - await this.fs.rename(path, deconflictedPath, true); - } finally { - this.fs.unlock(deconflictedPath); - } - } else { - await this.createParentDirectories(path); - } - } - - /** - * Update the file at the given path. - * - * Performs a 3-way merge before writing if the file's content differs from `expectedContent`. - * Does not recreate the file if it no longer exists, returning an empty array instead. - */ - public async write( - path: RelativePath, - expectedContent: Uint8Array, - newContent: Uint8Array - ): Promise { - if (!(await this.fs.exists(path))) { - this.logger.debug( - `The caller assumed ${path} exists, but it no longer, so we wont recreate it` - ); - return; - } - - if ( - !isFileTypeMergable( - path, - (await this.serverConfig.getConfig()).mergeableFileExtensions - ) || - isBinary(expectedContent) || - isBinary(newContent) - ) { - this.logger.debug( - `The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it` - ); - await this.fs.write( - path, - // `newContent` might not be binary so we still have to ensure the line endings are correct - this.toNativeLineEndings(newContent) - ); - return; - } - - const expectedText = new TextDecoder().decode(expectedContent); // this comes from a previous read which must only have \n line endings - const newText = new TextDecoder().decode(newContent); // this comes from the server which stores text with \n line endings - - await this.fs.atomicUpdateText( - path, - ({ text, cursors }: TextWithCursors): TextWithCursors => { - this.logger.debug( - `Performing a 3-way merge for ${path} with the expected content` - ); - - text = text.replaceAll(this.nativeLineEndings, "\n"); - const merged = reconcile( - expectedText, - { text, cursors }, - newText - ); - - const resultText = merged.text.replaceAll( - "\n", - this.nativeLineEndings - ); - - return { - text: resultText, - cursors: merged.cursors - }; - } + public async listFilesRecursively(): Promise { + return (await this.fs.listFilesRecursively()).filter( + (path) => !isInternalPath(path) ); } - public async delete(path: RelativePath): Promise { - if (await this.exists(path)) { - await this.fs.delete(path); - await this.deletingEmptyParentDirectoriesOfDeletedFile(path); - } else { - this.logger.debug(`No need to delete '${path}', it doesn't exist`); - } + public async snapshot(path: string): Promise { + const snapshot = await this.fs.readSnapshot(path); + return snapshot && toStoredSnapshot(snapshot); } - public async getFileSize(path: RelativePath): Promise { - return this.fs.getFileSize(path); - } - - public async exists(path: RelativePath): Promise { - return this.fs.exists(path); - } - - public async move( - oldPath: RelativePath, - newPath: RelativePath + public async captureChange( + change: LocalChange, + previous: readonly LocalChange[] ): Promise { - if (oldPath === newPath) { - return; - } - - await this.ensureClearPath(newPath); - - this.database.move(oldPath, newPath); - await this.fs.rename(oldPath, newPath); - await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath); + const state = structuredClone(this.physical ?? this.database.state); + if (!state.initialized) return; + for (const pending of unappliedChanges(state, previous)) + applyLocalChange(state, pending, () => false); + applyLocalChange(state, change, () => false); } - public reset(): void { - this.fs.reset(); - } - - private async deletingEmptyParentDirectoriesOfDeletedFile( - path: RelativePath + public async apply( + next: EngineState, + writes: Partial> = {}, + validatePlan: () => void = () => undefined ): Promise { - let directory = path; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - [directory] = FileOperations.getParentDirAndFile(directory); - if (directory.length === 0) { - break; - } - - const remainingContent = - await this.fs.listFilesRecursively(directory); - if (remainingContent.length === 0) { - this.logger.debug( - `Folder (${directory}) is now empty, deleting` - ); - await this.fs.delete(directory); - } else { - break; - } - } - } - - private fromNativeLineEndings(content: Uint8Array): Uint8Array { - if (isBinary(content)) { - return content; - } - - const decoder = new TextDecoder("utf-8"); - let text = decoder.decode(content); - text = text.replaceAll(this.nativeLineEndings, "\n"); - return new TextEncoder().encode(text); - } - - private toNativeLineEndings(content: Uint8Array): Uint8Array { - if (isBinary(content)) { - return content; - } - - const decoder = new TextDecoder("utf-8"); - let text = decoder.decode(content); - text = text.replaceAll("\n", this.nativeLineEndings); - return new TextEncoder().encode(text); - } - - private async createParentDirectories(path: string): Promise { - const components = path.split("/"); - if (components.length === 1) { - return; - } - for (let i = 1; i < components.length; i++) { - const parentDir = components.slice(0, i).join("/"); - if (!(await this.fs.exists(parentDir))) { - await this.fs.createDirectory(parentDir); - } - } - } - - /** - * Deconflicts the given path by appending (1), (2), etc. before the file extension until a non-existent path is found. - * The returned path has a lock acquired on it; it must be released by the caller when no longer needed. - * - * @param path The starting path to deconflict - * @returns a non-existent path with a lock acquired on it - */ - private async deconflictPath(path: RelativePath): Promise { - // eslint-disable-next-line prefer-const - let [directory, fileName] = FileOperations.getParentDirAndFile(path); - - if (directory) { - directory += "/"; - } - - const nameParts = fileName.split("."); - // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should have ".json" - const isDotfile = fileName.startsWith(".") && nameParts[0] === ""; - const extension = - nameParts.length > 1 && !(isDotfile && nameParts.length === 2) - ? "." + nameParts[nameParts.length - 1] - : ""; - let stem = extension ? nameParts.slice(0, -1).join(".") : fileName; - let currentCount = Number.parseInt( - FileOperations.PARENTHESES_REGEX.exec(stem)?.groups?.count ?? "0" - ); - stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); - - let newName = path; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - currentCount++; - newName = `${directory}${stem} (${currentCount})${extension}`; - - // Avoid multiple deconflictPath calls returning the same path - if (this.fs.tryLock(newName)) { - const newDocument = - this.database.getLatestDocumentByRelativePath(newName); + await this.localChanges.flush(); + const changes = this.localChanges.entries().length; + let applying = false; + const guard = (): void => { + // Once mutations begin, incorporate bound notifications between files + // and commit the batch before the next scan reads fresh content. + if (applying) return; + validatePlan(); + if (this.localChanges.entries().length > changes) + throw new LocalChangesDuringReconciliation(); + }; + const beginMutation = (): void => { + guard(); + applying = true; + }; + guard(); + const physical = structuredClone(this.database.state); + physical.initialized = next.initialized; + this.physical = physical; + const paths = new Set(); + try { + for (const id of new Set([ + ...Object.keys(physical.local), + ...Object.keys(next.local) + ])) { + await this.incorporateChanges(next, physical, guard); + let from = + Object.hasOwn(physical.documents, id) && + physical.documents[id].materialized + ? physical.local[id] + : undefined; + let to = Object.hasOwn(next.local, id) + ? next.local[id] + : undefined; + const write = writes[id]; + if (write === undefined && (from === undefined || from === to)) + continue; + if (from !== undefined) paths.add(from); + const info = + from !== undefined ? await this.fs.stat(from) : undefined; + guard(); if ( - newDocument?.isDeleted === false || // the document might have been confirmed by the server at a new path but haven't yet moved there locally - (await this.fs.exists(newName, true)) + from !== undefined && + info?.kind === "file" && + this.protectedFile(from, info.size) ) { - this.fs.unlock(newName); - } else { - return newName; + next.local[id] = from; + next.documents[id] = structuredClone( + physical.documents[id] + ); + (next.excluded ??= {})[id] = + physical.fileManifest.entries[id] ?? null; + (next.protectedPaths ??= []).push(from); + continue; } + if (from !== undefined && info?.kind !== "file") { + Reflect.deleteProperty(next.local, id); + continue; + } + if (to === undefined) { + if (from !== undefined) { + beginMutation(); + await this.fs.deleteFile(from); + Reflect.deleteProperty(physical.local, id); + await this.removeEmptyParents(from); + } + continue; + } + // Move an occupant aside in the visible namespace. This breaks + // cycles without hiding source files in a recovery directory. + if (from !== to) { + try { + to = await this.clearDestination( + next, + physical, + id, + to, + guard, + beginMutation + ); + await this.ensureParents(to); + } catch (error) { + to = await this.fallbackPath(next, id, to, error); + await this.ensureParents(to); + } + next.local[id] = to; + // Clearing the destination may itself have moved this source + // (for example, a file becoming its own parent directory). + from = + Object.hasOwn(physical.documents, id) && + physical.documents[id].materialized + ? physical.local[id] + : undefined; + if (from !== undefined) { + guard(); + // Bind notifications delivered from inside rename to the + // destination identity. A failed rename aborts this plan. + beginMutation(); + physical.local[id] = to; + await this.fs.rename(from, to); + paths.add(to); + await this.removeEmptyParents(from); + from = to; + } + } + if (write !== undefined) { + const current = + from !== undefined + ? await this.snapshot(from) + : undefined; + guard(); + // A detected deletion is a local edit, not an invitation to + // recreate a previously materialized document. + if (from !== undefined && !current) { + Reflect.deleteProperty(next.local, id); + continue; + } + const replacement = + current && write.expected + ? await mergeContent( + to, + write.expected, + current, + write.replacement, + (await this.serverConfig.getConfig()) + .mergeableFileExtensions + ) + : write.replacement; + guard(); + if (current?.hash !== replacement.hash) { + beginMutation(); + if (current) await this.fs.deleteFile(to); + guard(); + physical.local[id] = to; + physical.documents[id] = { + ...next.documents[id], + materialized: true + }; + await this.fs.write( + to, + fromStoredSnapshot(replacement) + ); + } + next.documents[id].observedHash = replacement.hash; + paths.add(to); + } + next.documents[id].materialized = true; + physical.local[id] = to; + physical.documents[id] = structuredClone(next.documents[id]); } + await this.incorporateChanges(next, physical, guard); + guard(); + await this.database.commit(next); + } finally { + this.physical = undefined; + this.filesApplied([...paths]); + } + } + + private async incorporateChanges( + next: EngineState, + physical: EngineState, + guard: () => void + ): Promise { + await this.localChanges.flush(); + guard(); + for (const change of unappliedChanges( + next, + this.localChanges.entries() + )) { + const ignored = (path: string): boolean => + isInternalPath(path) || this.protectedFile(path, 0); + applyLocalChange(physical, change, ignored); + applyLocalChange(next, change, ignored); + physical.lastAppliedLocalChangeId = next.lastAppliedLocalChangeId = + change.changeId; + } + } + + private async clearDestination( + next: EngineState, + physical: EngineState, + id: string, + path: string, + guard: () => void, + beginMutation: () => void + ): Promise { + for (;;) { + const obstruction = await this.obstruction(path); + if (!obstruction) return path; + guard(); + const info = await this.fs.stat(obstruction.path); + if ( + obstruction.kind === "file" && + info && + next.protectedPaths?.includes(obstruction.path) !== true && + !this.protectedFile(obstruction.path, info.size) + ) { + await this.moveAside( + next, + physical, + obstruction.path, + false, + beginMutation + ); + } else if (obstruction.kind === "directory") { + const children = await this.fs.listFilesRecursively( + obstruction.path + ); + let movable = true; + for (const child of children) { + const occupant = Object.keys(physical.local).find( + (other) => physical.local[other] === child + ); + const childInfo = await this.fs.stat(child); + if ( + occupant === undefined || + !Object.hasOwn(physical.documents, occupant) || + !physical.documents[occupant].materialized || + next.local[occupant] === child || + next.protectedPaths?.includes(child) === true || + childInfo === undefined || + this.protectedFile(child, childInfo.size) + ) { + movable = false; + break; + } + } + if (movable) { + // Only evacuate tracked sources already scheduled to leave. + // Root-level temporary names also break directory/file cycles. + for (const child of children) + await this.moveAside( + next, + physical, + child, + true, + beginMutation + ); + beginMutation(); + await this.fs.delete(obstruction.path); + } else path = await this.conflictPath(next, id, path); + } else { + path = await this.conflictPath(next, id, path); + } + } + } + + private async moveAside( + next: EngineState, + physical: EngineState, + path: string, + toRoot: boolean, + beginMutation: () => void + ): Promise { + const occupant = + Object.keys(physical.local).find( + (other) => + Object.hasOwn(physical.documents, other) && + physical.documents[other].materialized && + physical.local[other] === path + ) ?? uuid(); + const displaced = await this.conflictPath( + next, + occupant, + toRoot ? (path.split("/").at(-1) ?? path) : path + ); + await this.ensureParents(displaced); + beginMutation(); + physical.local[occupant] = displaced; + await this.fs.rename(path, displaced); + if (!Object.hasOwn(physical.documents, occupant)) { + const snapshot = await this.snapshot(displaced); + physical.documents[occupant] = { + materialized: true, + observedHash: snapshot?.hash + }; + next.documents[occupant] = structuredClone( + physical.documents[occupant] + ); + next.local[occupant] = displaced; + } else if (next.local[occupant] === path) + next.local[occupant] = displaced; + } + + private async obstruction( + path: string + ): Promise<{ path: string; kind: "file" | "directory" } | undefined> { + const parts = path.split("/"); + for (let i = 0; i < parts.length; i++) { + const prefix = parts.slice(0, i + 1).join("/"); + const info = await this.fs.stat(prefix); + if (info && (info.kind === "file" || i === parts.length - 1)) + return { path: prefix, kind: info.kind }; + } + } + + private async conflictPath( + next: EngineState, + id: string, + path: string + ): Promise { + const occupied = { ...next.local }; + Reflect.deleteProperty(occupied, id); + for (const [i, diskPath] of ( + await this.listFilesRecursively() + ).entries()) + occupied[`disk-${i}`] = diskPath; + return allocatePortablePath(path, id, occupied); + } + + private async fallbackPath( + next: EngineState, + id: string, + path: string, + error: unknown + ): Promise { + const code = + typeof error === "object" && error !== null && "code" in error + ? error.code + : undefined; + if ( + ![ + "ENAMETOOLONG", + "EINVAL", + "ENOTSUP", + "EOPNOTSUPP", + "EACCES", + "EPERM" + ].includes(String(code)) + ) + throw error; + const extension = + /\.[^.]*$/u.exec(path.split("/").at(-1) ?? "")?.[0] ?? ""; + return this.conflictPath(next, id, `Recovered ${id}${extension}`); + } + + private async ensureParents(path: string): Promise { + const parent = path.split("/").slice(0, -1).join("/"); + if (parent) await this.fs.createDirectory(parent); + } + + private async removeEmptyParents(path: string): Promise { + let parent = path.split("/").slice(0, -1).join("/"); + while (parent && !isInternalPath(parent)) { + const info = await this.obstruction(parent); + if ( + info?.kind !== "directory" || + (await this.fs.listFilesRecursively(parent)).length + ) + break; + try { + await this.fs.delete(parent); + } catch { + break; + } + parent = parent.split("/").slice(0, -1).join("/"); } } } diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 36dddfe6..d8fcf0ab 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -1,37 +1,35 @@ -import type { RelativePath } from "../persistence/database"; +import type { CursorPosition } from "reconcile-text"; -import type { TextWithCursors } from "reconcile-text"; - +export interface FileSnapshot { + content: Uint8Array; + /** Optional editor selections, moved with the file and applied with its bytes. */ + cursors?: CursorPosition[]; +} + +/** Filesystem operations act on the current user-owned namespace. + * Paths are vault-relative. Never follow symlinks (including ancestor symlinks). + * Only independent regular files are supported; reject hardlinks/special files. + * A successful scan is complete: unreadable directories must reject, not vanish. + */ export interface FileSystemOperations { - // List all files under root that should be synced. If root is undefined, return every file. - listFilesRecursively: ( - root: RelativePath | undefined - ) => Promise; - - // Read the content of a file. - read: (path: RelativePath) => Promise; - - // Create or overwrite a file with the given content. - write: (path: RelativePath, content: Uint8Array) => Promise; - - // Atomically update the content of a text file. - atomicUpdateText: ( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ) => Promise; - - // Get the size of a file in bytes. - getFileSize: (path: RelativePath) => Promise; - - // Check if a file exists. - exists: (path: RelativePath) => Promise; - - // Create a directory at the specified path. All parent directories must already exist. - createDirectory: (path: RelativePath) => Promise; - - // Delete a file. It is expected that the path points to an existing file. - delete: (path: RelativePath) => Promise; - - // Rename a file. It is expected that the oldPath points to an existing file and the newPath does not exist. - rename: (oldPath: RelativePath, newPath: RelativePath) => Promise; + listFilesRecursively: (root?: string) => Promise; + /** Coherent content snapshot; undefined only for an absent file. Reject read races. */ + readSnapshot: (path: string) => Promise; + /** Reject symlinks and special files rather than following them. */ + stat: ( + path: string + ) => Promise<{ kind: "file" | "directory"; size: number } | undefined>; + /** Includes directories. */ + exists: (path: string) => Promise; + /** Recursive, idempotent directory creation. */ + createDirectory: (path: string) => Promise; + /** Create a file exclusively; never overwrite a raced destination. + * Writes apply optional editor cursor metadata. No durability guarantee. */ + write: (path: string, snapshot: FileSnapshot) => Promise; + /** Move a file without replacing an existing destination. */ + rename: (from: string, to: string) => Promise; + /** Idempotently unlink one regular file. */ + deleteFile: (path: string) => Promise; + /** Prune directories using rmdir only. Reject files and nonempty directories. */ + delete: (path: string) => Promise; } diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts deleted file mode 100644 index 904bf805..00000000 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ /dev/null @@ -1,170 +0,0 @@ -import type { RelativePath } from "../persistence/database"; -import type { FileSystemOperations } from "./filesystem-operations"; -import type { Logger } from "../tracing/logger"; -import { Locks } from "../utils/data-structures/locks"; -import { FileNotFoundError } from "./file-not-found-error"; -import type { TextWithCursors } from "reconcile-text"; - -/** - * Decorates `FileSystemOperations` to replace errors with `FileNotFoundError` - * if the accessed file doesn't exist. It also ensures that there's at most a - * single request in-flight for any one file through the use of locks. - */ -export class SafeFileSystemOperations implements FileSystemOperations { - private readonly locks: Locks; - - public constructor( - private readonly fs: FileSystemOperations, - private readonly logger: Logger - ) { - this.locks = new Locks(logger); - } - - public async listFilesRecursively( - root: RelativePath | undefined - ): Promise { - this.logger.debug("Listing all files"); - const result = await this.fs.listFilesRecursively(root); - this.logger.debug(`Listed ${result.length} files`); - return result; - } - - public async read(path: RelativePath): Promise { - this.logger.debug(`Reading file '${path}'`); - return this.safeOperation( - path, - async () => - this.locks.withLock(path, async () => this.fs.read(path)), - "read" - ); - } - - public async write(path: RelativePath, content: Uint8Array): Promise { - this.logger.debug(`Writing to file '${path}'`); - return this.locks.withLock(path, async () => - this.fs.write(path, content) - ); - } - - public async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - this.logger.debug(`Atomically updating file '${path}'`); - return this.safeOperation( - path, - async () => - this.locks.withLock(path, async () => - this.fs.atomicUpdateText(path, updater) - ), - "atomicUpdateText" - ); - } - - public async getFileSize(path: RelativePath): Promise { - // Logging this would be too noisy - return this.safeOperation( - path, - async () => - this.locks.withLock(path, async () => - this.fs.getFileSize(path) - ), - "getFileSize" - ); - } - - public async exists( - path: RelativePath, - skipLock = false - ): Promise { - this.logger.debug(`Checking if file '${path}' exists`); - if (skipLock) { - return this.fs.exists(path); - } else { - return this.locks.withLock(path, async () => this.fs.exists(path)); - } - } - - public async createDirectory(path: RelativePath): Promise { - this.logger.debug(`Creating directory '${path}'`); - return this.locks.withLock(path, async () => - this.fs.createDirectory(path) - ); - } - - public async delete(path: RelativePath): Promise { - this.logger.debug(`Deleting file '${path}'`); - return this.locks.withLock(path, async () => this.fs.delete(path)); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath, - skipLock = false - ): Promise { - this.logger.debug(`Renaming file '${oldPath}' to '${newPath}'`); - return this.safeOperation( - oldPath, - async () => { - if (skipLock) { - return this.fs.rename(oldPath, newPath); - } else { - return this.locks.withLock([oldPath, newPath], async () => - this.fs.rename(oldPath, newPath) - ); - } - }, - "rename" - ); - } - - public tryLock(path: RelativePath): boolean { - return this.locks.tryLock(path); - } - - public async waitForLock(path: RelativePath): Promise { - return this.locks.waitForLock(path); - } - - public unlock(path: RelativePath): void { - this.locks.unlock(path); - } - - public reset(): void { - this.locks.reset(); - } - - /** - * Decorate an operation to ensure that the file exists before running it. - * If the operation fails, it will check if the file still exists and throw - * a FileNotFoundError if it doesn't. - */ - private async safeOperation( - path: RelativePath, - operation: () => Promise, - operationName: string - ): Promise { - if (!(await this.fs.exists(path))) { - throw new FileNotFoundError( - `File not found before trying to ${operationName}`, - path - ); - } - - try { - return await operation(); - } catch (error) { - // Without locking the file, this isn't atomic, however, it's good enough in practice. - // This will only break if the file exists, gets deleted and then immediately - // recreated while `operation` is running. - if (await this.fs.exists(path)) { - throw error; - } else { - throw new FileNotFoundError( - `File not found when trying to ${operationName}`, - path - ); - } - } - } -} diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index cfcc5071..8d1db5be 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -22,16 +22,21 @@ export { Logger, LogLevel, LogLine } from "./tracing/logger"; export { type SyncSettings, DEFAULT_SETTINGS } from "./persistence/settings"; export { rateLimit } from "./utils/rate-limit"; export type { RelativePath, StoredDatabase } from "./persistence/database"; -export type { FileSystemOperations } from "./file-operations/filesystem-operations"; +export type { + FileSystemOperations, + FileSnapshot +} from "./file-operations/filesystem-operations"; export type { PersistenceProvider } from "./persistence/persistence"; export type { CursorSpan } from "./services/types/CursorSpan"; export type { ClientCursors } from "./services/types/ClientCursors"; export type { NetworkConnectionStatus } from "./types/network-connection-status"; -export type { ServerVersionMismatchError } from "./services/server-version-mismatch-error"; -export type { AuthenticationError } from "./services/authentication-error"; +export type { + ServerVersionMismatchError, + AuthenticationError +} from "./errors/errors"; export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; export { DocumentSyncStatus } from "./types/document-sync-status"; -export { SyncClient } from "./sync-client"; +export { SyncClient, type StoredClient } from "./sync-client"; export type { TextWithCursors, CursorPosition } from "reconcile-text"; export const debugging = { @@ -47,3 +52,11 @@ export const utils = { awaitAll, removeFromArray }; + +export type { EventBatch } from "./services/types/EventBatch"; +export type { EventRecord } from "./services/types/EventRecord"; +export type { FileManifest } from "./services/types/FileManifest"; +export type { PushFileManifest } from "./services/types/PushFileManifest"; +export type { PutFileContent } from "./services/types/PutFileContent"; +export type { VaultSnapshot } from "./services/types/VaultSnapshot"; +export type { FileManifestEntries } from "./types/file-manifest-entries"; diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 86b2845c..ec47430d 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,374 +1,191 @@ -import type { Logger } from "../tracing/logger"; -import { EMPTY_HASH } from "../utils/hash"; -import { CoveredValues } from "../utils/data-structures/min-covered"; -import { awaitAll } from "../utils/await-all"; -import { removeFromArray } from "../utils/remove-from-array"; +import type { CursorPosition } from "reconcile-text"; +import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { FileManifestUpdateResponse } from "../services/types/FileManifestUpdateResponse"; +import type { PushFileManifest } from "../services/types/PushFileManifest"; +import type { PutFileContent } from "../services/types/PutFileContent"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import type { FileManifestEntries } from "../types/file-manifest-entries"; export type VaultUpdateId = number; export type DocumentId = string; export type RelativePath = string; +export interface StoredSnapshot { + contentBase64: string; + hash: string; + cursors?: CursorPosition[]; +} + export interface DocumentMetadata { - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath?: RelativePath; -} - -export interface StoredDocumentMetadata { - relativePath: RelativePath; - documentId: DocumentId; - parentVersionId: VaultUpdateId; - remoteRelativePath?: RelativePath; + parentVersionId: number; hash: string; } -export interface StoredDatabase { - documents: StoredDocumentMetadata[]; - lastSeenUpdateId: VaultUpdateId | undefined; - hasInitialSyncCompleted: boolean; +export interface DocumentState { + /** Local clean base retained while rebasing after a server restore. */ + recoveryBase?: StoredSnapshot; + base?: DocumentVersionWithoutContent & { hash: string }; + observedHash?: string; + materialized: boolean; + /** First contact with existing local content: there is no common ancestor. */ + bootstrap?: boolean; + /** Do not resubmit an unchanged payload rejected by the server. */ + rejected?: { hash: string; message: string }; } -/** - * Represents a document in the database. - * - * It is mutable and its content should always represent the latest - * state of the document on disk based on the update events we have seen. - */ +export type PendingRequest = { rejection?: string } & ( + | { + type: "content"; + documentId: string; + request: PutFileContent; + snapshot: StoredSnapshot; + response?: DocumentUpdateResponse; + } + | { + type: "fileManifest"; + request: PushFileManifest; + response?: FileManifestUpdateResponse; + } +); + +export interface EngineState { + /** Persisted reset intent; clear the transport checkpoint before bootstrap. */ + historyRecovery?: boolean; + vaultKey: string; + initialized: boolean; + fileManifest: FileManifest; + local: FileManifestEntries; + documents: Record; + lastSeenUpdateId: number; + remoteHeads: Record; + bootstrap?: VaultSnapshot; + pending?: PendingRequest; + /** A rejected retry does not prove an earlier attempt failed to commit. */ + unconfirmed?: PendingRequest[]; + /** Commits consumption of the saved notification queue with the identity map. */ + lastAppliedLocalChangeId?: string; + /** Physical exclusions, including files with no sync identity. */ + protectedPaths?: string[]; + /** Original server paths while local files are excluded; null means absent. */ + excluded?: Record; + rejectedManifest?: { entries: FileManifestEntries; message: string }; +} + +export type StoredDatabase = EngineState; + +// Read-only view used by cursor tracking and status reporting. export interface DocumentRecord { - relativePath: RelativePath; - documentId: DocumentId; - metadata: DocumentMetadata | undefined; - isDeleted: boolean; - updates: Promise[]; - parallelVersion: number; + documentId: string; + relativePath: string; + metadata?: DocumentMetadata; +} + +export function emptyState(vaultKey: string): StoredDatabase { + return { + vaultKey, + initialized: false, + fileManifest: { fileManifestId: 0, entries: {} }, + local: {}, + documents: {}, + remoteHeads: {}, + lastSeenUpdateId: 0 + }; +} + +function canRebindEmpty(state: Partial | undefined): boolean { + return ( + state !== undefined && + !state.initialized && + !state.pending && + !state.bootstrap && + (state.unconfirmed?.length ?? 0) === 0 && + Object.keys(state.local ?? {}).length === 0 + ); } export class Database { - private documents: DocumentRecord[]; - private lastSeenUpdateIds: CoveredValues; - private hasInitialSyncCompleted: boolean; - + public state: StoredDatabase; + private needsReload = false; public constructor( - private readonly logger: Logger, - initialState: Partial | undefined, - private readonly saveData: (data: StoredDatabase) => Promise + initial: Partial | undefined, + private readonly saveData: (data: StoredDatabase) => Promise, + vaultKey: string, + private readonly loadData: () => Promise ) { - initialState ??= {}; - - this.documents = - initialState.documents?.map( - ({ relativePath, documentId, ...metadata }) => ({ - relativePath, - documentId, - metadata, - isDeleted: false, - updates: [], - parallelVersion: 0 - }) - ) ?? []; - - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.length} documents`); - - const { lastSeenUpdateId } = initialState; - this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`); - this.lastSeenUpdateIds = new CoveredValues( - Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1 - ); - - this.documents.forEach((doc) => { - this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId); - }); - - this.hasInitialSyncCompleted = - initialState.hasInitialSyncCompleted ?? false; - this.logger.debug( - `Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}` - ); - } - - public get length(): number { - return this.documents.length; - } - - public get resolvedDocuments(): DocumentRecord[] { - const paths = new Map(); - this.documents - // eslint-disable-next-line no-restricted-syntax -- Type narrowing, not removing a specific item - .filter(({ metadata }) => metadata !== undefined) - .forEach((record) => - paths.set(record.relativePath, [ - record, - ...(paths.get(record.relativePath) ?? []) - ]) - ); - - return Array.from(paths.values()).map((records) => { - records.sort( - (a, b) => b.parallelVersion - a.parallelVersion // descending - ); - - if ( - records.length > 1 && - records.some((current, i) => - i === 0 - ? false - : records[i - 1].parallelVersion === - current.parallelVersion - ) - ) { - throw new Error( - `Multiple documents with the same parallel version and path at ${records[0].relativePath}` - ); - } - return records[0]; - }); - } - - public updateDocumentMetadata( - metadata: { - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath: RelativePath; - }, - toUpdate: DocumentRecord - ): void { - if (!this.documents.includes(toUpdate)) { - throw new Error("Document not found in database"); - } - - toUpdate.metadata = metadata; - - this.saveInTheBackground(); - } - - public removeDocumentPromise(promise: Promise): void { - const entry = this.documents.find(({ updates }) => - updates.includes(promise) - ); - - if (entry === undefined) { - // This method should be idempotent and tolerant of - // stragglers calling it after the databse has been reset. - return; - } - - removeFromArray(entry.updates, promise); - // No need to save as Promises don't get serialized - } - - public removeDocument(find: DocumentRecord): void { - removeFromArray(this.documents, find); - this.saveInTheBackground(); - } - - public getLatestDocumentByRelativePath( - find: RelativePath - ): DocumentRecord | undefined { - const candidates = this.documents.filter( - ({ relativePath }) => relativePath === find - ); - candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending - return candidates[0]; - } - - public async getResolvedDocumentByRelativePath( - relativePath: RelativePath, - promise: Promise - ): Promise { - const entry = this.getLatestDocumentByRelativePath(relativePath); - - if (entry === undefined) { + const hasState = initial && Object.keys(initial).length > 0; + if ( + hasState && + initial.vaultKey !== vaultKey && + !canRebindEmpty(initial) + ) { throw new Error( - `Document not found by relative path: ${relativePath}, ${JSON.stringify( - this.documents, - null, - 2 - )}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); } - - const currentPromises = entry.updates; - entry.updates = [...currentPromises, promise]; - await awaitAll(currentPromises); - - return entry; + this.state = hasState + ? { ...(initial as StoredDatabase), vaultKey } + : emptyState(vaultKey); } - public createNewPendingDocument( - documentId: DocumentId, - relativePath: RelativePath, - promise: Promise - ): DocumentRecord { - this.logger.debug( - `Creating new pending document: ${relativePath} (${documentId})` - ); - const previousEntry = - this.getLatestDocumentByRelativePath(relativePath); - - const entry = { - relativePath, - documentId, - metadata: undefined, - isDeleted: false, - updates: [promise], - parallelVersion: - previousEntry?.parallelVersion === undefined - ? 0 - : previousEntry.parallelVersion + 1 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public createNewEmptyDocument( - documentId: DocumentId, - parentVersionId: VaultUpdateId, - relativePath: RelativePath - ): DocumentRecord { - const entry = { - relativePath, - documentId, - metadata: { - parentVersionId, - hash: EMPTY_HASH, - remoteRelativePath: relativePath - }, - isDeleted: false, - updates: [], - parallelVersion: 0 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public getDocumentByDocumentId( - find: DocumentId - ): DocumentRecord | undefined { - return this.documents.find(({ documentId }) => documentId === find); - } - - public move( - oldRelativePath: RelativePath, - newRelativePath: RelativePath - ): void { - const oldDocument = - this.getLatestDocumentByRelativePath(oldRelativePath); - - if (oldDocument === undefined) { - return; - } - - const newDocument = - this.getLatestDocumentByRelativePath(newRelativePath); - if (newDocument?.isDeleted === false) { + /** Bind an empty database to the configured vault. */ + public async bindVault(vaultKey: string): Promise { + if (this.state.vaultKey === vaultKey) return; + if (!canRebindEmpty(this.state)) throw new Error( - `Document already exists at new location: ${newRelativePath}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); - } - - oldDocument.relativePath = newRelativePath; - // We're in a strange state where the target of the move has just got deleted, - // however, its metadata might already have a bunch of updates queued up for - // the document at the new location. We need to keep these updates. - oldDocument.parallelVersion = - newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - - this.saveInTheBackground(); + await this.commit({ ...this.state, vaultKey }); } - public delete(relativePath: RelativePath): void { - const candidate = this.getLatestDocumentByRelativePath(relativePath); - if (candidate === undefined) { + public async commit(next: StoredDatabase): Promise { + const snapshot = structuredClone(next); + if (this.needsReload) throw new Error( - `Document not found by relative path: ${relativePath}` + "Reload saved state before retrying an uncertain save" ); - } - candidate.isDeleted = true; + this.needsReload = true; + await this.saveData(snapshot); + this.state = snapshot; + this.needsReload = false; } - public getHasInitialSyncCompleted(): boolean { - return this.hasInitialSyncCompleted; - } - - public setHasInitialSyncCompleted(value: boolean): void { - this.hasInitialSyncCompleted = value; - this.saveInTheBackground(); - } - - public getLastSeenUpdateId(): VaultUpdateId { - return this.lastSeenUpdateIds.min; - } - - public addSeenUpdateId(value: number): void { - const previousMin = this.lastSeenUpdateIds.min; - this.lastSeenUpdateIds.add(value); - if (previousMin !== this.lastSeenUpdateIds.min) { - this.saveInTheBackground(); - } - } - - public setLastSeenUpdateId(value: number): void { - this.lastSeenUpdateIds.min = value; - this.saveInTheBackground(); - } - - public reset(): void { - this.documents = []; - this.lastSeenUpdateIds = new CoveredValues( - 0 // the first updateId will be 1 which is the first integer after -1 - ); - this.hasInitialSyncCompleted = false; - this.saveInTheBackground(); + public async recoverPersistence(): Promise { + if (!this.needsReload) return; + const saved = await this.loadData(); + if (saved) this.state = saved; + this.needsReload = false; } public async save(): Promise { - return this.saveData({ - documents: this.resolvedDocuments.map( - ({ relativePath, documentId, metadata }) => ({ - documentId, - relativePath, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ...metadata! // `resolvedDocuments` only returns docs with metadata set - }) - ), - lastSeenUpdateId: this.lastSeenUpdateIds.min, - hasInitialSyncCompleted: this.hasInitialSyncCompleted - }); + await this.commit(this.state); } - private ensureConsistency(): void { - const idToPath = new Map(); - - this.resolvedDocuments.forEach(({ relativePath, documentId }) => { - idToPath.set(documentId, [ - ...(idToPath.get(documentId) ?? []), - relativePath - ]); - }); - - const duplicates = Array.from(idToPath.entries()) - .filter(([_, paths]) => paths.length > 1) - .map(([id, paths]) => `${id} (${paths.join(", ")})`); - - if (duplicates.length > 0) { - throw new Error( - "Document IDs are not unique, found duplicates: " + - duplicates.join("; ") - ); - } + public get length(): number { + return Object.keys(this.state.local).length; } - private saveInTheBackground(): void { - this.ensureConsistency(); - void this.save().catch((error: unknown) => { - this.logger.error(`Error saving data: ${error}`); - }); + public getDocumentByDocumentId(id: string): DocumentRecord | undefined { + const relativePath = this.state.local[id]; + if (relativePath === undefined) return undefined; + const base = this.state.documents[id]?.base; + return { + documentId: id, + relativePath, + metadata: base && { + parentVersionId: base.vaultUpdateId, + hash: base.hash + } + }; + } + + public getLatestDocumentByRelativePath( + path: string + ): DocumentRecord | undefined { + const id = Object.keys(this.state.local).find( + (id) => this.state.local[id] === path + ); + return id === undefined ? undefined : this.getDocumentByDocumentId(id); } } diff --git a/frontend/sync-client/src/persistence/persistence.ts b/frontend/sync-client/src/persistence/persistence.ts index d226e611..ac6f724a 100644 --- a/frontend/sync-client/src/persistence/persistence.ts +++ b/frontend/sync-client/src/persistence/persistence.ts @@ -1,3 +1,7 @@ +/** Replace the complete metadata value atomically. Interrupted or failed saves + * must leave an old/new complete value (or no value for the first save), never + * partially parsed state. No ordering or durability of user files is required. + */ export interface PersistenceProvider { load: () => Promise; save: (data: T) => Promise; diff --git a/frontend/sync-client/src/persistence/settings.test.ts b/frontend/sync-client/src/persistence/settings.test.ts new file mode 100644 index 00000000..cff24895 --- /dev/null +++ b/frontend/sync-client/src/persistence/settings.test.ts @@ -0,0 +1,89 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { Logger, LogLevel } from "../tracing/logger"; +import { Settings } from "./settings"; + +describe("Settings logging", () => { + it("redacts tokens while loading and updating settings", async () => { + const logger = new Logger(); + const settings = new Settings( + logger, + { token: "loaded-secret" }, + async () => {} + ); + await settings.setSettings({ token: "updated-secret" }); + + const messages = logger + .getMessages(LogLevel.DEBUG) + .map((entry) => entry.message) + .join("\n"); + assert.doesNotMatch(messages, /loaded-secret|updated-secret/u); + assert.match(messages, /\[REDACTED\]/u); + }); + + it("retains and persists the configured diff cache size", async () => { + let saved = 0; + const settings = new Settings( + new Logger(), + { diffCacheSizeMB: 7 }, + async (value) => { + saved = value.diffCacheSizeMB; + } + ); + + assert.equal(settings.getSettings().diffCacheSizeMB, 7); + await settings.setSetting("diffCacheSizeMB", 9); + assert.equal(settings.getSettings().diffCacheSizeMB, 9); + assert.equal(saved, 9); + }); +}); + +it("owns settings arrays instead of sharing them with callers and listeners", async () => { + const original = ["original"]; + const settings = new Settings( + new Logger(), + { ignorePatterns: original }, + async () => {} + ); + original.push("external mutation"); + assert.deepEqual(settings.getSettings().ignorePatterns, ["original"]); + const update = ["updated"]; + await settings.setSettings({ ignorePatterns: update }); + update.push("external mutation"); + assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]); + settings.onSettingsChanged.add((_, old) => { + old.ignorePatterns.push("listener mutation"); + }); + await settings.setSettings({ maxFileSizeMB: 12 }); + assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]); +}); + +it("file eligibility follows successfully saved settings", async () => { + let fail = false; + const settings = new Settings( + new Logger(), + { + ignorePatterns: ["private/**"], + maxFileSizeMB: 1 + }, + async () => { + if (fail) throw new Error("save failed"); + } + ); + assert(settings.isIgnored("private/note.md")); + assert(settings.isIgnored(".vault-link-sync/state.json")); + assert(!settings.isIgnored("public/note.md")); + assert(!settings.isOversized(1024 * 1024)); + assert(settings.isOversized(1024 * 1024 + 1)); + fail = true; + await assert.rejects( + settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 }), + /save failed/ + ); + assert(settings.isIgnored("private/note.md")); + assert(settings.isOversized(1024 * 1024 + 1)); + fail = false; + await settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 }); + assert(!settings.isIgnored("private/note.md")); + assert(!settings.isOversized(1024 * 1024 + 1)); +}); diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index d78170e6..9a93e0dd 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -1,4 +1,6 @@ import type { Logger } from "../tracing/logger"; +import { globsToRegexes } from "../utils/globs-to-regexes"; +import { isInternalPath } from "../utils/portable-path"; import { Lock } from "../utils/data-structures/locks"; import { EventListeners } from "../utils/data-structures/event-listeners"; @@ -6,38 +8,44 @@ export interface SyncSettings { remoteUri: string; token: string; vaultName: string; - syncConcurrency: number; isSyncEnabled: boolean; maxFileSizeMB: number; ignorePatterns: string[]; webSocketRetryIntervalMs: number; diffCacheSizeMB: number; enableTelemetry: boolean; + requestTimeoutMs: number; networkRetryIntervalMs: number; - minimumSaveIntervalMs: number; + syncIntervalMs?: number; } export const DEFAULT_SETTINGS: SyncSettings = { remoteUri: "", token: "", vaultName: "default", - syncConcurrency: 1, isSyncEnabled: false, maxFileSizeMB: 10, ignorePatterns: [], webSocketRetryIntervalMs: 3500, diffCacheSizeMB: 4, enableTelemetry: false, + requestTimeoutMs: 30_000, networkRetryIntervalMs: 1000, - minimumSaveIntervalMs: 1000 + syncIntervalMs: undefined }; +const redactSettings = (value: Partial): object => ({ + ...value, + ...(value.token === undefined ? {} : { token: "[REDACTED]" }) +}); + export class Settings { public readonly onSettingsChanged = new EventListeners< (newSettings: SyncSettings, oldSettings: SyncSettings) => unknown >(); private settings: SyncSettings; + private ignorePatterns: RegExp[]; private readonly lock: Lock = new Lock(); public constructor( @@ -47,16 +55,35 @@ export class Settings { ) { this.settings = { ...DEFAULT_SETTINGS, - ...(initialState ?? {}) + ...structuredClone(initialState ?? {}) }; + this.ignorePatterns = globsToRegexes( + this.settings.ignorePatterns, + logger + ); + this.logger.debug( - `Loaded settings: ${JSON.stringify(this.settings, null, 2)}` + `Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}` ); } + public isIgnored(path: string): boolean { + return ( + isInternalPath(path) || + this.ignorePatterns.some((pattern) => pattern.test(path)) + ); + } + + public isOversized(size: number): boolean { + return size > this.settings.maxFileSizeMB * 1024 * 1024; + } + public getSettings(): SyncSettings { - return this.settings; + return { + ...this.settings, + ignorePatterns: [...this.settings.ignorePatterns] + }; } public async setSetting( @@ -68,27 +95,32 @@ export class Settings { }); } - public async setSettings(value: Partial): Promise { - await this.lock.withLock(async () => { + public async setSettings( + value: Partial, + notify = true + ): Promise { + const update = structuredClone(value); + + const change = await this.lock.withLock(async () => { this.logger.debug( - `Updating settings with: ${JSON.stringify(value)}` - ); - const oldSettings = this.settings; - this.settings = { - ...this.settings, - ...value - }; - - await this.onSettingsChanged.triggerAsync( - this.settings, - oldSettings + `Updating settings with: ${JSON.stringify(redactSettings(update))}` ); - await this.save(); + const oldSettings = this.getSettings(); + const next = { ...this.settings, ...update }; + await this.saveData(next); + this.settings = next; + if (update.ignorePatterns !== undefined) + this.ignorePatterns = globsToRegexes( + next.ignorePatterns, + this.logger + ); + + return [this.getSettings(), oldSettings] as const; }); - } - private async save(): Promise { - await this.saveData(this.settings); + // Listeners can themselves update settings. Never await them while + // holding a lock needed by the operation they invoke. + if (notify) await this.onSettingsChanged.triggerAsync(...change); } } diff --git a/frontend/sync-client/src/services/authentication-error.ts b/frontend/sync-client/src/services/authentication-error.ts deleted file mode 100644 index 6be4af24..00000000 --- a/frontend/sync-client/src/services/authentication-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class AuthenticationError extends Error { - public constructor(message: string) { - super(message); - this.name = "AuthenticationError"; - } -} diff --git a/frontend/sync-client/src/services/fetch-controller.test.ts b/frontend/sync-client/src/services/fetch-controller.test.ts deleted file mode 100644 index 94fa8424..00000000 --- a/frontend/sync-client/src/services/fetch-controller.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import type { Mock } from "node:test"; -import { describe, it, mock, beforeEach, afterEach } from "node:test"; -import assert from "node:assert"; -import { FetchController } from "./fetch-controller"; -import { Logger } from "../tracing/logger"; -import { SyncResetError } from "./sync-reset-error"; -import { sleep } from "../utils/sleep"; - -describe("FetchController", () => { - const createMockFetch = ( - shouldSleep: boolean - ): Mock<() => Promise> => - mock.fn(async () => { - if (shouldSleep) { - await sleep(30); - } - return Promise.resolve(new Response("OK", { status: 200 })); - }); - - beforeEach(() => { - mock.timers.enable({ apis: ["setTimeout"] }); - }); - - afterEach(() => { - mock.timers.reset(); - }); - - it("should allow fetch when canFetch is true", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(false); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - const response = await controlledFetch("http://example.com"); - - assert.strictEqual(await response.text(), "OK"); - assert.strictEqual(mockFetch.mock.calls.length, 1); - }); - - it("should block fetch until canFetch becomes true", async () => { - const logger = new Logger(); - const controller = new FetchController(false, logger); - const mockFetch = createMockFetch(true); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - const fetchPromise = controlledFetch("http://example.com"); - assert.strictEqual(mockFetch.mock.calls.length, 0); - - controller.canFetch = true; - await Promise.resolve(); - mock.timers.tick(30); - - const response = await fetchPromise; - assert.strictEqual(await response.text(), "OK"); - assert.strictEqual(mockFetch.mock.calls.length, 1); - }); - - it("should reject during reset", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(true); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - const firstRequest = controlledFetch("http://example.com"); - assert.strictEqual(mockFetch.mock.calls.length, 1); - - controller.startReset(); - - const secondRequest = controlledFetch("http://example.com"); - - await assert.rejects( - firstRequest, - (error: unknown) => error instanceof SyncResetError - ); - await assert.rejects( - secondRequest, - (error: unknown) => error instanceof SyncResetError - ); - assert.strictEqual(mockFetch.mock.calls.length, 1); - }); - - it("should allow fetch after reset finishes", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(false); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - controller.startReset(); - controller.finishReset(); - - const response = await controlledFetch("http://example.com"); - assert.strictEqual(await response.text(), "OK"); - }); - - it("should defer canFetch changes during reset", async () => { - const logger = new Logger(); - const controller = new FetchController(false, logger); - const mockFetch = createMockFetch(true); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - controller.startReset(); - controller.canFetch = true; - - await assert.rejects( - async () => controlledFetch("http://example.com"), - (error: unknown) => error instanceof SyncResetError - ); - - controller.finishReset(); - - const fetchPromise = controlledFetch("http://example.com"); - mock.timers.tick(30); - - const response = await fetchPromise; - assert.strictEqual(await response.text(), "OK"); - }); - - it("should handle different input types", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(false); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - await controlledFetch("http://example.com"); - await controlledFetch(new URL("http://example.com")); - await controlledFetch( - new Request("http://example.com", { method: "POST" }) - ); - - assert.strictEqual(mockFetch.mock.calls.length, 3); - }); - - it("should handle fetch errors", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = mock.fn(async () => { - throw new Error("Network error"); - }); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - await assert.rejects( - async () => controlledFetch("http://example.com"), - (error: unknown) => - error instanceof Error && error.message === "Network error" - ); - }); - - it("should not create unhandled rejection on reset with no waiting fetches", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - - controller.startReset(); - mock.timers.tick(10); - controller.finishReset(); - }); -}); diff --git a/frontend/sync-client/src/services/fetch-controller.ts b/frontend/sync-client/src/services/fetch-controller.ts deleted file mode 100644 index 77b87e3a..00000000 --- a/frontend/sync-client/src/services/fetch-controller.ts +++ /dev/null @@ -1,149 +0,0 @@ -import type { Logger } from "../tracing/logger"; -import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "./sync-reset-error"; - -/** - * Offers a resettable fetch implementation that waits until syncing is enabled - * and aborts outstanding requests when a reset is started. - */ -export class FetchController { - private static readonly UNTIL_RESOLUTION = Symbol(); - - private isResetting = false; - - // Promise resolves on the next state change: sync enabled/disabled or reset started/ended - private until: Promise; - private resolveUntil: (result: symbol) => unknown; - private rejectUntil: (reason: unknown) => unknown; - - public constructor( - private _canFetch: boolean, - private readonly logger: Logger - ) { - [this.until, this.resolveUntil, this.rejectUntil] = - createPromise(); - } - - /** - * Whether the fetch implementation can immediately send requests once outside of a reset. - */ - public get canFetch(): boolean { - return this._canFetch; - } - - /** - * Allow or disallow fetching. The changes only take effect if not resetting. - * When called during a reset, its effect is deferred until the reset is finished. - * - * @param canFetch Whether fetching is enabled - */ - public set canFetch(canFetch: boolean) { - this._canFetch = canFetch; - - if (!this.isResetting) { - const previousResolve = this.resolveUntil; - [this.until, this.resolveUntil, this.rejectUntil] = - createPromise(); - previousResolve(FetchController.UNTIL_RESOLUTION); - } - } - - private static getUrlFromInput(input: RequestInfo | URL): string { - if (input instanceof URL) { - return input.href; - } - if (typeof input === "string") { - return input; - } - return input.url; - } - - /** - * Starts a reset, causing all ongoing and future fetches to be rejected - * with a SyncResetError until finishReset is called. - */ - public startReset(): void { - this.isResetting = true; - this.rejectUntil(new SyncResetError()); - // Catch unhandled rejection if no fetches are waiting - this.until.catch(() => { - // Intentionally ignore - this rejection is handled by waiting fetches - }); - } - - /** - * Finishes a reset, allowing fetches to proceed or wait again depending on - * the current sync settings. - */ - public finishReset(): void { - if (!this.isResetting) { - return; - } - - this.isResetting = false; - [this.until, this.resolveUntil, this.rejectUntil] = createPromise(); - } - - /** - * - * |------------------|---------------|-----------------------------------------------------| - * | | Sync enabled | Sync disabled | - * |------------------|-------------- |-----------------------------------------------------| - * | During reset | Rejects with SyncResetError without sending request | - * |------------------|-------------- |-----------------------------------------------------| - * | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch | - * |------------------|---------------|-----------------------------------------------------| - * - * @param logger for errors - * @param fetch to wrap - * @returns a wrapped fetch implementation affected by the FetchController state - */ - public getControlledFetchImplementation( - logger: Logger, - fetch: typeof globalThis.fetch = globalThis.fetch - ): typeof globalThis.fetch { - return async ( - input: RequestInfo | URL, - init?: RequestInit - ): Promise => { - while (!this.canFetch || this.isResetting) { - await this.until; - } - - try { - // https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21 - const _input = - typeof Request !== "undefined" && input instanceof Request - ? input.clone() - : input; - - const fetchPromise = fetch(_input, init); - - // We only want to catch rejections from `this.until` - let result: symbol | Response | undefined = undefined; - do { - result = await Promise.race([this.until, fetchPromise]); - } while (result === FetchController.UNTIL_RESOLUTION); - - const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - if (!fetchResult.ok) { - this.logger.warn( - `Fetch for ${FetchController.getUrlFromInput( - input - )}, got status ${fetchResult.status}` - ); - } - - return fetchResult; - } catch (error) { - logger.warn( - `Fetch for ${FetchController.getUrlFromInput( - input - )}, got error: ${error}` - ); - throw error; - } - }; - } -} diff --git a/frontend/sync-client/src/services/server-config.test.ts b/frontend/sync-client/src/services/server-config.test.ts new file mode 100644 index 00000000..d82f8f06 --- /dev/null +++ b/frontend/sync-client/src/services/server-config.test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { ServerConfig } from "./server-config"; +import { SyncService } from "./sync-service"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { SUPPORTED_API_VERSION } from "../consts"; + +const config = { + supportedApiVersion: SUPPORTED_API_VERSION, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" +}; +const make = (fetch: typeof globalThis.fetch) => + new ServerConfig( + new SyncService( + "device", + new Settings( + new Logger(), + { remoteUri: "http://test" }, + async () => {} + ), + fetch + ) + ); + +test("configuration shares one validated request and retries failures", async () => { + let calls = 0; + const server = make(async () => { + if (++calls === 1) throw new Error("offline"); + return Response.json(config); + }); + await assert.rejects(server.getConfig(), /offline/); + const [a, b] = await Promise.all([server.getConfig(), server.getConfig()]); + assert.equal(a, b); + assert.equal(calls, 2); + await server.getConfig(); + assert.equal(calls, 2); +}); + +test("connection checks are independent and validate authentication and version", async () => { + let response = config; + const server = make(async () => Response.json(response)); + const cached = await server.getConfig(); + response = { ...config, isAuthenticated: false }; + assert.equal((await server.checkConnection()).isSuccessful, false); + response = { ...config, supportedApiVersion: SUPPORTED_API_VERSION + 1 }; + assert.equal((await server.checkConnection()).isSuccessful, false); + assert.equal(await server.getConfig(), cached); + server.reset(); + await assert.rejects(server.getConfig(), /Unsupported API/); + response = config; + assert.deepEqual(await server.getConfig(), config); +}); + +for (const fail of [false, true]) { + test(`reset fences a late ${fail ? "failed" : "successful"} configuration request`, async () => { + const first = Promise.withResolvers(); + let calls = 0; + const server = make(async () => + ++calls === 1 ? first.promise : Response.json(config) + ); + const old = server.getConfig(); + server.reset(); + const current = await server.getConfig(); + if (fail) { + first.reject(new Error("old request")); + await assert.rejects(old, /old request/); + } else { + first.resolve( + Response.json({ ...config, mergeableFileExtensions: ["txt"] }) + ); + await old; + } + assert.equal(await server.getConfig(), current); + assert.equal(calls, 2); + }); +} diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts index 309c637c..102cf425 100644 --- a/frontend/sync-client/src/services/server-config.ts +++ b/frontend/sync-client/src/services/server-config.ts @@ -1,8 +1,9 @@ import { SUPPORTED_API_VERSION } from "../consts"; -import { AuthenticationError } from "./authentication-error"; -import { ServerVersionMismatchError } from "./server-version-mismatch-error"; +import { + AuthenticationError, + ServerVersionMismatchError +} from "../errors/errors"; import type { SyncService } from "./sync-service"; -import type { PingResponse } from "./types/PingResponse"; export interface ServerConfigData { mergeableFileExtensions: string[]; @@ -11,8 +12,7 @@ export interface ServerConfigData { } export class ServerConfig { - private response: Promise | undefined; - private config: ServerConfigData | undefined; + private config: Promise | undefined; public constructor(private readonly syncService: SyncService) {} @@ -39,29 +39,16 @@ export class ServerConfig { await this.getConfig(); } - public async checkConnection(forceUpdate = false): Promise<{ + public async checkConnection(): Promise<{ isSuccessful: boolean; message: string; }> { try { - let { response } = this; - if (!response || forceUpdate) { - response = this.response = this.syncService.ping(); - } - - const result: PingResponse = await response; // it must be defined, otherwise we would have thrown above - this.config = result; - - if (result.isAuthenticated) { - return { - isSuccessful: true, - message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` - }; - } - + const result = await this.syncService.ping(); + ServerConfig.validateConfig(result); return { - isSuccessful: false, - message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` + isSuccessful: true, + message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` }; } catch (e) { return { @@ -73,17 +60,22 @@ export class ServerConfig { public async getConfig(): Promise { if (!this.config) { - this.response ??= this.syncService.ping(); - this.config = await this.response; + const pending = this.syncService + .ping() + .then((config) => { + ServerConfig.validateConfig(config); + return config; + }) + .catch((error: unknown) => { + if (this.config === pending) this.config = undefined; + throw error; + }); + this.config = pending; } - - ServerConfig.validateConfig(this.config); - return this.config; } public reset(): void { - this.response = undefined; this.config = undefined; } } diff --git a/frontend/sync-client/src/services/server-version-mismatch-error.ts b/frontend/sync-client/src/services/server-version-mismatch-error.ts deleted file mode 100644 index 0b9960ea..00000000 --- a/frontend/sync-client/src/services/server-version-mismatch-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class ServerVersionMismatchError extends Error { - public constructor(message: string) { - super(message); - this.name = "ServerVersionMismatchError"; - } -} diff --git a/frontend/sync-client/src/services/sync-reset-error.ts b/frontend/sync-client/src/services/sync-reset-error.ts deleted file mode 100644 index 7b74e0b9..00000000 --- a/frontend/sync-client/src/services/sync-reset-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class SyncResetError extends Error { - public constructor() { - super("SyncClient has been reset, cleaning up"); - this.name = "SyncResetError"; - } -} diff --git a/frontend/sync-client/src/services/sync-service.test.ts b/frontend/sync-client/src/services/sync-service.test.ts new file mode 100644 index 00000000..bf56fec4 --- /dev/null +++ b/frontend/sync-client/src/services/sync-service.test.ts @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { SyncResetError } from "../errors/errors"; +import { SyncService } from "./sync-service"; + +const settings = () => + new Settings( + new Logger(), + { + remoteUri: "http://test", + vaultName: "test", + requestTimeoutMs: 1000 + }, + async () => {} + ); + +test("paused sync sends no requests, while connection checks still work", async () => { + let requests = 0; + const service = new SyncService("device", settings(), async () => { + requests++; + return Response.json({ headEventId: 0, events: [] }); + }); + await assert.rejects(service.events(0), SyncResetError); + assert.equal(requests, 0); + await service.ping(); + assert.equal(requests, 1); + service.resume(); + assert.equal((await service.events(0)).headEventId, 0); + assert.equal(requests, 2); +}); + +test("pause aborts requests and fences late responses from an injected fetch", async () => { + const response = Promise.withResolvers(); + const signals: AbortSignal[] = []; + const saved: (string | undefined)[] = []; + const service = new SyncService( + "device", + settings(), + async (_, init) => { + signals.push(init!.signal!); + return signals.length === 1 + ? response.promise + : Response.json({ headEventId: 0, events: [] }); + }, + { + get: () => undefined, + save: async (value) => { + saved.push(value); + } + } + ); + service.resume(); + const first = service.events(0); + service.pause(); + await assert.rejects(first, SyncResetError); + assert.equal(signals[0].aborted, true); + service.resume(); + await service.events(0); + assert.equal(signals[1].aborted, false); + response.resolve( + Response.json( + {}, + { headers: { "X-Vault-Link-History": "99:abandoned" } } + ) + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(saved, []); +}); + +for (const binary of [false, true]) { + test(`pause interrupts a stalled ${binary ? "binary" : "JSON"} response body`, async () => { + const headersSaved = Promise.withResolvers(); + let body!: ReadableStreamDefaultController; + const service = new SyncService( + "device", + settings(), + async () => + new Response( + new ReadableStream({ + start(controller) { + body = controller; + } + }), + { headers: { "X-Vault-Link-History": "1:head" } } + ), + { + get: () => undefined, + save: async () => { + headersSaved.resolve(); + } + } + ); + service.resume(); + const request = binary + ? service.getDocumentVersionContent({ + documentId: "a", + vaultUpdateId: 1 + }) + : service.events(0); + await headersSaved.promise; + await new Promise((resolve) => setImmediate(resolve)); + service.pause(); + await assert.rejects(request, SyncResetError); + body.enqueue(new TextEncoder().encode("{}")); + body.close(); + }); +} + +test("pause waits for a checkpoint save already in progress", async () => { + const saving = Promise.withResolvers(); + const release = Promise.withResolvers(); + let settled = false; + let checkpoint: string | undefined; + const service = new SyncService( + "device", + settings(), + async () => + Response.json( + {}, + { + headers: { "X-Vault-Link-History": "1:head" } + } + ), + { + get: () => checkpoint, + save: async (value) => { + saving.resolve(); + await release.promise; + checkpoint = value; + } + } + ); + service.resume(); + const request = service.events(0).finally(() => { + settled = true; + }); + await saving.promise; + service.pause(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(settled, false); + release.resolve(); + await assert.rejects(request, SyncResetError); + assert.equal(checkpoint, "1:head"); +}); + +test("history saves only new checkpoints, including a changed token at the same event ID", async () => { + let checkpoint: string | undefined; + let responseCheckpoint = "1:first"; + const saved: (string | undefined)[] = []; + const service = new SyncService( + "device", + settings(), + async () => + Response.json( + {}, + { headers: { "X-Vault-Link-History": responseCheckpoint } } + ), + { + get: () => checkpoint, + save: async (value) => { + checkpoint = value; + saved.push(value); + } + } + ); + service.resume(); + for (const value of [ + "1:first", + "1:first", + "2:next", + "1:first", + "2:replacement", + "2:replacement" + ]) { + responseCheckpoint = value; + await service.events(0); + } + assert.deepEqual(saved, ["1:first", "2:next", "2:replacement"]); +}); diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 8190a638..24bac853 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -1,434 +1,212 @@ -import type { - DocumentId, - RelativePath, - VaultUpdateId -} from "../persistence/database"; - -import type { Logger } from "../tracing/logger"; import type { Settings } from "../persistence/settings"; -import type { FetchController } from "./fetch-controller"; -import { sleep } from "../utils/sleep"; -import { SyncResetError } from "./sync-reset-error"; -import type { SerializedError } from "./types/SerializedError"; -import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; -import type { DocumentVersion } from "./types/DocumentVersion"; -import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse"; +import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; +import type { EventBatch } from "./types/EventBatch"; +import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse"; +import type { PutFileContent } from "./types/PutFileContent"; +import type { PushFileManifest } from "./types/PushFileManifest"; +import type { VaultSnapshot } from "./types/VaultSnapshot"; import type { PingResponse } from "./types/PingResponse"; -import type { DeleteDocumentVersion } from "./types/DeleteDocumentVersion"; -import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion"; +import { + AuthenticationError, + PermanentSyncError, + ServerHistoryChangedError, + SyncResetError +} from "../errors/errors"; + +// Injected fetch implementations may ignore cancellation. Stop waiting anyway, +// and never let their late responses reach checkpoint or engine persistence. +async function abortable( + signal: AbortSignal, + operation: () => Promise +): Promise { + signal.throwIfAborted(); + let abort = (): void => undefined; + try { + return await Promise.race([ + new Promise((_, reject) => { + abort = (): void => { + // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- Only reset errors and timeout errors reach this helper. + reject(signal.reason); + }; + signal.addEventListener("abort", abort, { once: true }); + }), + operation() + ]); + } finally { + signal.removeEventListener("abort", abort); + } +} export class SyncService { - private readonly client: typeof globalThis.fetch; - private readonly pingClient: typeof globalThis.fetch; + private session = new AbortController(); public constructor( private readonly deviceId: string, - private readonly fetchController: FetchController, private readonly settings: Settings, - private readonly logger: Logger, - fetchImplementation: typeof globalThis.fetch = globalThis.fetch + private readonly fetchImplementation: typeof fetch = globalThis.fetch, + private readonly history?: { + get: () => string | undefined; + save: (checkpoint: string | undefined) => Promise; + } ) { - // ensure that if it's called a method, `this` won't be bound to the instance - const unboundFetch: typeof globalThis.fetch = async (...args) => - fetchImplementation(...args); - - this.client = this.fetchController.getControlledFetchImplementation( - this.logger, - unboundFetch - ); - this.pingClient = unboundFetch; + this.pause(); } - private static async errorFromResponse( - response: Response - ): Promise { - if ( - response.headers - .get("Content-Type") - ?.includes("application/json") == true - ) { - const result: SerializedError = - (await response.json()) as SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - return SyncService.formatError(result); - } - return `HTTP ${response.status}: ${response.statusText}`; + public get verifiesHistory(): boolean { + return this.history !== undefined; + } + public get hasHistoryCheckpoint(): boolean { + return this.history?.get() !== undefined; } - private static formatError(error: SerializedError): string { - let result = error.message; - if (error.causes.length > 0) { - const causes = error.causes.join(", "); - result += ` caused by: ${causes}`; - } - - return result; + public resume(): void { + this.session = new AbortController(); } - public async create({ - documentId, - relativePath, - contentBytes - }: { - documentId?: DocumentId; - relativePath: RelativePath; - contentBytes: Uint8Array; - }): Promise { - return this.retryForever(async () => { - const formData = new FormData(); - if (documentId !== undefined) { - formData.append("document_id", documentId); - } - formData.append("relative_path", relativePath); - formData.append( - "content", - new Blob([new Uint8Array(contentBytes)]) - ); - - this.logger.debug( - `Creating document with id ${documentId} and relative path ${relativePath}` - ); - - const response = await this.client(this.getUrl("/documents"), { - method: "POST", - body: formData, - headers: this.getDefaultHeaders() - }); - - if (!response.ok) { - throw new Error( - `Failed to create document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentVersionWithoutContent = - (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug(`Created document ${JSON.stringify(result)}`); - - return result; - }); + public pause(): void { + this.session.abort(new SyncResetError()); } - public async putText({ - parentVersionId, - documentId, - relativePath, - content - }: { - parentVersionId: VaultUpdateId; - documentId: DocumentId; - relativePath: RelativePath; - content: (number | string)[]; - }): Promise { - return this.retryForever(async () => { - this.logger.debug( - `Updating text document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}, content [${content.join(", ")}]` - ); - - const request: UpdateTextDocumentVersion = { - parentVersionId, - relativePath, - content - }; - - const response = await this.client( - this.getUrl(`/documents/${documentId}/text`), - { - method: "PUT", - body: JSON.stringify(request), - headers: this.getDefaultHeaders({ type: "json" }) - } - ); - - if (!response.ok) { - throw new Error( - `Failed to update document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentUpdateResponse = - (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Updated document ${JSON.stringify(result)} with id ${ - result.documentId - }}` - ); - - return result; - }); + public async resetHistory(): Promise { + await this.history?.save(undefined); } - public async putBinary({ - parentVersionId, - documentId, - relativePath, - contentBytes - }: { - parentVersionId: VaultUpdateId; - documentId: DocumentId; - relativePath: RelativePath; - contentBytes: Uint8Array; - }): Promise { - return this.retryForever(async () => { - this.logger.debug( - `Updating binary document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` - ); - const formData = new FormData(); - formData.append("parent_version_id", parentVersionId.toString()); - formData.append("relative_path", relativePath); - formData.append( - "content", - new Blob([new Uint8Array(contentBytes)]) - ); - - const response = await this.client( - this.getUrl(`/documents/${documentId}/binary`), - { - method: "PUT", - body: formData, - headers: this.getDefaultHeaders() - } - ); - - if (!response.ok) { - throw new Error( - `Failed to update document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentUpdateResponse = - (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Updated document ${JSON.stringify(result)} with id ${ - result.documentId - }}` - ); - - return result; - }); + public async ping(): Promise { + return this.request("/ping", undefined, true); } - public async delete({ - documentId, - relativePath - }: { - documentId: DocumentId; - relativePath: RelativePath; - }): Promise { - return this.retryForever(async () => { - const request: DeleteDocumentVersion = { - relativePath - }; - - this.logger.debug( - `Delete document with id ${documentId} and relative path ${relativePath}` - ); - - const response = await this.client( - this.getUrl(`/documents/${documentId}`), - { - method: "DELETE", - body: JSON.stringify(request), - headers: this.getDefaultHeaders({ type: "json" }) - } - ); - - if (!response.ok) { - throw new Error( - `Failed to delete document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentVersionWithoutContent = - (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Deleted document ${relativePath} with id ${documentId}` - ); - - return result; - }); + public async vaultSnapshot(): Promise { + return this.request("/vault-snapshot"); } - public async get({ - documentId - }: { - documentId: DocumentId; - }): Promise { - return this.retryForever(async () => { - this.logger.debug(`Getting document with id ${documentId}`); + public async events(after: number): Promise { + return this.request(`/events-since?after=${after}`); + } - const response = await this.client( - this.getUrl(`/documents/${documentId}`), - { - headers: this.getDefaultHeaders() - } - ); + public async pushFileManifest( + request: PushFileManifest + ): Promise { + return this.request("/file-manifest", request); + } - if (!response.ok) { - throw new Error( - `Failed to get document: ${await SyncService.errorFromResponse( - response - )}` - ); - } + public async putFileContent( + id: string, + request: PutFileContent + ): Promise { + return this.request(`/documents/${id}`, request); + } - const result: DocumentVersion = - (await response.json()) as DocumentVersion; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug(`Got document ${JSON.stringify(result)}`); - - return result; - }); + public async metadata(id: string): Promise { + return this.request(`/documents/${id}/metadata`); } public async getDocumentVersionContent({ documentId, vaultUpdateId }: { - documentId: DocumentId; - vaultUpdateId: VaultUpdateId; + documentId: string; + vaultUpdateId: number; }): Promise { - return this.retryForever(async () => { - this.logger.debug( - `Getting document with id ${documentId} and version ${vaultUpdateId}` - ); - - const response = await this.client( - this.getUrl( - `/documents/${documentId}/versions/${vaultUpdateId}/content` - ), - { - headers: this.getDefaultHeaders() - } - ); - - if (!response.ok) { - throw new Error( - `Failed to get document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result = await response.bytes(); - this.logger.debug( - `Got document version content for document ${documentId} version ${vaultUpdateId}` - ); - return result; - }); - } - - public async getAll( - since?: VaultUpdateId - ): Promise { - return this.retryForever(async () => { - this.logger.debug( - "Getting all documents" + - (since != null ? ` since ${since}` : "") - ); - - const url = new URL(this.getUrl("/documents")); - if (since !== undefined) { - url.searchParams.append("since", since.toString()); - } - const response = await this.client(url.toString(), { - headers: this.getDefaultHeaders() - }); - - if (!response.ok) { - throw new Error( - `Failed to get documents: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: FetchLatestDocumentsResponse = - (await response.json()) as FetchLatestDocumentsResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Got ${result.latestDocuments.length} document metadata` - ); - - return result; - }); - } - - public async ping(): Promise { - this.logger.debug("Pinging server"); - const response = await this.pingClient(this.getUrl("/ping"), { - headers: this.getDefaultHeaders() - }); - - if (!response.ok) { - throw new Error( - `Failed to ping server: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: PingResponse = (await response.json()) as PingResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Pinged server, got response: ${JSON.stringify(result)}` + return this.request( + `/documents/${documentId}/versions/${vaultUpdateId}/content`, + undefined, + false, + async (response) => new Uint8Array(await response.arrayBuffer()) ); + } - return result; + private historyHeaders(): Record { + const checkpoint = this.history?.get(); + return checkpoint !== undefined && checkpoint !== "" + ? { "X-Vault-Link-History": checkpoint } + : {}; + } + + private async recordHistory(response: Response): Promise { + const checkpoint = response.headers.get("x-vault-link-history"); + if ( + checkpoint !== null && + checkpoint !== "" && + this.history !== undefined + ) { + const previous = this.history.get(); + if (checkpoint === previous) return; + if ( + previous === undefined || + previous === "" || + Number(checkpoint.split(":")[0]) >= + Number(previous.split(":")[0]) + ) + await this.history.save(checkpoint); + } + } + + private checkHistory(response: Response): void { + if (response.headers.get("x-vault-link-history-mismatch") === "1") + throw new ServerHistoryChangedError( + "Server history changed; recovering local work" + ); } private getUrl(path: string): string { - const { vaultName, remoteUri } = this.settings.getSettings(); - const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, ""); - const encodedVaultName = encodeURIComponent(vaultName.trim()); - return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`; + const { remoteUri, vaultName } = this.settings.getSettings(); + return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`; } - private getDefaultHeaders( - { type }: { type?: "json" } = { type: undefined } - ): Record { - const headers: Record = { - "device-id": this.deviceId, - authorization: `Bearer ${this.settings.getSettings().token}` - }; - - if (type === "json") { - headers["Content-Type"] = "application/json"; - } - - return headers; - } - - private async retryForever(fn: () => Promise): Promise { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - try { - return await fn(); - } catch (e) { - // We must not retry errors coming from reset - if (e instanceof SyncResetError) { - throw e; + private async request( + path: string, + body?: unknown, + raw = false, + decode: (response: Response) => Promise = async (response) => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Endpoint methods supply the generated protocol response type. + response.json() as Promise + ): Promise { + const timeout = AbortSignal.timeout( + this.settings.getSettings().requestTimeoutMs + ); + const signal = raw + ? timeout + : AbortSignal.any([timeout, this.session.signal]); + const response = await abortable(signal, async () => + this.fetchImplementation(this.getUrl(path), { + method: body === undefined ? "GET" : "PUT", + body: body === undefined ? undefined : JSON.stringify(body), + signal, + headers: { + ...(raw ? {} : this.historyHeaders()), + Authorization: `Bearer ${this.settings.getSettings().token}`, + "Device-Id": this.deviceId, + "Content-Type": "application/json" } - - const retryInterval = - this.settings.getSettings().networkRetryIntervalMs; - this.logger.error( - `Failed network call (${e}), retrying in ${retryInterval}ms` - ); - await sleep(retryInterval); - } + }) + ); + signal.throwIfAborted(); + this.checkHistory(response); + if (!response.ok) { + this.throwForErrorResponse( + response, + `HTTP ${response.status}: ${await abortable(signal, async () => response.text())}` + ); } + + // Finish metadata saves even if the transport is paused meanwhile. + if (!raw) await this.recordHistory(response); + return abortable(signal, async () => decode(response)); + } + + private throwForErrorResponse(response: Response, message: string): never { + if (response.status === 401 || response.status === 403) { + throw new AuthenticationError(message); + } + if ( + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 + ) { + throw new PermanentSyncError(message); + } + throw new Error(message); } } diff --git a/frontend/sync-client/src/services/types/ClientCursors.ts b/frontend/sync-client/src/services/types/ClientCursors.ts index e8c9b93d..bb629100 100644 --- a/frontend/sync-client/src/services/types/ClientCursors.ts +++ b/frontend/sync-client/src/services/types/ClientCursors.ts @@ -1,8 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentWithCursors } from "./DocumentWithCursors"; -export interface ClientCursors { - userName: string; - deviceId: string; - documentsWithCursors: DocumentWithCursors[]; -} +export type ClientCursors = { userName: string, deviceId: string, documentsWithCursors: Array, }; diff --git a/frontend/sync-client/src/services/types/CreateDocumentVersion.ts b/frontend/sync-client/src/services/types/CreateDocumentVersion.ts deleted file mode 100644 index ed921f18..00000000 --- a/frontend/sync-client/src/services/types/CreateDocumentVersion.ts +++ /dev/null @@ -1,13 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export interface CreateDocumentVersion { - /** - * The client can decide the document id (if it wishes to) in order - * to help with syncing. If the client does not provide a document id, - * the server will generate one. If the client provides a document id - * it must not already exist in the database. - */ - document_id: string | null; - relative_path: string; - content: number[]; -} diff --git a/frontend/sync-client/src/services/types/CursorPositionFromClient.ts b/frontend/sync-client/src/services/types/CursorPositionFromClient.ts index ee937f4e..60b48e5e 100644 --- a/frontend/sync-client/src/services/types/CursorPositionFromClient.ts +++ b/frontend/sync-client/src/services/types/CursorPositionFromClient.ts @@ -1,6 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentWithCursors } from "./DocumentWithCursors"; -export interface CursorPositionFromClient { - documentsWithCursors: DocumentWithCursors[]; -} +export type CursorPositionFromClient = { documentsWithCursors: Array, }; diff --git a/frontend/sync-client/src/services/types/CursorPositionFromServer.ts b/frontend/sync-client/src/services/types/CursorPositionFromServer.ts index 52a24f27..c8444892 100644 --- a/frontend/sync-client/src/services/types/CursorPositionFromServer.ts +++ b/frontend/sync-client/src/services/types/CursorPositionFromServer.ts @@ -1,6 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ClientCursors } from "./ClientCursors"; -export interface CursorPositionFromServer { - clients: ClientCursors[]; -} +export type CursorPositionFromServer = { clients: Array, }; diff --git a/frontend/sync-client/src/services/types/CursorSpan.ts b/frontend/sync-client/src/services/types/CursorSpan.ts index 2cc2b7fc..d0bce6ea 100644 --- a/frontend/sync-client/src/services/types/CursorSpan.ts +++ b/frontend/sync-client/src/services/types/CursorSpan.ts @@ -1,6 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface CursorSpan { - start: number; - end: number; -} +export type CursorSpan = { start: number, end: number, }; diff --git a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts index 7fd06c7a..85dc56ab 100644 --- a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts @@ -1,10 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { DocumentVersion } from "./DocumentVersion"; import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; /** * Response to an update document request. */ -export type DocumentUpdateResponse = - | ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent) - | ({ type: "MergingUpdate" } & DocumentVersion); +export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersionWithoutContent; diff --git a/frontend/sync-client/src/services/types/DocumentVersion.ts b/frontend/sync-client/src/services/types/DocumentVersion.ts index 3b9aa37b..930b4899 100644 --- a/frontend/sync-client/src/services/types/DocumentVersion.ts +++ b/frontend/sync-client/src/services/types/DocumentVersion.ts @@ -1,12 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DocumentVersion { - vaultUpdateId: number; - documentId: string; - relativePath: string; - updatedDate: string; - contentBase64: string; - isDeleted: boolean; - userId: string; - deviceId: string; -} +export type DocumentVersion = { contentBase64: string, vaultUpdateId: number, documentId: string, updatedDate: string, userId: string, deviceId: string, contentSize: number, }; diff --git a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts index 4b24e7c5..fac3b420 100644 --- a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts +++ b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts @@ -1,12 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DocumentVersionWithoutContent { - vaultUpdateId: number; - documentId: string; - relativePath: string; - updatedDate: string; - isDeleted: boolean; - userId: string; - deviceId: string; - contentSize: number; -} +export type DocumentVersionWithoutContent = { vaultUpdateId: number, documentId: string, updatedDate: string, userId: string, deviceId: string, contentSize: number, }; diff --git a/frontend/sync-client/src/services/types/DocumentWithCursors.ts b/frontend/sync-client/src/services/types/DocumentWithCursors.ts index dcfe6e2d..38857a35 100644 --- a/frontend/sync-client/src/services/types/DocumentWithCursors.ts +++ b/frontend/sync-client/src/services/types/DocumentWithCursors.ts @@ -1,9 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { CursorSpan } from "./CursorSpan"; -export interface DocumentWithCursors { - vault_update_id: number | null; - document_id: string; - relative_path: string; - cursors: CursorSpan[]; -} +export type DocumentWithCursors = { vault_update_id: number | null, document_id: string, relative_path: string, cursors: Array, }; diff --git a/frontend/sync-client/src/services/types/EventBatch.ts b/frontend/sync-client/src/services/types/EventBatch.ts new file mode 100644 index 00000000..a3bfb573 --- /dev/null +++ b/frontend/sync-client/src/services/types/EventBatch.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EventRecord } from "./EventRecord"; + +export type EventBatch = { headEventId: number, +/** + * Last event in this page; `head_event_id` is the vault's current head. + */ +endEventId?: number, events: Array, }; diff --git a/frontend/sync-client/src/services/types/EventRecord.ts b/frontend/sync-client/src/services/types/EventRecord.ts new file mode 100644 index 00000000..47895639 --- /dev/null +++ b/frontend/sync-client/src/services/types/EventRecord.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; +import type { FileManifest } from "./FileManifest"; + +export type EventRecord = { eventId: number, requestId: string, } & ({ "type": "content", document: DocumentVersionWithoutContent, } | { "type": "fileManifest", fileManifest: FileManifest, }); diff --git a/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts b/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts deleted file mode 100644 index 160c9279..00000000 --- a/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts +++ /dev/null @@ -1,13 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; - -/** - * Response to a fetch latest documents request. - */ -export interface FetchLatestDocumentsResponse { - latestDocuments: DocumentVersionWithoutContent[]; - /** - * The update ID of the latest document in the response. - */ - lastUpdateId: bigint; -} diff --git a/frontend/sync-client/src/services/types/DeleteDocumentVersion.ts b/frontend/sync-client/src/services/types/FileManifest.ts similarity index 55% rename from frontend/sync-client/src/services/types/DeleteDocumentVersion.ts rename to frontend/sync-client/src/services/types/FileManifest.ts index 99ecc9e7..1d7ff7fc 100644 --- a/frontend/sync-client/src/services/types/DeleteDocumentVersion.ts +++ b/frontend/sync-client/src/services/types/FileManifest.ts @@ -1,5 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DeleteDocumentVersion { - relativePath: string; -} +export type FileManifest = { fileManifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts b/frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts new file mode 100644 index 00000000..4ccb541f --- /dev/null +++ b/frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FileManifest } from "./FileManifest"; + +export type FileManifestUpdateResponse = { "type": "Accepted", fileManifestId: number, } | { "type": "StaleBase" } & FileManifest; diff --git a/frontend/sync-client/src/services/types/PingResponse.ts b/frontend/sync-client/src/services/types/PingResponse.ts index 6db66354..c38845d2 100644 --- a/frontend/sync-client/src/services/types/PingResponse.ts +++ b/frontend/sync-client/src/services/types/PingResponse.ts @@ -3,23 +3,22 @@ /** * Response to a ping request. */ -export interface PingResponse { - /** - * Semantic version of the server. - */ - serverVersion: string; - /** - * Whether the client is authenticated based on the sent Authorization - * header. - */ - isAuthenticated: boolean; - /** - * List of file extensions that are allowed to be merged. - */ - mergeableFileExtensions: string[]; - /** - * API version ensuring backwards & forwards compatibility between the client - * and server. - */ - supportedApiVersion: number; -} +export type PingResponse = { +/** + * Semantic version of the server. + */ +serverVersion: string, +/** + * Whether the client is authenticated based on the sent Authorization + * header. + */ +isAuthenticated: boolean, +/** + * List of file extensions that are allowed to be merged. + */ +mergeableFileExtensions: Array, +/** + * API version ensuring backwards & forwards compatibility between the client + * and server. + */ +supportedApiVersion: number, }; diff --git a/frontend/sync-client/src/services/types/PushContent.ts b/frontend/sync-client/src/services/types/PushContent.ts new file mode 100644 index 00000000..6a4820ac --- /dev/null +++ b/frontend/sync-client/src/services/types/PushContent.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PushContent = { "type": "Snapshot", "value": string } | { "type": "Diff", "value": Array }; diff --git a/frontend/sync-client/src/services/types/PushFileManifest.ts b/frontend/sync-client/src/services/types/PushFileManifest.ts new file mode 100644 index 00000000..f7db488a --- /dev/null +++ b/frontend/sync-client/src/services/types/PushFileManifest.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PushFileManifest = { requestId: string, parentFileManifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/PutFileContent.ts b/frontend/sync-client/src/services/types/PutFileContent.ts new file mode 100644 index 00000000..c5a86c9b --- /dev/null +++ b/frontend/sync-client/src/services/types/PutFileContent.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { PushContent } from "./PushContent"; + +export type PutFileContent = { requestId: string, parentVersionId: number | null, content: PushContent, }; diff --git a/frontend/sync-client/src/services/types/SerializedError.ts b/frontend/sync-client/src/services/types/SerializedError.ts index ec1c4503..5e3fa9b9 100644 --- a/frontend/sync-client/src/services/types/SerializedError.ts +++ b/frontend/sync-client/src/services/types/SerializedError.ts @@ -1,7 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface SerializedError { - errorType: string; - message: string; - causes: string[]; -} +export type SerializedError = { errorType: string, message: string, causes: Array, }; diff --git a/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts b/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts deleted file mode 100644 index 4e57a297..00000000 --- a/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts +++ /dev/null @@ -1,7 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export interface UpdateDocumentVersion { - parent_version_id: bigint; - relative_path: string; - content: number[]; -} diff --git a/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts b/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts deleted file mode 100644 index 46f36bd0..00000000 --- a/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts +++ /dev/null @@ -1,7 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export interface UpdateTextDocumentVersion { - parentVersionId: number; - relativePath: string; - content: (number | string)[]; -} diff --git a/frontend/sync-client/src/services/types/VaultEvent.ts b/frontend/sync-client/src/services/types/VaultEvent.ts new file mode 100644 index 00000000..885a608b --- /dev/null +++ b/frontend/sync-client/src/services/types/VaultEvent.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; +import type { FileManifest } from "./FileManifest"; + +export type VaultEvent = { "type": "content", document: DocumentVersionWithoutContent, } | { "type": "fileManifest", fileManifest: FileManifest, }; diff --git a/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts b/frontend/sync-client/src/services/types/VaultSnapshot.ts similarity index 51% rename from frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts rename to frontend/sync-client/src/services/types/VaultSnapshot.ts index f1ea0f80..10b2470d 100644 --- a/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts +++ b/frontend/sync-client/src/services/types/VaultSnapshot.ts @@ -1,7 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; +import type { FileManifest } from "./FileManifest"; -export interface WebSocketVaultUpdate { - documents: DocumentVersionWithoutContent[]; - isInitialSync: boolean; -} +export type VaultSnapshot = { headEventId: number, fileManifest: FileManifest, documents: Array, }; diff --git a/frontend/sync-client/src/services/types/WebSocketClientMessage.ts b/frontend/sync-client/src/services/types/WebSocketClientMessage.ts index 9608f3af..5765a0d0 100644 --- a/frontend/sync-client/src/services/types/WebSocketClientMessage.ts +++ b/frontend/sync-client/src/services/types/WebSocketClientMessage.ts @@ -2,6 +2,4 @@ import type { CursorPositionFromClient } from "./CursorPositionFromClient"; import type { WebSocketHandshake } from "./WebSocketHandshake"; -export type WebSocketClientMessage = - | ({ type: "handshake" } & WebSocketHandshake) - | ({ type: "cursorPositions" } & CursorPositionFromClient); +export type WebSocketClientMessage = { "type": "handshake" } & WebSocketHandshake | { "type": "cursorPositions" } & CursorPositionFromClient; diff --git a/frontend/sync-client/src/services/types/WebSocketHandshake.ts b/frontend/sync-client/src/services/types/WebSocketHandshake.ts index a2910f49..310c0bb8 100644 --- a/frontend/sync-client/src/services/types/WebSocketHandshake.ts +++ b/frontend/sync-client/src/services/types/WebSocketHandshake.ts @@ -1,7 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface WebSocketHandshake { - token: string; - deviceId: string; - lastSeenVaultUpdateId: number | null; -} +export type WebSocketHandshake = { token: string, deviceId: string, }; diff --git a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts index fd250b7b..c9759e04 100644 --- a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts +++ b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts @@ -1,7 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { CursorPositionFromServer } from "./CursorPositionFromServer"; -import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate"; -export type WebSocketServerMessage = - | ({ type: "vaultUpdate" } & WebSocketVaultUpdate) - | ({ type: "cursorPositions" } & CursorPositionFromServer); +export type WebSocketServerMessage = { "type": "vaultChanged" } | { "type": "cursorPositions" } & CursorPositionFromServer; diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts index fef901e7..645fa627 100644 --- a/frontend/sync-client/src/services/websocket-manager.test.ts +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -1,9 +1,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ import { describe, it, beforeEach } from "node:test"; import assert from "node:assert"; +import { createServer } from "node:net"; import { WebSocketManager } from "./websocket-manager"; -import type { Logger } from "../tracing/logger"; -import type { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { Settings } from "../persistence/settings"; // eslint-disable-next-line @typescript-eslint/no-require-imports const WebSocket = require("ws") as typeof globalThis.WebSocket; @@ -134,9 +135,15 @@ describe("WebSocketManager", () => { const mockWs = (manager as unknown as { webSocket: MockWebSocket }) .webSocket; - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + mockWs.simulateMessage({ + type: "vaultChanged" + }); + mockWs.simulateMessage({ + type: "vaultChanged" + }); + mockWs.simulateMessage({ + type: "vaultChanged" + }); await new Promise((resolve) => setTimeout(resolve, 100)); @@ -195,8 +202,7 @@ describe("WebSocketManager", () => { manager.sendHandshakeMessage({ type: "handshake", token: "test", - deviceId: "test", - lastSeenVaultUpdateId: null + deviceId: "test" }); }); @@ -218,6 +224,31 @@ describe("WebSocketManager", () => { assert.ok(true); }); + it("uses a secure URL, preserves the configured prefix, and encodes the vault", async () => { + mockSettings = { + getSettings: () => ({ + remoteUri: "https://example.com/sync/", + vaultName: "team/a ?", + webSocketRetryIntervalMs: 1000 + }) + } as unknown as Settings; + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + manager.start(); + const mockWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + assert.strictEqual( + String(mockWs.url), + "wss://example.com/sync/vaults/team%2Fa%20%3F/ws" + ); + await manager.stop(); + }); + it("clears old handlers on reconnection", async () => { const manager = new WebSocketManager( deviceId, @@ -278,7 +309,9 @@ describe("WebSocketManager", () => { const mockWs = (manager as unknown as { webSocket: MockWebSocket }) .webSocket; - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + mockWs.simulateMessage({ + type: "vaultChanged" + }); await new Promise((resolve) => setTimeout(resolve, 10)); @@ -296,3 +329,176 @@ describe("WebSocketManager", () => { await manager.stop(); }); }); + +it( + "refused WebSocket connections and replacing a connecting socket do not emit unhandled errors", + { timeout: 5_000 }, + async () => { + const server = createServer(); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve) + ); + const address = server.address(); + assert(address && typeof address !== "string"); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ); + const logger = new Logger(); + const settings = new Settings( + logger, + { + remoteUri: `http://127.0.0.1:${address.port}`, + webSocketRetryIntervalMs: 10 + }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + WebSocket + ); + let closes = 0; + const disconnected = Promise.withResolvers(); + manager.onWebSocketStatusChanged.add((connected) => { + if (!connected && ++closes >= 2) disconnected.resolve(); + }); + try { + manager.start(); + manager.start(); + await disconnected.promise; + assert.equal(manager.isWebSocketConnected, false); + } finally { + await manager.stop(); + } + } +); + +for (const failedAttempt of [1, 2]) { + it(`recovers from a WebSocket constructor failure on attempt ${failedAttempt}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + let attempts = 0; + class FailingSocket extends MockWebSocket { + static OPEN = 1; + constructor(url: string) { + if (++attempts === failedAttempt) + throw new Error("socket allocation failed"); + super(url); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test", webSocketRetryIntervalMs: 1000 }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + FailingSocket as unknown as typeof WebSocket + ); + try { + assert.doesNotThrow(() => manager.start()); + context.mock.timers.tick(0); + if (failedAttempt === 2) { + const socket = ( + manager as unknown as { webSocket: MockWebSocket } + ).webSocket; + socket.close(); + } + assert.doesNotThrow(() => context.mock.timers.tick(1000)); + context.mock.timers.tick(1000); + assert.equal(attempts, failedAttempt + 1); + assert.equal(manager.isWebSocketConnected, true); + } finally { + await manager.stop(); + } + }); +} + +for (const closeBehavior of ["throws", "stays open"]) { + it(`stop retires a socket even when close ${closeBehavior}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + class BrokenCloseSocket extends MockWebSocket { + static OPEN = 1; + close() { + if (closeBehavior === "throws") throw new Error("close failed"); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test" }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + BrokenCloseSocket as unknown as typeof WebSocket + ); + let connected = false, + delivered = 0; + manager.onWebSocketStatusChanged.add((status) => { + connected = status; + }); + manager.onRemoteVaultUpdateReceived.add(async () => { + delivered++; + }); + manager.start(); + context.mock.timers.tick(0); + const socket = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + const stopping = manager.stop(); + context.mock.timers.tick(10_000); + await stopping; + assert.equal(manager.isWebSocketConnected, false); + assert.equal(connected, false); + socket.simulateMessage({ + type: "vaultChanged" + }); + await manager.waitUntilFinished(); + assert.equal(delivered, 0); + }); +} + +it("a handshake send failure retires the connection without escaping the open callback", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + class BrokenSendSocket extends MockWebSocket { + static OPEN = 1; + send() { + throw new Error("transport closed during open"); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test" }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + BrokenSendSocket as unknown as typeof WebSocket + ); + manager.onWebSocketStatusChanged.add((connected) => { + if (connected) + manager.sendHandshakeMessage({ + type: "handshake", + token: "test", + deviceId: "test" + }); + }); + const statuses: boolean[] = []; + manager.onWebSocketStatusChanged.add((status) => statuses.push(status)); + try { + manager.start(); + assert.doesNotThrow(() => context.mock.timers.tick(0)); + assert.equal(manager.isWebSocketConnected, false); + assert.equal(statuses.at(-1), false); + } finally { + await manager.stop(); + } +}); diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 09787bce..8d90fbbd 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -5,7 +5,6 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { ClientCursors } from "./types/ClientCursors"; import { createPromise } from "../utils/create-promise"; -import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate"; import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; import { removeFromArray } from "../utils/remove-from-array"; import { EventListeners } from "../utils/data-structures/event-listeners"; @@ -17,7 +16,7 @@ export class WebSocketManager { >(); public readonly onRemoteVaultUpdateReceived = new EventListeners< - (update: WebSocketVaultUpdate) => Promise + () => Promise >(); public readonly onRemoteCursorsUpdateReceived = new EventListeners< @@ -26,6 +25,7 @@ export class WebSocketManager { private isStopped = true; private resolveDisconnectingPromise: null | (() => unknown) = null; + private receiveTimeoutId: ReturnType | undefined; private reconnectTimeoutId: ReturnType | undefined; private readonly outstandingPromises: Promise[] = []; @@ -56,12 +56,14 @@ export class WebSocketManager { public get isWebSocketConnected(): boolean { return ( - this.webSocket?.readyState === - this.webSocketFactoryImplementation.OPEN + this.webSocket !== undefined && + this.webSocket.readyState === + this.webSocketFactoryImplementation.OPEN ); } public start(): void { + if (!this.isStopped) return; this.isStopped = false; this.initializeWebSocket(); } @@ -71,13 +73,20 @@ export class WebSocketManager { this.resolveDisconnectingPromise = resolve; this.isStopped = true; + clearTimeout(this.receiveTimeoutId); if (this.reconnectTimeoutId !== undefined) { clearTimeout(this.reconnectTimeoutId); this.reconnectTimeoutId = undefined; } - this.webSocket?.close(1000, "WebSocketManager has been stopped"); + const socket = this.webSocket; + try { + socket?.close(1000, "WebSocketManager has been stopped"); + } catch (error) { + this.logger.warn(`Failed to close WebSocket: ${String(error)}`); + if (socket) this.reconnect(socket); + } // eslint-disable-next-line @typescript-eslint/init-declarations let timeoutId: ReturnType | undefined; @@ -107,6 +116,9 @@ export class WebSocketManager { if (timeoutId !== undefined) { clearTimeout(timeoutId); } + // A throwing, silent or timed-out close must still fence callbacks + // before a stopped client can release its state or restart. + if (socket) this.reconnect(socket); } await this.waitUntilFinished(); @@ -132,6 +144,7 @@ export class WebSocketManager { this.logger.error( `Failed to send handshake message: ${String(error)}` ); + this.reconnect(webSocket); throw error; } } @@ -162,7 +175,66 @@ export class WebSocketManager { } } + private expectResponse(socket: WebSocket): void { + clearTimeout(this.receiveTimeoutId); + // Cursor heartbeats are acknowledged every 15 seconds, including when + // there are no open editors. An OPEN TCP socket alone proves nothing. + this.receiveTimeoutId = setTimeout(() => { + if (this.webSocket !== socket || this.isStopped) return; + this.logger.warn("WebSocket receive deadline expired"); + this.reconnect(socket); + }, 45_000); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Browser timers have no unref method. + this.receiveTimeoutId.unref?.(); + } + + private reconnect(socket: WebSocket): void { + if (this.webSocket !== socket) return; + clearTimeout(this.receiveTimeoutId); + socket.onopen = null; + socket.onclose = null; + socket.onmessage = null; + socket.onerror = (): void => { + /* Retired transports may still emit errors. */ + }; + this.webSocket = undefined; + this.onWebSocketStatusChanged.trigger(false); + try { + socket.close(); + } catch { + /* The transport may already be gone. */ + } + if (this.isStopped) { + this.resolveDisconnectingPromise?.(); + this.resolveDisconnectingPromise = null; + } else { + this.scheduleReconnect(); + } + } + + private scheduleReconnect(): void { + if (this.isStopped) return; + clearTimeout(this.reconnectTimeoutId); + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = undefined; + this.initializeWebSocket(); + }, this.settings.getSettings().webSocketRetryIntervalMs); + } + private initializeWebSocket(): void { + if (this.isStopped) return; + try { + this.openWebSocket(); + } catch (error) { + this.logger.warn(`Cannot initialize WebSocket: ${String(error)}`); + this.webSocket = undefined; + clearTimeout(this.receiveTimeoutId); + this.onWebSocketStatusChanged.trigger(false); + this.scheduleReconnect(); + } + } + + private openWebSocket(): void { // Clean up old WebSocket handlers to prevent race conditions if (this.webSocket) { try { @@ -170,7 +242,9 @@ export class WebSocketManager { this.webSocket.onopen = null; this.webSocket.onclose = null; this.webSocket.onmessage = null; - this.webSocket.onerror = null; + this.webSocket.onerror = (): void => { + /* Retired transports may still emit errors. */ + }; this.webSocket.close(); } catch (e) { this.logger.error( @@ -179,13 +253,23 @@ export class WebSocketManager { } } - const wsUri = new URL(this.settings.getSettings().remoteUri); - wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws"; - wsUri.pathname = `/vaults/${this.settings.getSettings().vaultName}/ws`; + const { remoteUri, vaultName } = this.settings.getSettings(); + const wsUri = new URL(remoteUri); + wsUri.protocol = wsUri.protocol === "https:" ? "wss:" : "ws:"; + wsUri.pathname = `${wsUri.pathname.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}/ws`; + wsUri.search = ""; + wsUri.hash = ""; this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); - this.webSocket = new this.webSocketFactoryImplementation(wsUri); + const socket = new this.webSocketFactoryImplementation(wsUri); + this.webSocket = socket; + this.expectResponse(socket); + + this.webSocket.onerror = (): void => { + this.logger.warn("WebSocket transport error"); + this.reconnect(socket); + }; this.webSocket.onopen = (): void => { // Check if we've been stopped while connecting @@ -197,7 +281,18 @@ export class WebSocketManager { return; } this.logger.info("WebSocket connection opened"); - this.onWebSocketStatusChanged.trigger(true); + try { + this.onWebSocketStatusChanged.trigger(true); + // A handshake failure can retire this socket from inside an + // observer. Finish delivery with its actual connection state. + if (this.webSocket !== socket) + this.onWebSocketStatusChanged.trigger(false); + } catch (error) { + this.logger.warn( + `WebSocket handshake failed: ${String(error)}` + ); + this.reconnect(socket); + } }; this.webSocket.onmessage = (event): void => { @@ -207,6 +302,8 @@ export class WebSocketManager { event.data ) as WebSocketServerMessage; + if (["vaultChanged", "cursorPositions"].includes(message.type)) + this.expectResponse(socket); // Track the message handling promise const messageHandlingPromise = this.handleWebSocketMessage( message @@ -235,25 +332,15 @@ export class WebSocketManager { this.logger.warn( `WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})` ); - this.onWebSocketStatusChanged.trigger(false); - - if (this.isStopped) { - this.resolveDisconnectingPromise?.(); - this.resolveDisconnectingPromise = null; - } else { - this.reconnectTimeoutId = setTimeout(() => { - this.reconnectTimeoutId = undefined; - this.initializeWebSocket(); - }, this.settings.getSettings().webSocketRetryIntervalMs); - } + this.reconnect(socket); }; } private async handleWebSocketMessage( message: WebSocketServerMessage ): Promise { - if (message.type === "vaultUpdate") { - await this.onRemoteVaultUpdateReceived.triggerAsync(message); + if (message.type === "vaultChanged") { + await this.onRemoteVaultUpdateReceived.triggerAsync(); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (message.type === "cursorPositions") { diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 2a272c86..7ea60cf1 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -1,62 +1,235 @@ -import type { PersistenceProvider } from "./persistence/persistence"; -import type { HistoryEntry, HistoryStats } from "./tracing/sync-history"; -import { SyncHistory } from "./tracing/sync-history"; -import { Logger, LogLevel, LogLine } from "./tracing/logger"; -import type { RelativePath, StoredDatabase } from "./persistence/database"; -import { Database } from "./persistence/database"; import * as Sentry from "@sentry/browser"; -import type { SyncSettings } from "./persistence/settings"; -import { DEFAULT_SETTINGS, Settings } from "./persistence/settings"; -import { SyncService } from "./services/sync-service"; -import { Syncer } from "./sync-operations/syncer"; +import { setUpTelemetry } from "./utils/set-up-telemetry"; +import type { PersistenceProvider } from "./persistence/persistence"; +import { Database, type StoredDatabase } from "./persistence/database"; +import { + Settings, + DEFAULT_SETTINGS, + type SyncSettings +} from "./persistence/settings"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; -import { FetchController } from "./services/fetch-controller"; -import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer"; -import { rateLimit } from "./utils/rate-limit"; -import type { NetworkConnectionStatus } from "./types/network-connection-status"; -import { DocumentSyncStatus } from "./types/document-sync-status"; -import { WebSocketManager } from "./services/websocket-manager"; -import { createClientId } from "./utils/create-client-id"; -import { CursorTracker } from "./sync-operations/cursor-tracker"; -import type { CursorSpan } from "./services/types/CursorSpan"; -import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; -import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; -import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; -import { setUpTelemetry } from "./utils/set-up-telemetry"; -import { DIFF_CACHE_SIZE_MB } from "./consts"; +import { Logger, LogLevel } from "./tracing/logger"; +import { SyncHistory } from "./tracing/sync-history"; +import { SyncService } from "./services/sync-service"; import { ServerConfig } from "./services/server-config"; -import type { EventListeners } from "./utils/data-structures/event-listeners"; +import { WebSocketManager } from "./services/websocket-manager"; +import { Syncer } from "./sync-operations/syncer"; +import { CursorTracker } from "./sync-operations/cursor-tracker"; +import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; +import { createClientId } from "./utils/create-client-id"; +import { DocumentSyncStatus } from "./types/document-sync-status"; +import type { CursorSpan } from "./services/types/CursorSpan"; +import { isInternalPath } from "./utils/portable-path"; +import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; +import { Lock } from "./utils/data-structures/locks"; +import { + unappliedChanges, + type LocalChange +} from "./sync-operations/local-changes"; + +export type StoredClient = Partial<{ + settings: Partial; + database: Partial; + localChanges: ( + | LocalChange + | { type: "update"; path: string; changeId?: string } + )[]; + localChangesVaultKey: string; + historyCheckpoint: { vaultKey: string; checkpoint?: string }; +}>; + +const vaultKey = (settings: SyncSettings): string => + JSON.stringify([ + settings.remoteUri.replace(/\/$/u, ""), + settings.vaultName + ]); export class SyncClient { - private hasStartedOfflineSync = false; - private hasFinishedOfflineSync = false; - private hasStarted = false; - private hasBeenDestroyed = false; + private readonly lifecycle = new Lock(); + private destroying?: Promise; + private started = false; + private destroyed = false; private unloadTelemetry?: () => void; - private isDestroying = false; - private readonly eventUnsubscribers: (() => void)[] = []; - + private readonly unsubscribeLog: () => void; private constructor( + public readonly logger: Logger, private readonly history: SyncHistory, private readonly settings: Settings, private readonly database: Database, private readonly syncer: Syncer, private readonly webSocketManager: WebSocketManager, - public readonly logger: Logger, - private readonly fetchController: FetchController, + private readonly service: SyncService, + private readonly serverConfig: ServerConfig, private readonly cursorTracker: CursorTracker, private readonly fileChangeNotifier: FileChangeNotifier, private readonly contentCache: FixedSizeDocumentCache, - private readonly fileOperations: FileOperations, - private readonly serverConfig: ServerConfig, - private readonly persistence: PersistenceProvider< - Partial<{ - settings: Partial; - database: Partial; - }> - > - ) {} + private readonly persistence: PersistenceProvider + ) { + syncer.onServerHistoryChanged.add(() => { + cursorTracker.reset(); + }); + syncer.onReadyForEvents.add(() => { + if ( + this.started && + !this.destroyed && + settings.getSettings().isSyncEnabled + ) + webSocketManager.start(); + }); + this.unsubscribeLog = logger.onLogEmitted.add((log) => { + if (log.level === LogLevel.ERROR && Sentry.isInitialized()) + Sentry.captureMessage(log.message); + }); + } + + public static async create({ + fs, + persistence, + fetch, + webSocket + }: { + fs: FileSystemOperations; + persistence: PersistenceProvider; + fetch?: typeof globalThis.fetch; + webSocket?: typeof globalThis.WebSocket; + }): Promise { + const logger = new Logger(); + let stored = (await persistence.load()) ?? {}; + let saving: Promise = Promise.resolve(); + let reloadBeforeSave = false; + // Settings and engine state share a store. Serialize complete metadata + // replacements so one cannot overwrite a newer save from the other. + const save = async (update: StoredClient): Promise => { + const operation = saving.then(async () => { + if (reloadBeforeSave) { + stored = (await persistence.load()) ?? {}; + reloadBeforeSave = false; + } + const next = { ...stored, ...structuredClone(update) }; + try { + await persistence.save(next); + stored = next; + } catch (error) { + // A failed save may have committed. Do not let a queued + // settings/notification save roll back that engine state. + reloadBeforeSave = true; + throw error; + } + }); + saving = operation.catch(() => {}); + return operation; + }; + const settings = new Settings(logger, stored.settings, async (data) => + save({ settings: data }) + ); + const database = new Database( + stored.database, + async (data) => save({ database: data }), + vaultKey(settings.getSettings()), + async () => { + await saving; + stored = (await persistence.load()) ?? {}; + return stored.database as StoredDatabase | undefined; + } + ); + // Older stores also queued content updates. Trim the acknowledged prefix + // before discarding them, since its marker may belong to an update. + const localChanges = unappliedChanges( + database.state, + stored.localChanges ?? [] + // eslint-disable-next-line no-restricted-syntax -- Migration needs a narrowed copy, not removal of a specific object. + ).filter((change) => change.type !== "update"); + if ( + localChanges.length && + stored.localChangesVaultKey !== vaultKey(settings.getSettings()) + ) { + throw new Error( + "Offline notifications belong to another vault; use a separate state store" + ); + } + const deviceId = createClientId(); + const service = new SyncService(deviceId, settings, fetch, { + get: () => + stored.historyCheckpoint?.vaultKey === + vaultKey(settings.getSettings()) + ? stored.historyCheckpoint.checkpoint + : undefined, + save: async (checkpoint) => + save({ + historyCheckpoint: { + vaultKey: vaultKey(settings.getSettings()), + checkpoint + } + }) + }); + const serverConfig = new ServerConfig(service); + const notifier = new FileChangeNotifier(); + let syncer: Syncer; + const files = new FileOperations( + fs, + database, + serverConfig, + (paths) => { + for (const path of paths) notifier.notifyOfFileChange(path); + }, + { + entries: () => localChanges, + flush: async () => syncer?.flushLocalChanges() + }, + (path, size) => + settings.isIgnored(path) || settings.isOversized(size) + ); + const websocket = new WebSocketManager( + deviceId, + logger, + settings, + webSocket + ); + const history = new SyncHistory(logger); + const contentCache = new FixedSizeDocumentCache( + settings.getSettings().diffCacheSizeMB * 1024 * 1024 + ); + syncer = new Syncer( + deviceId, + logger, + database, + settings, + service, + websocket, + files, + serverConfig, + history, + contentCache, + { + entries: localChanges, + save: async (entries) => + save({ + localChanges: entries, + localChangesVaultKey: vaultKey(settings.getSettings()) + }) + } + ); + const cursors = new CursorTracker(database, websocket, files, notifier); + return new SyncClient( + logger, + history, + settings, + database, + syncer, + websocket, + service, + serverConfig, + cursors, + notifier, + contentCache, + persistence + ); + } + + private check(): void { + if (this.destroyed) throw new Error("SyncClient has been destroyed"); + } public get documentCount(): number { return this.database.length; @@ -66,478 +239,227 @@ export class SyncClient { return this.webSocketManager.isWebSocketConnected; } - public get onSyncHistoryUpdated(): EventListeners< - (stats: HistoryStats) => unknown - > { - this.checkIfDestroyed("onSyncHistoryUpdated getter"); + public get onSyncHistoryUpdated() { return this.history.onHistoryUpdated; } - public get onSettingsChanged(): EventListeners< - (newSettings: SyncSettings, oldSettings: SyncSettings) => unknown - > { - this.checkIfDestroyed("onSettingsChanged getter"); + public get onSettingsChanged() { return this.settings.onSettingsChanged; } - public get onRemainingOperationsCountChanged(): EventListeners< - (remainingOperationsCount: number) => unknown - > { - this.checkIfDestroyed("onRemainingOperationsCountChanged getter"); + public get onRemainingOperationsCountChanged() { return this.syncer.onRemainingOperationsCountChanged; } - public get onWebSocketStatusChanged(): EventListeners< - (isConnected: boolean) => unknown - > { - this.checkIfDestroyed("onWebSocketStatusChanged getter"); + public get onWebSocketStatusChanged() { return this.webSocketManager.onWebSocketStatusChanged; } - public get onRemoteCursorsUpdated(): EventListeners< - (cursors: MaybeOutdatedClientCursors[]) => unknown - > { - this.checkIfDestroyed("onRemoteCursorsUpdated getter"); + public get onRemoteCursorsUpdated() { return this.cursorTracker.onRemoteCursorsUpdated; } - public static async create({ - fs, - persistence, - fetch, - webSocket, - nativeLineEndings = "\n" - }: { - fs: FileSystemOperations; - persistence: PersistenceProvider< - Partial<{ - settings: Partial; - database: Partial; - }> - >; - fetch?: typeof globalThis.fetch; - webSocket?: typeof globalThis.WebSocket; - nativeLineEndings?: string; - }): Promise { - const logger = new Logger(); - - const deviceId = createClientId(); - - logger.info(`Creating SyncClient with client id ${deviceId}`); - - const history = new SyncHistory(logger); - - let state = (await persistence.load()) ?? { - settings: undefined, - database: undefined - }; - - const settings = new Settings( - logger, - state.settings, - async (data): Promise => { - state = { ...state, settings: data }; - // we're not rate-limiting settings saves as (1) we need to initialise the settings to know the rate limit - // and (2) settings changes are infrequent enough that rate-limiting is not necessary - await persistence.save(state); - } - ); - - const rateLimitedSave = rateLimit( - persistence.save, - () => settings.getSettings().minimumSaveIntervalMs - ); - - const database = new Database( - logger, - state.database, - async (data): Promise => { - state = { ...state, database: data }; - await rateLimitedSave(state); - } - ); - - const fetchController = new FetchController( - settings.getSettings().isSyncEnabled, - logger - ); - - const syncService = new SyncService( - deviceId, - fetchController, - settings, - logger, - fetch - ); - - const serverConfig = new ServerConfig(syncService); - - const fileOperations = new FileOperations( - logger, - database, - fs, - serverConfig, - nativeLineEndings - ); - - const contentCache = new FixedSizeDocumentCache( - 1024 * 1024 * DIFF_CACHE_SIZE_MB - ); - const unrestrictedSyncer = new UnrestrictedSyncer( - logger, - database, - settings, - syncService, - fileOperations, - history, - contentCache, - serverConfig - ); - - const webSocketManager = new WebSocketManager( - deviceId, - logger, - settings, - webSocket - ); - - const syncer = new Syncer( - deviceId, - logger, - database, - settings, - syncService, - webSocketManager, - fileOperations, - unrestrictedSyncer - ); - - const fileChangeNotifier = new FileChangeNotifier(); - const cursorTracker = new CursorTracker( - database, - webSocketManager, - fileOperations, - fileChangeNotifier - ); - const client = new SyncClient( - history, - settings, - database, - syncer, - webSocketManager, - logger, - fetchController, - cursorTracker, - fileChangeNotifier, - contentCache, - fileOperations, - serverConfig, - persistence - ); - - logger.info("SyncClient created successfully"); - - return client; - } - - public async start(): Promise { - this.checkIfDestroyed("start"); - - if (this.hasStarted) { - throw new Error("SyncClient has already been started"); - } - this.hasStarted = true; - - if ( - !this.unloadTelemetry && - this.settings.getSettings().enableTelemetry - ) { - this.unloadTelemetry = setUpTelemetry(); - } - - this.eventUnsubscribers.push( - this.settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) { - this.fetchController.canFetch = newSettings.isSyncEnabled; - } - }) - ); - - this.eventUnsubscribers.push( - this.logger.onLogEmitted.add((log): void => { - if (log.level === LogLevel.ERROR && Sentry.isInitialized()) { - Sentry.captureMessage(log.message); - } - }) - ); - - this.eventUnsubscribers.push( - this.settings.onSettingsChanged.add( - this.onSettingsChange.bind(this) - ) - ); - - if (this.settings.getSettings().isSyncEnabled) { - this.logger.info("Starting SyncClient"); - await this.startSyncing(); - this.logger.info("SyncClient has successfully started"); - } - } - - /** - * Reload settings from disk overriding current in-memory settings. - * Missing values will be filled in from DEFAULT_SETTINGS rather than - * retaining current in-memory settings. - */ - public async reloadSettings(): Promise { - this.checkIfDestroyed("reloadSettings"); - - const state = (await this.persistence.load()) ?? { - settings: undefined - }; - - const settings = { - ...DEFAULT_SETTINGS, - ...(state.settings ?? {}) - }; - - await this.setSettings(settings); - } - - public async checkConnection(): Promise { - this.checkIfDestroyed("checkConnection"); - - const server = await this.serverConfig.checkConnection(true); - return { - isSuccessful: server.isSuccessful, - serverMessage: server.message, - isWebSocketConnected: this.webSocketManager.isWebSocketConnected - }; - } - - public getHistoryEntries(): readonly HistoryEntry[] { + public getHistoryEntries() { return this.history.entries; } - /** - * Wait for the in-flight operations to finish, reset all tracking, - * and the local database but retain the settings. - * The SyncClient can be used again after calling this method. - */ - public async reset(): Promise { - this.checkIfDestroyed("reset"); - - this.logger.info( - "Stopping SyncClient to apply changed connection settings" - ); - await this.pause(); - - // clear all local state - this.logger.info("Resetting SyncClient's local state"); - this.database.reset(); - await this.database.save(); // ensure the new database reads as empty - this.resetInMemoryState(); - this.hasStartedOfflineSync = false; - this.hasFinishedOfflineSync = false; - this.serverConfig.reset(); - - await this.startSyncing(); - } - public getSettings(): SyncSettings { return this.settings.getSettings(); } - public async setSetting( - key: T, - value: SyncSettings[T] - ): Promise { - this.checkIfDestroyed("setSetting"); - - await this.settings.setSetting(key, value); - } - - public async setSettings(value: Partial): Promise { - this.checkIfDestroyed("setSettings"); - - await this.settings.setSettings(value); - } - - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { - this.checkIfDestroyed("syncLocallyCreatedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyCreatedFile(relativePath); - } - - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - this.checkIfDestroyed("syncLocallyDeletedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyDeletedFile(relativePath); - } - - public async syncLocallyUpdatedFile({ - oldPath, - relativePath - }: { - oldPath?: RelativePath; - relativePath: RelativePath; - }): Promise { - this.checkIfDestroyed("syncLocallyUpdatedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyUpdatedFile({ - oldPath, - relativePath - }); - } - - public getDocumentSyncingStatus( - relativePath: RelativePath - ): DocumentSyncStatus { - this.checkIfDestroyed("getDocumentSyncingStatus"); - - if (!this.settings.getSettings().isSyncEnabled) { - return DocumentSyncStatus.SYNCING_IS_DISABLED; - } - - if (!this.syncer.isFirstSyncComplete || !this.hasFinishedOfflineSync) { - return DocumentSyncStatus.SYNCING; - } - - const document = - this.database.getLatestDocumentByRelativePath(relativePath); - if (document === undefined) { - return DocumentSyncStatus.SYNCING; - } - return document.updates.length > 0 - ? DocumentSyncStatus.SYNCING - : DocumentSyncStatus.UP_TO_DATE; - } - - public async updateLocalCursors( - documentToCursors: Record - ): Promise { - this.checkIfDestroyed("updateLocalCursors"); - - await this.cursorTracker.sendLocalCursorsToServer(documentToCursors); - } - - public async waitUntilFinished(): Promise { - this.checkIfDestroyed("waitUntilIdle"); - await this.syncer.waitUntilFinished(); - await this.webSocketManager.waitUntilFinished(); - await this.database.save(); // flush all changes to disk - } - - /** - * Completely destroy the SyncClient, cancelling all in-progress operations. - * After calling this method, the SyncClient cannot be used again. - */ - public async destroy(): Promise { - this.checkIfDestroyed("destroy"); - - // Prevent concurrent destroy calls - if (this.isDestroying) { - this.logger.warn( - "destroy() called while already destroying, ignoring" + public async start(): Promise { + await this.lifecycle.withLock(async () => { + this.check(); + if (this.started) + throw new Error("SyncClient has already been started"); + await this.syncer.reloadLocalState(); + await this.database.bindVault( + vaultKey(this.settings.getSettings()) ); - return; - } - this.isDestroying = true; - - // cancel everything that's in progress - await this.pause(); - - this.hasBeenDestroyed = true; - - this.resetInMemoryState(); - - // Clean up event listeners to prevent memory leaks - this.eventUnsubscribers.forEach((unsubscribe) => { - unsubscribe(); + this.started = true; + this.configureTelemetry(); + if (this.settings.getSettings().isSyncEnabled) await this.resume(); }); - this.eventUnsubscribers.length = 0; - - this.logger.info("SyncClient has been successfully disposed"); - - this.unloadTelemetry?.(); } - private async startSyncing(): Promise { - this.checkIfDestroyed("startSyncing"); - this.fetchController.finishReset(); - - await this.serverConfig.initialize(); - this.webSocketManager.start(); - - if (!this.hasStartedOfflineSync) { - this.hasStartedOfflineSync = true; - await this.syncer.scheduleSyncForOfflineChanges(); - } - - this.hasFinishedOfflineSync = true; + private async resume(): Promise { + if (this.destroyed) return; + if (!this.settings.getSettings().isSyncEnabled) return; + await this.database.bindVault(vaultKey(this.settings.getSettings())); + this.service.resume(); + this.syncer.start(); + await this.syncer.waitUntilFinished(); } private async pause(): Promise { - this.fetchController.startReset(); + const stopped = this.syncer.stop(); + this.service.pause(); await this.webSocketManager.stop(); - await this.waitUntilFinished(); + await stopped; } - private resetInMemoryState(): void { - this.history.reset(); - this.contentCache.reset(); - // don't reset the logger - this.cursorTracker.reset(); - this.syncer.reset(); - this.fileOperations.reset(); + /** Restart transports; never discard pending requests. */ + public async reset(): Promise { + await this.lifecycle.withLock(async () => { + this.check(); + await this.pause(); + this.serverConfig.reset(); + this.cursorTracker.reset(); + await this.syncer.retryRejectedRequests(); + if (this.started && this.settings.getSettings().isSyncEnabled) + await this.resume(); + }); } - private async onSettingsChange( - newSettings: SyncSettings, - oldSettings: SyncSettings - ): Promise { - this.checkIfDestroyed("onSettingsChange"); + public async setSettings(change: Partial): Promise { + const update = structuredClone(change); + const notification = await this.lifecycle.withLock(async () => { + this.check(); + const old = this.settings.getSettings(); + const next = { ...old, ...update }; + if ( + vaultKey(next) !== vaultKey(old) && + (this.started || + this.database.state.initialized || + this.database.state.bootstrap || + this.database.state.pending || + this.syncer.hasLocalChanges || + Object.keys(this.database.state.local).length) + ) + throw new Error( + "Changing vaults requires a separate state store and client; existing sync state must not be reset" + ); - if ( - newSettings.vaultName !== oldSettings.vaultName || - newSettings.remoteUri !== oldSettings.remoteUri - ) { - await this.reset(); - } - - if (newSettings.isSyncEnabled !== oldSettings.isSyncEnabled) { - if (newSettings.isSyncEnabled) { - await this.startSyncing(); - } else { - await this.pause(); + try { + if (this.started) await this.pause(); + await this.database.recoverPersistence(); + await this.settings.setSettings(update, false); + await this.syncer.retryRejectedRequests(); + this.contentCache.resize(next.diffCacheSizeMB * 1024 * 1024); + await this.database.bindVault(vaultKey(next)); + } catch (error) { + this.serverConfig.reset(); + this.contentCache.resize( + this.settings.getSettings().diffCacheSizeMB * 1024 * 1024 + ); + this.configureTelemetry(); + if (this.started && this.settings.getSettings().isSyncEnabled) + await this.resume().catch((resumeError: unknown) => { + this.logger.error( + `Settings recovery could not resume sync: ${String(resumeError)}` + ); + }); + throw error; } - } + this.serverConfig.reset(); + this.configureTelemetry(); + if (this.started && next.isSyncEnabled) await this.resume(); + return [this.settings.getSettings(), old] as const; + }); + await this.settings.onSettingsChanged.triggerAsync(...notification); + } - if (newSettings.diffCacheSizeMB !== oldSettings.diffCacheSizeMB) { - this.contentCache.resize(newSettings.diffCacheSizeMB * 1024 * 1024); - } + public async setSetting( + key: K, + value: SyncSettings[K] + ): Promise { + await this.setSettings({ [key]: value }); + } - if (newSettings.enableTelemetry !== oldSettings.enableTelemetry) { - if (newSettings.enableTelemetry) { - this.unloadTelemetry = setUpTelemetry(); - } else { + public async reloadSettings(): Promise { + const stored = await this.persistence.load(); + await this.setSettings({ ...DEFAULT_SETTINGS, ...stored?.settings }); + } + + public async checkConnection() { + const result = await this.serverConfig.checkConnection(); + return { + isSuccessful: result.isSuccessful, + serverMessage: result.message, + isWebSocketConnected: this.isWebSocketConnected + }; + } + + /** Notifications enqueue work. Use waitUntilFinished() to await convergence. */ + public async syncLocallyCreatedFile(path: string): Promise { + this.check(); + if (isInternalPath(path)) return; + this.fileChangeNotifier.notifyOfFileChange(path); + await this.syncer.syncLocallyCreatedFile(path); + } + + public async syncLocallyDeletedFile(path: string): Promise { + this.check(); + if (isInternalPath(path)) return; + this.fileChangeNotifier.notifyOfFileChange(path); + await this.syncer.syncLocallyDeletedFile(path); + } + + public async syncLocallyUpdatedFile(change: { + oldPath?: string; + relativePath: string; + }): Promise { + this.check(); + if ( + isInternalPath(change.relativePath) || + (change.oldPath && isInternalPath(change.oldPath)) + ) + return; + this.fileChangeNotifier.notifyOfFileChange(change.relativePath); + await this.syncer.syncLocallyUpdatedFile(change); + } + + public async updateLocalCursors( + cursors: Record + ): Promise { + this.check(); + await this.cursorTracker.sendLocalCursorsToServer(cursors); + } + + public getDocumentSyncingStatus(path: string): DocumentSyncStatus { + this.check(); + if (!this.settings.getSettings().isSyncEnabled) + return DocumentSyncStatus.SYNCING_IS_DISABLED; + return this.syncer.isDocumentUpToDate(path) + ? DocumentSyncStatus.UP_TO_DATE + : DocumentSyncStatus.SYNCING; + } + + public async waitUntilFinished(): Promise { + this.check(); + await this.syncer.waitUntilFinished(); + } + + private configureTelemetry(): void { + if ( + this.settings.getSettings().enableTelemetry && + typeof window !== "undefined" + ) { + this.unloadTelemetry ??= setUpTelemetry(); + } else { + this.unloadTelemetry?.(); + this.unloadTelemetry = undefined; + } + } + + public async destroy(): Promise { + if (this.destroying) return this.destroying; + this.destroyed = true; + // Interrupt a start/reset awaiting network progress before waiting for + // its lifecycle operation to finish. + this.service.pause(); + const stopped = this.syncer.stop(); + this.destroying = this.lifecycle.withLock(async () => { + try { + await this.webSocketManager.stop(); + await stopped; + await this.syncer.flushLocalChanges(); + } finally { + this.cursorTracker.reset(); + this.unsubscribeLog(); this.unloadTelemetry?.(); } - } - } - - private checkIfDestroyed(origin: string): void { - if (this.hasBeenDestroyed) { - throw new Error( - `SyncClient has been destroyed and can no longer be used; called from ${origin}` - ); - } + }); + return this.destroying; } } diff --git a/frontend/sync-client/src/sync-operations/content.test.ts b/frontend/sync-client/src/sync-operations/content.test.ts new file mode 100644 index 00000000..db446097 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/content.test.ts @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { mergeContent, fromStoredSnapshot, toStoredSnapshot } from "./content"; + +const extensions = ["md"]; +const snapshot = async (text: string) => + toStoredSnapshot({ content: new TextEncoder().encode(text) }); + +describe("content reconciliation without file timestamps", () => { + it("round-trips content and editor selections without timestamps", async () => { + const original = { + content: new Uint8Array([0, 255, 13, 10]), + cursors: [] + }; + assert.deepEqual( + fromStoredSnapshot(await toStoredSnapshot(original)), + original + ); + }); + + it("keeps local binary edits when the server content is unchanged", async () => { + const base = await snapshot("\0base"); + const local = await snapshot("\0local"); + const merged = await mergeContent( + "file.bin", + base, + local, + base, + extensions + ); + assert.equal(merged.contentBase64, local.contentBase64); + }); + + it("incorporates server binary edits when local content is unchanged", async () => { + const base = await snapshot("\0base"); + const remote = await snapshot("\0remote"); + const merged = await mergeContent( + "file.bin", + base, + base, + remote, + extensions + ); + assert.equal(merged.contentBase64, remote.contentBase64); + }); + + it("keeps the server on concurrent binary or unmergeable text edits", async () => { + for (const [path, prefix] of [ + ["file.bin", "\0"], + ["file.md", "\0"], + ["file.json", ""] + ]) { + const base = await snapshot(`${prefix}base`); + const local = await snapshot(`${prefix}local`); + const remote = await snapshot(`${prefix}remote`); + const merged = await mergeContent( + path, + base, + local, + remote, + extensions + ); + assert.equal(merged.contentBase64, remote.contentBase64, path); + } + }); + + it("keeps the server for differing binary content without a common base", async () => { + const local = await snapshot("\0local"); + const remote = await snapshot("\0remote"); + const merged = await mergeContent( + "file.bin", + undefined, + local, + remote, + extensions + ); + assert.equal(merged.contentBase64, remote.contentBase64); + }); + + it("still merges independent text edits from both sides", async () => { + const merged = await mergeContent( + "note.md", + await snapshot("First paragraph.\n\nLast paragraph.\n"), + await snapshot("Updated first paragraph.\n\nLast paragraph.\n"), + await snapshot("First paragraph.\n\nUpdated last paragraph.\n"), + extensions + ); + assert.equal( + new TextDecoder().decode(fromStoredSnapshot(merged).content), + "Updated first paragraph.\n\nUpdated last paragraph.\n" + ); + }); +}); diff --git a/frontend/sync-client/src/sync-operations/content.ts b/frontend/sync-client/src/sync-operations/content.ts new file mode 100644 index 00000000..8a5aafaf --- /dev/null +++ b/frontend/sync-client/src/sync-operations/content.ts @@ -0,0 +1,68 @@ +import { base64ToBytes, bytesToBase64 } from "byte-base64"; +import { reconcile } from "reconcile-text"; +import type { FileSnapshot } from "../file-operations/filesystem-operations"; +import type { StoredSnapshot } from "../persistence/database"; +import { hash } from "../utils/hash"; +import { isBinary } from "../utils/is-binary"; +import { isFileTypeMergable } from "../utils/is-file-type-mergable"; + +export async function toStoredSnapshot( + snapshot: FileSnapshot +): Promise { + return { + contentBase64: bytesToBase64(snapshot.content), + cursors: snapshot.cursors, + hash: await hash(snapshot.content) + }; +} + +export function fromStoredSnapshot(snapshot: StoredSnapshot): FileSnapshot { + return { + content: base64ToBytes(snapshot.contentBase64), + cursors: snapshot.cursors + }; +} + +export async function mergeContent( + path: string, + base: StoredSnapshot | undefined, + local: StoredSnapshot, + remote: StoredSnapshot, + extensions: string[] +): Promise { + if (local.contentBase64 === remote.contentBase64) + return { + ...remote, + cursors: local.cursors + }; + + if (!local.cursors && local.contentBase64 === base?.contentBase64) + return remote; + + if (remote.contentBase64 === base?.contentBase64) return local; + + const inputs = [ + base?.contentBase64 ?? "", + local.contentBase64, + remote.contentBase64 + ].map(base64ToBytes); + + // Concurrent unmergeable edits keep the server's content. + if (!isFileTypeMergable(path, extensions) || inputs.some(isBinary)) + return remote; + + const [parentText, localText, remoteText] = inputs.map((bytes) => + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes) + ); + + const merged = reconcile( + parentText, + { text: localText, cursors: local.cursors }, + remoteText + ); + + return toStoredSnapshot({ + content: new TextEncoder().encode(merged.text), + cursors: local.cursors ? merged.cursors : undefined + }); +} diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts new file mode 100644 index 00000000..9b1c80da --- /dev/null +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts @@ -0,0 +1,275 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import assert from "node:assert/strict"; +import { it } from "node:test"; +import type { Database } from "../persistence/database"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { ClientCursors } from "../services/types/ClientCursors"; +import { DocumentUpToDateness } from "../types/document-up-to-dateness"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { hash } from "../utils/hash"; +import { CursorTracker } from "./cursor-tracker"; +import { FileChangeNotifier } from "./file-change-notifier"; + +it("retains future cursors and publishes them when content catches up", async () => { + const content = new TextEncoder().encode("current"); + const contentHash = await hash(content); + const database = { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata: { parentVersionId: 2, hash: contentHash } + }) + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const files = { read: async () => content } as unknown as FileOperations; + const notifier = new FileChangeNotifier(); + const tracker = new CursorTracker(database, websocket, files, notifier); + const future = { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "old-name.md", + vault_update_id: 2, + cursors: [{ start: 0, end: 1 }] + } + ], + upToDateness: DocumentUpToDateness.Later + }; + const internals = tracker as unknown as { + knownRemoteCursors: (typeof future)[]; + getRelevantAndPruneKnownClientCursors: () => ClientCursors[]; + }; + internals.knownRemoteCursors = [future]; + + assert.deepEqual(internals.getRelevantAndPruneKnownClientCursors(), []); + assert.strictEqual(internals.knownRemoteCursors.length, 1); + + let received: ClientCursors[] = []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + received = cursors; + }); + await notifier.onFileChanged.triggerAsync("note.md"); + + assert.strictEqual(received.length, 1); + assert.strictEqual( + received[0].documentsWithCursors[0].relative_path, + "note.md" + ); +}); + +it("refreshes unchanged cursors on reconnect and heartbeat, and follows the document through renames", async (context) => { + context.mock.timers.enable({ apis: ["setInterval"] }); + const content = new TextEncoder().encode("hello"); + let path = "note.md"; + const record = () => ({ + documentId: "document", + relativePath: path, + metadata: { parentVersionId: 1, hash: contentHash } + }); + const contentHash = await hash(content); + const database = { + getLatestDocumentByRelativePath: (candidate: string) => + candidate === path ? record() : undefined, + getDocumentByDocumentId: () => record() + } as unknown as Database; + const sent: { + documentsWithCursors: { + relative_path: string; + vault_update_id: number | null; + }[]; + }[] = []; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: (value: (typeof sent)[number]) => { + sent.push(value); + } + } as unknown as WebSocketManager; + const notifier = new FileChangeNotifier(); + const tracker = new CursorTracker( + database, + websocket, + { read: async () => content } as unknown as FileOperations, + notifier + ); + const drain = async () => { + for (let i = 0; i < 20; i++) + await new Promise((resolve) => setImmediate(resolve)); + }; + try { + await tracker.sendLocalCursorsToServer({ + "note.md": [{ start: 1, end: 2 }] + }); + assert.equal(sent.length, 1); + await tracker.sendLocalCursorsToServer({ + "note.md": [{ start: 1, end: 2 }] + }); + assert.equal(sent.length, 1); + websocket.onWebSocketStatusChanged.trigger(true); + await drain(); + assert.equal(sent.length, 2); + context.mock.timers.tick(15_000); + await drain(); + assert.equal(sent.length, 3); + path = "renamed.md"; + await notifier.onFileChanged.triggerAsync(path); + assert.equal(sent.at(-1)!.documentsWithCursors[0].relative_path, path); + websocket.onWebSocketStatusChanged.trigger(false); + const count = sent.length; + context.mock.timers.tick(30_000); + await drain(); + assert.equal(sent.length, count); + websocket.onWebSocketStatusChanged.trigger(true); + await drain(); + assert.equal(sent.length, count + 1); + } finally { + tracker.reset(); + context.mock.timers.reset(); + } +}); + +for (const interruption of ["disconnect", "reset"] as const) { + it(`an in-flight cursor read cannot resurrect presence after ${interruption}`, async () => { + const content = new TextEncoder().encode("current"); + const contentHash = await hash(content); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const database = { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata: { parentVersionId: 1, hash: contentHash } + }) + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const tracker = new CursorTracker( + database, + websocket, + { + read: async () => { + entered.resolve(); + await release.promise; + return content; + } + } as unknown as FileOperations, + new FileChangeNotifier() + ); + let received: ClientCursors[] = []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + received = cursors; + }); + const cursor: ClientCursors = { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 1 }] + } + ] + }; + const first = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + cursor + ]); + await entered.promise; + const queued = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + cursor + ]); + if (interruption === "reset") tracker.reset(); + else websocket.onWebSocketStatusChanged.trigger(false); + assert.deepEqual(received, []); + release.resolve(); + await first; + await queued; + assert.deepEqual( + received, + [], + "a retired connection must not restore its cursors" + ); + tracker.reset(); + }); +} + +it("a remote cursor becomes outdated if the document advances during its snapshot read", async () => { + const content = new TextEncoder().encode("old"); + let metadata = { parentVersionId: 1, hash: await hash(content) }; + const entered = Promise.withResolvers(), + release = Promise.withResolvers(); + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const tracker = new CursorTracker( + { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata + }) + } as unknown as Database, + websocket, + { + read: async () => { + entered.resolve(); + await release.promise; + return content; + } + } as unknown as FileOperations, + new FileChangeNotifier() + ); + let outdated: boolean | undefined; + tracker.onRemoteCursorsUpdated.add((cursors) => { + outdated = cursors[0]?.isOutdated; + }); + const update = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 1 }] + } + ] + } + ]); + await entered.promise; + metadata = { + parentVersionId: 2, + hash: await hash(new TextEncoder().encode("new content")) + }; + release.resolve(); + await update; + assert.equal(outdated, true); + tracker.reset(); +}); diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index bdd7d9b7..a6170195 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -23,6 +23,7 @@ export class CursorTracker { >(); private readonly updateLock = new Lock(); + private sessionGeneration = 0; private knownRemoteCursors: (ClientCursors & { upToDateness: DocumentUpToDateness; @@ -31,6 +32,10 @@ export class CursorTracker { private lastLocalCursorState: DocumentWithCursors[] = []; private lastLocalCursorStateWithoutDirtyDocuments: DocumentWithCursors[] = []; + private localCursors: Record = {}; + private localCursorDocuments: Record = {}; + private cursorTimer?: ReturnType; + private readonly sendLock = new Lock(); public constructor( private readonly database: Database, @@ -38,9 +43,31 @@ export class CursorTracker { private readonly fileOperations: FileOperations, private readonly fileChangeNotifier: FileChangeNotifier ) { + this.webSocketManager.onWebSocketStatusChanged.add((connected) => { + this.sessionGeneration++; + clearInterval(this.cursorTimer); + this.lastLocalCursorState = []; + this.lastLocalCursorStateWithoutDirtyDocuments = []; + if (!connected) { + this.knownRemoteCursors = []; + this.onRemoteCursorsUpdated.trigger([]); + return; + } + const refresh = () => { + void this.publishLocalCursors(true).catch(() => { + // A concurrent move/deletion can invalidate an editor view; + // the next notification or heartbeat retries the snapshot. + }); + }; + refresh(); + this.cursorTimer = setInterval(refresh, 15_000); + this.cursorTimer.unref?.(); + }); this.webSocketManager.onRemoteCursorsUpdateReceived.add( async (clientCursors) => { + const generation = this.sessionGeneration; await this.updateLock.withLock(async () => { + if (generation !== this.sessionGeneration) return; // The latest message will contain all active clients, so we can delete the ones // from the local list which are no longer active. const allIds = new Set( @@ -51,11 +78,7 @@ export class CursorTracker { allIds.has(c.deviceId) ); - for (const cursor of clientCursors.filter((client) => - client.documentsWithCursors.every( - (doc) => doc.vault_update_id != null - ) - )) { + for (const cursor of clientCursors) { updatedKnownRemoteCursors.push({ ...cursor, upToDateness: @@ -63,30 +86,42 @@ export class CursorTracker { }); } + if (generation !== this.sessionGeneration) return; this.knownRemoteCursors = updatedKnownRemoteCursors; + this.onRemoteCursorsUpdated.trigger( + this.getRelevantAndPruneKnownClientCursors() + ); }); - - this.onRemoteCursorsUpdated.trigger( - this.getRelevantAndPruneKnownClientCursors() - ); } ); - this.fileChangeNotifier.onFileChanged.add(async (relativePath) => - this.updateLock.withLock(async () => { + this.fileChangeNotifier.onFileChanged.add(async (relativePath) => { + const generation = this.sessionGeneration; + await this.publishLocalCursors(true); + const cursors = await this.updateLock.withLock(async () => { + if (generation !== this.sessionGeneration) return []; for (const clientCursor of this.knownRemoteCursors) { if ( clientCursor.documentsWithCursors.some( (document) => - document.relative_path === relativePath + document.relative_path === relativePath || + this.database.getDocumentByDocumentId( + document.document_id + )?.relativePath === relativePath ) ) { clientCursor.upToDateness = await this.getDocumentsUpToDateness(clientCursor); } } - }) - ); + return generation === this.sessionGeneration + ? this.getRelevantAndPruneKnownClientCursors() + : []; + }); + + if (generation === this.sessionGeneration) + this.onRemoteCursorsUpdated.trigger(cursors); + }); } /// Update the local cursors for the given documents. @@ -95,98 +130,149 @@ export class CursorTracker { public async sendLocalCursorsToServer( documentToCursors: Record ): Promise { - const documentsWithCursors: DocumentWithCursors[] = []; + this.localCursors = structuredClone(documentToCursors); + this.localCursorDocuments = {}; + await this.publishLocalCursors(false); + } - for (const [relativePath, cursors] of Object.entries( - documentToCursors - )) { - const record = - this.database.getLatestDocumentByRelativePath(relativePath); + private async publishLocalCursors(force: boolean): Promise { + const generation = this.sessionGeneration; + await this.sendLock.withLock(async () => { + if (generation !== this.sessionGeneration) return; + const documentsWithCursors: DocumentWithCursors[] = []; - if (!record) { - continue; // Let's wait for the file to be created before sending cursors + for (const [editorPath, cursors] of Object.entries( + this.localCursors + )) { + const id = this.localCursorDocuments[editorPath]; + const record = id + ? this.database.getDocumentByDocumentId(id) + : this.database.getLatestDocumentByRelativePath(editorPath); + + if (!record) { + continue; // Let's wait for the file to be created before sending cursors + } + this.localCursorDocuments[editorPath] = record.documentId; + const { relativePath } = record; + + if (!record.metadata) { + continue; // this is a new document, no need to sync the cursors + } + + documentsWithCursors.push({ + relative_path: relativePath, + document_id: record.documentId, + vault_update_id: record.metadata.parentVersionId, + cursors: cursors.map(({ start, end }) => ({ + start: Math.min(start, end), + end: Math.max(start, end) + })) // the client might send directional selections + }); } - if (!record.metadata) { - continue; // this is a new document, no need to sync the cursors + if ( + !force && + JSON.stringify(this.lastLocalCursorState) === + JSON.stringify(documentsWithCursors) + ) { + // Caching step to avoid reading the edited files all the time + return; + } + this.lastLocalCursorState = structuredClone(documentsWithCursors); + + for (const doc of documentsWithCursors) { + let readContent: Uint8Array; + try { + readContent = await this.fileOperations.read( + doc.relative_path + ); + } catch { + doc.vault_update_id = null; + continue; + } + const digest = await hash(readContent); + const record = this.database.getLatestDocumentByRelativePath( + doc.relative_path + ); + if ( + record?.documentId !== doc.document_id || + record.relativePath !== doc.relative_path || + record.metadata?.parentVersionId !== doc.vault_update_id || + record.metadata.hash !== digest + ) { + doc.vault_update_id = null; + } } - documentsWithCursors.push({ - relative_path: relativePath, - document_id: record.documentId, - vault_update_id: record.metadata.parentVersionId, - cursors: cursors.map(({ start, end }) => ({ - start: Math.min(start, end), - end: Math.max(start, end) - })) // the client might send directional selections - }); - } - - if ( - JSON.stringify(this.lastLocalCursorState) === - JSON.stringify(documentsWithCursors) - ) { - // Caching step to avoid reading the edited files all the time - return; - } - this.lastLocalCursorState = documentsWithCursors; - - for (const doc of documentsWithCursors) { - const readContent = await this.fileOperations.read( - doc.relative_path - ); - const record = this.database.getLatestDocumentByRelativePath( - doc.relative_path - ); - if (record?.metadata?.hash !== hash(readContent)) { - doc.vault_update_id = null; + if (generation !== this.sessionGeneration) return; + if ( + !force && + JSON.stringify( + this.lastLocalCursorStateWithoutDirtyDocuments + ) === JSON.stringify(documentsWithCursors) + ) { + return; } - } - if ( - JSON.stringify(this.lastLocalCursorStateWithoutDirtyDocuments) === - JSON.stringify(documentsWithCursors) - ) { - return; - } + this.lastLocalCursorStateWithoutDirtyDocuments = + documentsWithCursors; - this.lastLocalCursorStateWithoutDirtyDocuments = documentsWithCursors; - - this.webSocketManager.updateLocalCursors({ documentsWithCursors }); + this.webSocketManager.updateLocalCursors({ documentsWithCursors }); + }); } public reset(): void { + this.sessionGeneration++; + clearInterval(this.cursorTimer); this.knownRemoteCursors = []; + this.onRemoteCursorsUpdated.trigger([]); this.lastLocalCursorState = []; this.lastLocalCursorStateWithoutDirtyDocuments = []; - this.updateLock.reset(); + // Keep serialization intact while old reads drain. A generation fence + // discards both active and queued work from the retired connection. } private getRelevantAndPruneKnownClientCursors(): MaybeOutdatedClientCursors[] { const result: MaybeOutdatedClientCursors[] = []; const included = new Set(); - const relevantCursors = []; + const retainedCursors = []; + const retainedFuture = new Set(); for (const clientCursors of [...this.knownRemoteCursors].reverse()) { - if (included.has(clientCursors.deviceId)) { + if (clientCursors.upToDateness === DocumentUpToDateness.Later) { + // Retain the latest future position so it can become relevant + // after the corresponding content or manifest is applied. + if (!retainedFuture.has(clientCursors.deviceId)) { + retainedCursors.unshift(clientCursors); + retainedFuture.add(clientCursors.deviceId); + } continue; } - if (clientCursors.upToDateness === DocumentUpToDateness.Later) { + if (included.has(clientCursors.deviceId)) { continue; } result.push({ ...clientCursors, + documentsWithCursors: clientCursors.documentsWithCursors.map( + (document) => ({ + ...document, + relative_path: + this.database.getDocumentByDocumentId( + document.document_id + )?.relativePath ?? document.relative_path + }) + ), isOutdated: clientCursors.upToDateness === DocumentUpToDateness.Prior }); included.add(clientCursors.deviceId); - relevantCursors.unshift(clientCursors); // to reverse order back to normal + retainedCursors.unshift(clientCursors); // to reverse order back to normal } - this.knownRemoteCursors = relevantCursors; + this.knownRemoteCursors = retainedCursors; return result; } @@ -223,8 +309,10 @@ export class CursorTracker { private async getDocumentUpToDateness( document: DocumentWithCursors ): Promise { - const record = this.database.getLatestDocumentByRelativePath( - document.relative_path + if (document.vault_update_id === null) + return DocumentUpToDateness.Prior; + const record = this.database.getDocumentByDocumentId( + document.document_id ); if (!record) { @@ -233,25 +321,33 @@ export class CursorTracker { } if ( - (record.metadata?.parentVersionId ?? 0) < - (document.vault_update_id ?? 0) + (record.metadata?.parentVersionId ?? 0) < document.vault_update_id ) { return DocumentUpToDateness.Later; } else if ( - (document.vault_update_id ?? 0) < - (record.metadata?.parentVersionId ?? 0) + document.vault_update_id < (record.metadata?.parentVersionId ?? 0) ) { // the document of the cursor must be from the past return DocumentUpToDateness.Prior; } - const currentContent = await this.fileOperations.read( - document.relative_path - ); + let currentContent: Uint8Array; + try { + currentContent = await this.fileOperations.read( + record.relativePath + ); + } catch { + return DocumentUpToDateness.Prior; + } - return this.database.getLatestDocumentByRelativePath( - document.relative_path - )?.metadata?.hash === hash(currentContent) + const contentHash = await hash(currentContent); + const current = this.database.getDocumentByDocumentId( + document.document_id + ); + return current?.metadata?.parentVersionId === + document.vault_update_id && + current.relativePath === record.relativePath && + current.metadata.hash === contentHash ? DocumentUpToDateness.UpToDate : DocumentUpToDateness.Prior; } diff --git a/frontend/sync-client/src/sync-operations/file-manifest.ts b/frontend/sync-client/src/sync-operations/file-manifest.ts new file mode 100644 index 00000000..ad2d393a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/file-manifest.ts @@ -0,0 +1,69 @@ +import type { FileManifestEntries } from "../types/file-manifest-entries"; +import { + allocatePortablePath, + validatePortablePaths +} from "../utils/portable-path"; + +export function validateFileManifest(entries: FileManifestEntries): void { + validatePortablePaths(Object.values(entries)); +} + +export function sameFileManifest( + a: FileManifestEntries, + b: FileManifestEntries +): boolean { + return ( + Object.keys(a).length === Object.keys(b).length && + Object.keys(a).every((id) => a[id] === b[id]) + ); +} + +export function resolvePaths( + proposed: FileManifestEntries, + remote: FileManifestEntries, + protectedIds: readonly string[] = [], + reservedPaths: readonly string[] = [] +): FileManifestEntries { + const result: FileManifestEntries = {}; + for (const id of protectedIds) + if (proposed[id] !== undefined) result[id] = proposed[id]; + const occupied = { ...result }; + for (const [index, path] of reservedPaths.entries()) + if (!Object.values(result).includes(path)) + occupied[`reserved-${index}`] = path; + const ids = Object.keys(proposed).sort( + (a, b) => + Number(remote[b] === proposed[b]) - + Number(remote[a] === proposed[a]) || + (a < b ? -1 : a > b ? 1 : 0) + ); + for (const id of ids.filter( + (candidate) => result[candidate] === undefined + )) { + result[id] = allocatePortablePath(proposed[id], id, occupied); + occupied[id] = result[id]; + } + return result; +} + +export function mergeFileManifests( + base: FileManifestEntries, + local: FileManifestEntries, + remote: FileManifestEntries +): FileManifestEntries { + const proposed: FileManifestEntries = {}; + for (const id of new Set([ + ...Object.keys(base), + ...Object.keys(local), + ...Object.keys(remote) + ])) { + const path = + local[id] === base[id] + ? remote[id] + : remote[id] === base[id] + ? local[id] + : remote[id]; + if (path !== undefined) proposed[id] = path; + } + return resolvePaths(proposed, remote); +} diff --git a/frontend/sync-client/src/sync-operations/local-changes.ts b/frontend/sync-client/src/sync-operations/local-changes.ts new file mode 100644 index 00000000..209c6853 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/local-changes.ts @@ -0,0 +1,80 @@ +import { v4 as uuid } from "uuid"; +import type { EngineState } from "../persistence/database"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import { arePathAliases } from "../utils/portable-path"; + +export type LocalChange = { + changeId?: string; + /** Identities at notification time, before a remote application can reuse paths. */ + identities?: Record; +} & ( + | { type: "move"; oldPath: string; relativePath: string } + | { type: "create"; path: string; documentId?: string } + | { type: "delete"; path: string } +); + +export function unappliedChanges( + state: EngineState, + changes: readonly T[] +): T[] { + for (const change of changes) change.changeId ??= uuid(); + const applied = changes.findIndex( + (change) => change.changeId === state.lastAppliedLocalChangeId + ); + return changes.slice(applied + 1); +} + +export function applyLocalChange( + next: EngineState, + change: LocalChange, + ignored: (path: string) => boolean, + initial?: VaultSnapshot +): void { + if (change.type === "create") { + if ( + ignored(change.path) || + Object.values(next.local).includes(change.path) + ) + return; + const id = (change.documentId ??= + (initial && + Object.keys(initial.fileManifest.entries).find( + (id) => + !next.local[id] && + arePathAliases( + initial.fileManifest.entries[id], + change.path + ) + )) || + uuid()); + next.local[id] = change.path; + next.documents[id] ??= { + materialized: true, + bootstrap: initial?.fileManifest.entries[id] !== undefined + }; + return; + } + const from = change.type === "move" ? change.oldPath : change.path; + change.identities ??= Object.fromEntries( + Object.entries(next.local).filter( + ([id, path]) => + next.documents[id]?.materialized && + (path === from || path.startsWith(from + "/")) + ) + ); + for (const [id, path] of Object.entries(change.identities)) { + if (change.type === "delete") { + if (ignored(path)) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = next.fileManifest.entries[id] ?? null; + } + delete next.local[id]; + } else { + const to = change.relativePath + path.slice(from.length); + for (const [other, otherPath] of Object.entries(next.local)) + if (other !== id && otherPath === to) delete next.local[other]; + next.local[id] = to; + } + } +} diff --git a/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts new file mode 100644 index 00000000..5ccfa7a8 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { CursorTracker } from "./cursor-tracker"; +import { FileChangeNotifier } from "./file-change-notifier"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { hash } from "../utils/hash"; +import type { Database, DocumentRecord } from "../persistence/database"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { DocumentWithCursors } from "../services/types/DocumentWithCursors"; + +for (const change of ["version", "identity", "path", "delete"] as const) + test(`outgoing cursor is dirty when ${change} changes during its snapshot read`, async () => { + // Equal bytes deliberately defeat a freshness check that compares only hashes. + const content = new TextEncoder().encode("same bytes"); + const digest = await hash(content); + let current: DocumentRecord | undefined = { + documentId: "a", + relativePath: "a.md", + metadata: { parentVersionId: 1, hash: digest } + }; + const sent: { documentsWithCursors: DocumentWithCursors[] }[] = []; + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: (data: { + documentsWithCursors: DocumentWithCursors[]; + }) => sent.push(data) + }; + const files = { + read: async () => { + if (change === "delete") current = undefined; + else if (change === "version") + current!.metadata!.parentVersionId = 2; + else if (change === "identity") + current!.documentId = "replacement"; + else current!.relativePath = "moved.md"; + return content; + } + }; + const tracker = new CursorTracker( + { + getDocumentByDocumentId: (id: string) => + current?.documentId === id ? current : undefined, + getLatestDocumentByRelativePath: (path: string) => + current?.relativePath === path ? current : undefined + } as unknown as Database, + websocket as unknown as WebSocketManager, + files as unknown as FileOperations, + new FileChangeNotifier() + ); + await tracker.sendLocalCursorsToServer({ + "a.md": [{ start: 4, end: 4 }] + }); + assert.equal(sent.length, 1); + assert.equal(sent[0].documentsWithCursors[0].vault_update_id, null); + tracker.reset(); + }); diff --git a/frontend/sync-client/src/sync-operations/scan.ts b/frontend/sync-client/src/sync-operations/scan.ts new file mode 100644 index 00000000..90aa9fdb --- /dev/null +++ b/frontend/sync-client/src/sync-operations/scan.ts @@ -0,0 +1,152 @@ +import { v4 as uuid } from "uuid"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { EngineState } from "../persistence/database"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import { EMPTY_HASH, hash } from "../utils/hash"; +import { + arePathAliases, + findPathWithSameSpelling +} from "../utils/portable-path"; + +import { + applyLocalChange, + unappliedChanges, + type LocalChange +} from "./local-changes"; +export type { LocalChange } from "./local-changes"; + +interface ScanOptions { + next: EngineState; + guard: () => void; + changes: LocalChange[]; + files: FileOperations; + ignored: (path: string) => boolean; + oversized: (size: number) => boolean; + commit: (next: EngineState) => Promise; +} + +/** Scans are complete or fail. Missing unmaterialized/ignored entries are not deletions. */ +export async function scanLocalFiles( + { + next, + guard, + changes: queuedChanges, + files, + ignored, + oversized, + commit + }: ScanOptions, + initial?: VaultSnapshot +): Promise> { + // Leave the prefix queued until its resulting state is saved. + // A persisted change ID makes replay safe after an uncertain save. + const changes = [...queuedChanges]; + const unsyncablePaths = new Set(); + for (const change of unappliedChanges(next, changes)) { + applyLocalChange(next, change, ignored, initial); + next.lastAppliedLocalChangeId = change.changeId; + } + const paths: string[] = []; + const reportedPaths = await files.listFilesRecursively(); + for (const path of reportedPaths) { + // Some filesystems may report a different Unicode spelling after a + // rename. Keep an already-accessible spelling when possible. + const alias = findPathWithSameSpelling(Object.values(next.local), path); + paths.push( + alias && + !reportedPaths.includes(alias) && + (await files.fs.exists(alias)) + ? alias + : path + ); + } + const present = new Set(paths); + const snapshots = new Map(); + for (const path of paths) { + if (ignored(path)) { + unsyncablePaths.add(path); + continue; + } + const info = await files.fs.stat(path); + if (!info || info.kind !== "file") + throw new Error(`File changed during scan: ${path}`); + if (oversized(info.size)) { + unsyncablePaths.add(path); + continue; + } + const snapshot = await files.fs.readSnapshot(path); + if (!snapshot) throw new Error(`File changed during scan: ${path}`); + if (!oversized(snapshot.content.length)) { + snapshots.set(path, { hash: await hash(snapshot.content) }); + } else { + unsyncablePaths.add(path); + } + } + const missing = new Set( + Object.keys(next.local).filter( + (id) => + next.documents[id]?.materialized && + !ignored(next.local[id]) && + !present.has(next.local[id]) + ) + ); + const knownPaths = new Set(Object.values(next.local)); + const unknown = [...snapshots.keys()].filter( + (path) => !knownPaths.has(path) + ); + for (const path of unknown) { + const snapshot = snapshots.get(path)!; + const matches = [...missing].filter( + (id) => next.documents[id]?.observedHash === snapshot.hash + ); + const sameHash = unknown.filter( + (other) => snapshots.get(other)?.hash === snapshot.hash + ); + let id: string | undefined; + if ( + snapshot.hash !== EMPTY_HASH && + matches.length === 1 && + sameHash.length === 1 + ) { + id = matches[0]; + missing.delete(id); + } else if (initial) { + id = Object.keys(initial.fileManifest.entries).find( + (candidate) => + !next.local[candidate] && + arePathAliases( + initial.fileManifest.entries[candidate], + path + ) + ); + } + id ??= uuid(); + next.local[id] = path; + next.documents[id] ??= { + materialized: true, + bootstrap: initial?.fileManifest.entries[id] !== undefined + }; + } + for (const id of missing) delete next.local[id]; + for (const [id, path] of Object.entries(next.local)) { + if (unsyncablePaths.has(path) || ignored(path)) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = next.fileManifest.entries[id] ?? null; + } + const snapshot = snapshots.get(path); + if (snapshot) { + next.documents[id] ??= { materialized: true }; + if (!next.documents[id].materialized && !next.documents[id].base) + next.documents[id].bootstrap = true; + next.documents[id].materialized = true; + next.documents[id].observedHash = snapshot.hash; + } + } + next.protectedPaths = [...unsyncablePaths]; + guard(); + await commit(next); + queuedChanges.splice(0, changes.length); + guard(); + return unsyncablePaths; +} diff --git a/frontend/sync-client/src/sync-operations/syncer.test.ts b/frontend/sync-client/src/sync-operations/syncer.test.ts new file mode 100644 index 00000000..524fbabb --- /dev/null +++ b/frontend/sync-client/src/sync-operations/syncer.test.ts @@ -0,0 +1,327 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import assert from "node:assert"; +import { afterEach, beforeEach, describe, it, mock } from "node:test"; +import type { Database } from "../persistence/database"; +import { + DEFAULT_SETTINGS, + Settings, + type SyncSettings +} from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { SyncService } from "../services/sync-service"; +import type { ServerConfig } from "../services/server-config"; +import type { WebSocketManager } from "../services/websocket-manager"; +import { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { Syncer } from "./syncer"; +import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { toStoredSnapshot } from "./content"; +import { undiff } from "reconcile-text"; + +function createSyncer(overrides: Partial = {}): { + syncer: Syncer; + runCount: () => number; +} { + const settings = new Settings( + new Logger(), + overrides, + async () => undefined + ); + const database = { + state: {} + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners<() => unknown>() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + {} as Logger, + database, + settings, + {} as SyncService, + websocket, + {} as FileOperations, + {} as ServerConfig, + {} as SyncHistory, + new FixedSizeDocumentCache(1024) + ); + let runs = 0; + const internals = syncer as unknown as { + dirty: boolean; + run: () => Promise; + }; + internals.run = async (): Promise => { + runs++; + internals.dirty = false; + }; + + return { syncer, runCount: () => runs }; +} + +describe("Syncer interval", () => { + beforeEach(() => { + mock.timers.enable({ apis: ["setTimeout"] }); + }); + + afterEach(() => { + mock.timers.reset(); + }); + + it("does not schedule a timer when syncIntervalMs is unset", async () => { + const { syncer, runCount } = createSyncer({ + syncIntervalMs: undefined + }); + + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("keeps periodic reconciliation opt-in by default", async () => { + assert.strictEqual(DEFAULT_SETTINGS.syncIntervalMs, undefined); + const { syncer, runCount } = createSyncer(); + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("does not poll when explicitly disabled with zero", async () => { + const { syncer, runCount } = createSyncer({ syncIntervalMs: 0 }); + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("wakes after the configured sync interval", async () => { + const { syncer, runCount } = createSyncer({ syncIntervalMs: 25 }); + + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(24); + assert.strictEqual(runCount(), 1); + + mock.timers.tick(1); + assert.strictEqual(runCount(), 2); + await syncer.waitUntilFinished(); + await syncer.stop(); + }); +}); + +describe("Syncer per-file state and event delivery", () => { + const head = { + vaultUpdateId: 1, + documentId: "document", + updatedDate: "2026-01-01T00:00:00Z", + userId: "user", + deviceId: "device", + contentSize: 4 + }; + + function fixture() { + const database = { + state: { + initialized: true, + local: { document: "note.md" }, + documents: { + document: { + materialized: true, + observedHash: "hash", + base: { ...head, hash: "hash" } + } + }, + fileManifest: { + fileManifestId: 1, + entries: { document: "note.md" } + }, + remoteHeads: { document: head }, + lastSeenUpdateId: 0 + }, + getLatestDocumentByRelativePath: (path: string) => + path === "note.md" + ? { + documentId: "document", + relativePath: path + } + : undefined + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + () => Promise + >() + } as unknown as WebSocketManager; + const service = { + events: async () => { + return { headEventId: 0, events: [] }; + } + } as unknown as SyncService; + const syncer = new Syncer( + "device", + {} as Logger, + database, + new Settings( + new Logger(), + { syncIntervalMs: 0 }, + async () => undefined + ), + service, + websocket, + {} as FileOperations, + {} as ServerConfig, + {} as SyncHistory, + new FixedSizeDocumentCache(1024) + ); + (syncer as unknown as { hasScanned: boolean }).hasScanned = true; + return { + database, + websocket, + syncer + }; + } + + it("reports only a genuinely current path as up to date", () => { + const { syncer } = fixture(); + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true); + assert.strictEqual(syncer.isDocumentUpToDate("missing.md"), false); + + const internals = syncer as unknown as { + dirty: boolean; + unsyncablePaths: Set; + }; + internals.dirty = true; + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true); + internals.unsyncablePaths.add("note.md"); + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), false); + }); + + it("projects content responses to metadata before persistence", () => { + const { syncer } = fixture(); + const projected = ( + syncer as unknown as { + documentHead: (value: typeof head) => Record; + } + ).documentHead({ ...head, contentBase64: "secret" } as typeof head); + assert.deepEqual(projected, head); + assert.strictEqual("contentBase64" in projected, false); + }); +}); + +describe("Syncer diff uploads", () => { + const encoder = new TextEncoder(); + + function fixture(cacheSize = 1024): { + syncer: Syncer; + parent: Uint8Array; + } { + const cache = new FixedSizeDocumentCache(cacheSize); + const parent = encoder.encode("one two three\n"); + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners<() => unknown>() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + {} as Logger, + { state: {} } as Database, + new Settings( + new Logger(), + { syncIntervalMs: 0 }, + async () => undefined + ), + { + getDocumentVersionContent: async () => parent + } as unknown as SyncService, + websocket, + {} as FileOperations, + { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig, + {} as SyncHistory, + cache + ); + return { syncer, parent }; + } + + it("sends a diff when the fetched parent is still cached", async () => { + const { syncer, parent } = fixture(); + const internals = syncer as unknown as { + remoteSnapshot: ( + head: { + vaultUpdateId: number; + documentId: string; + }, + path: string + ) => Promise; + pushContent: ( + path: string, + parentVersionId: number, + snapshot: Awaited> + ) => Promise< + | { type: "Diff"; value: (number | string)[] } + | { type: "Snapshot"; value: string } + >; + }; + await internals.remoteSnapshot( + { vaultUpdateId: 7, documentId: "document" }, + "note.md" + ); + const changed = "one changed three\n"; + const payload = await internals.pushContent( + "note.md", + 7, + await toStoredSnapshot({ content: encoder.encode(changed) }) + ); + + assert.equal(payload.type, "Diff"); + if (payload.type === "Diff") { + assert.equal( + undiff(new TextDecoder().decode(parent), payload.value), + changed + ); + } + }); + + it("falls back to a snapshot when the parent cannot fit in the cache", async () => { + const { syncer } = fixture(0); + const internals = syncer as unknown as { + remoteSnapshot: ( + head: { + vaultUpdateId: number; + documentId: string; + }, + path: string + ) => Promise; + pushContent: ( + path: string, + parentVersionId: number, + snapshot: Awaited> + ) => Promise<{ type: string; value: unknown }>; + }; + await internals.remoteSnapshot( + { vaultUpdateId: 7, documentId: "document" }, + "note.md" + ); + const payload = await internals.pushContent( + "note.md", + 7, + await toStoredSnapshot({ content: encoder.encode("changed") }) + ); + + assert.equal(payload.type, "Snapshot"); + }); +}); diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 71dedd85..47ba91d0 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,522 +1,1269 @@ +import type { SyncHistory } from "../tracing/sync-history"; +import { SyncStatus, SyncType } from "../tracing/sync-history"; +import { v4 as uuid } from "uuid"; +import { base64ToBytes } from "byte-base64"; +import { diff } from "reconcile-text"; import type { Database, - DocumentId, - DocumentRecord, - RelativePath + EngineState, + StoredSnapshot } from "../persistence/database"; -import type { SyncService } from "../services/sync-service"; -import type { Logger } from "../tracing/logger"; -import PQueue from "p-queue"; -import { hash } from "../utils/hash"; -import { v4 as uuidv4 } from "uuid"; +import { emptyState } from "../persistence/database"; +import type { + FileOperations, + FileWrite +} from "../file-operations/file-operations"; import type { Settings } from "../persistence/settings"; -import type { FileOperations } from "../file-operations/file-operations"; -import { findMatchingFile } from "../utils/find-matching-file"; -import type { UnrestrictedSyncer } from "./unrestricted-syncer"; -import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "../services/sync-reset-error"; -import { Locks } from "../utils/data-structures/locks"; -import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate"; +import type { SyncService } from "../services/sync-service"; +import type { ServerConfig } from "../services/server-config"; import type { WebSocketManager } from "../services/websocket-manager"; -import type { WebSocketClientMessage } from "../services/types/WebSocketClientMessage"; -import { awaitAll } from "../utils/await-all"; +import type { DocumentVersion } from "../services/types/DocumentVersion"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import type { EventBatch } from "../services/types/EventBatch"; +import type { EventRecord } from "../services/types/EventRecord"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import type { PushContent } from "../services/types/PushContent"; +import type { Logger } from "../tracing/logger"; import { EventListeners } from "../utils/data-structures/event-listeners"; +import { mergeContent, toStoredSnapshot } from "./content"; +import { scanLocalFiles, type LocalChange } from "./scan"; +import { + mergeFileManifests, + resolvePaths, + sameFileManifest, + validateFileManifest +} from "./file-manifest"; +import { removeFromArray } from "../utils/remove-from-array"; +import { isInternalPath } from "../utils/portable-path"; +import { isBinary } from "../utils/is-binary"; +import { isFileTypeMergable } from "../utils/is-file-type-mergable"; +import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { + AuthenticationError, + LocalChangesDuringReconciliation, + PermanentSyncError, + ServerVersionMismatchError, + SyncResetError, + ServerHistoryChangedError +} from "../errors/errors"; export class Syncer { + public readonly onReadyForEvents = new EventListeners<() => void>(); + public readonly onServerHistoryChanged = new EventListeners<() => void>(); public readonly onRemainingOperationsCountChanged = new EventListeners< - (remainingOperations: number) => unknown + (count: number) => unknown >(); - private readonly remoteDocumentsLock: Locks; - - // FIFO to limit the number of concurrent sync operations - private readonly syncQueue: PQueue; - - private _isFirstSyncComplete = false; - private runningScheduleSyncForOfflineChanges: Promise | undefined; - private previousRemainingOperationsCount = 0; + private running?: Promise; + private timer?: ReturnType; + private stopped = true; + private dirty = false; + private localGeneration = 0; + private failure?: unknown; + private readonly changes: LocalChange[]; + private notificationSaving: Promise = Promise.resolve(); + private notificationSaveFailed = false; + private readonly pendingPaths = new Set(); + private unsyncablePaths = new Set(); + private hasScanned = false; public constructor( - private readonly deviceId: string, + deviceId: string, private readonly logger: Logger, private readonly database: Database, private readonly settings: Settings, - private readonly syncService: SyncService, - private readonly webSocketManager: WebSocketManager, - private readonly operations: FileOperations, - private readonly internalSyncer: UnrestrictedSyncer + private readonly service: SyncService, + websocket: WebSocketManager, + private readonly files: FileOperations, + private readonly serverConfig: ServerConfig, + private readonly history: SyncHistory, + private readonly contentCache: FixedSizeDocumentCache, + private readonly localChangeStorage: { + entries: LocalChange[]; + save: (entries: LocalChange[]) => Promise; + } = { entries: [], save: async () => {} } ) { - this.syncQueue = new PQueue({ - concurrency: settings.getSettings().syncConcurrency - }); - - this.remoteDocumentsLock = new Locks(this.logger); - - settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) { - this.syncQueue.concurrency = newSettings.syncConcurrency; + this.changes = localChangeStorage.entries; + websocket.onWebSocketStatusChanged.add((connected) => { + if (connected) { + websocket.sendHandshakeMessage({ + type: "handshake", + token: settings.getSettings().token, + deviceId + }); } + // Failed upgrades and lost transports still need HTTP catchup, + // including when the optional polling interval is disabled. + this.wake(); }); - - this.syncQueue.on("active", () => { - if (this.previousRemainingOperationsCount !== this.syncQueue.size) { - this.previousRemainingOperationsCount = this.syncQueue.size; - this.onRemainingOperationsCountChanged.trigger( - this.syncQueue.size - ); - } + websocket.onRemoteVaultUpdateReceived.add(async () => { + this.wake(); }); - - this.webSocketManager.onWebSocketStatusChanged.add((isConnected) => { - if (isConnected) { - // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message - this.sendHandshakeMessage(); - } - }); - this.webSocketManager.onRemoteVaultUpdateReceived.add( - this.syncRemotelyUpdatedFile.bind(this) - ); } public get isFirstSyncComplete(): boolean { - return this._isFirstSyncComplete; + return this.database.state.initialized; } - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { + public get hasLocalChanges(): boolean { + return this.changes.length > 0; + } + + public get isBusy(): boolean { + return ( + !!this.running || + this.dirty || + !!this.database.state.pending || + !!this.failure || + !!this.rejectionMessage + ); + } + + public isDocumentUpToDate(path: string): boolean { if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === false + !this.database.state.initialized || + !this.hasScanned || + this.pendingPaths.has(path) || + this.unsyncablePaths.has(path) || + this.settings.isIgnored(path) ) { - this.logger.debug( - `Document ${relativePath} already exists in the database, skipping` - ); + return false; + } + + const record = this.database.getLatestDocumentByRelativePath(path); + if (!record) return false; + const document = this.database.state.documents[record.documentId]; + if ( + !document?.materialized || + !document.base || + document.bootstrap || + document.observedHash !== document.base.hash || + this.database.state.fileManifest.entries[record.documentId] !== path + ) { + return false; + } + + const remote = this.database.state.remoteHeads[record.documentId]; + if (remote && remote.vaultUpdateId > document.base.vaultUpdateId) + return false; + + const { pending } = this.database.state; + if ( + pending?.type === "content" && + pending.documentId === record.documentId + ) { + return false; + } + if ( + pending?.type === "fileManifest" && + pending.request.entries[record.documentId] !== + this.database.state.fileManifest.entries[record.documentId] + ) { + return false; + } + + return true; + } + + public start(): void { + this.stopped = false; + this.failure = undefined; + this.hasScanned = false; + this.wake(); + } + + public async stop(): Promise { + this.stopped = true; + clearTimeout(this.timer); + await this.running; + } + + public wake(): void { + if (this.stopped) { return; } - const [promise, resolve, reject] = createPromise(); + this.dirty = true; - const id = uuidv4(); - const document = this.database.createNewPendingDocument( - id, - relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - } - - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { - // This is must be a consequence of us deleting a file because of a remote update - // which triggered a local delete, so we don't need to do anything here. - this.logger.debug( - `Document ${relativePath} has already been markes as deleted, skipping` - ); + if (this.running) { return; } - // We have to have a record of the delete in case there's an in-flight update for the same - // document which finishes after the delete has succeeded and would introduce a phantom metadata record. - this.database.delete(relativePath); + clearTimeout(this.timer); - const [promise, resolve, reject] = createPromise(); - - const document = await this.database.getResolvedDocumentByRelativePath( - relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document) + this.running = this.run().finally(() => { + this.running = undefined; + this.onRemainingOperationsCountChanged.trigger( + this.failure || + this.database.state.pending || + this.rejectionMessage + ? 1 + : 0 ); - resolve(); + if (this.stopped) { + return; + } - this.database.removeDocument(document); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); + if (this.dirty && !this.failure) { + this.wake(); + return; + } + + if ( + this.failure instanceof PermanentSyncError || + this.failure instanceof AuthenticationError || + this.failure instanceof ServerVersionMismatchError + ) { + return; + } + + const intervalMs = + this.failure === undefined + ? this.settings.getSettings().syncIntervalMs + : this.settings.getSettings().networkRetryIntervalMs; + if (intervalMs === undefined || intervalMs === 0) { + return; + } + + this.timer = setTimeout(() => { + this.wake(); + }, intervalMs); + }); + } + + public async waitUntilFinished(): Promise { + while (this.running) { + await this.running; } + if (this.failure) { + throw this.failure; + } + if (this.rejectionMessage) + throw new PermanentSyncError(this.rejectionMessage); + } + + public async syncLocallyCreatedFile(_path: string): Promise { + this.localGeneration++; + this.pendingPaths.add(_path); + await this.recordLocalChange({ type: "create", path: _path }); + } + + public async syncLocallyDeletedFile(_path: string): Promise { + this.localGeneration++; + this.pendingPaths.add(_path); + await this.recordLocalChange({ type: "delete", path: _path }); } public async syncLocallyUpdatedFile({ oldPath, relativePath }: { - oldPath?: RelativePath; - relativePath: RelativePath; + oldPath?: string; + relativePath: string; }): Promise { - if (oldPath !== undefined) { - // We might have moved the document in the database before calling this method, - // in that case, we mustn't move it again. - if ( - this.database.getLatestDocumentByRelativePath(relativePath) === - undefined || - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { - if (oldPath === relativePath) { - throw new Error( - `Old path and new path are the same: ${oldPath}` - ); - } - - this.database.move(oldPath, relativePath); - } - } - - let document = - this.database.getLatestDocumentByRelativePath(relativePath); - + // Content snapshots are re-read before writing. Only namespace + // notifications invalidate an identity plan; keystrokes must not starve + // a complete scan of unrelated documents. + if (oldPath !== undefined && oldPath !== "") this.localGeneration++; + this.pendingPaths.add(relativePath); + if (oldPath) this.pendingPaths.add(oldPath); if ( - oldPath !== undefined && - document?.metadata?.remoteRelativePath === relativePath + oldPath && + !isInternalPath(oldPath) && + !isInternalPath(relativePath) ) { - this.logger.debug( - `Document ${relativePath} has been moved as a result of a remote update, skipping sync` - ); - return; + await this.recordLocalChange({ + type: "move", + oldPath, + relativePath + }); } + this.wake(); + } - if (document === undefined) { - this.logger.debug( - `Cannot find document ${relativePath} in the database, skipping` - ); - return; - } - - if (document.isDeleted) { - this.logger.debug( - `Document ${relativePath} has been deleted locally, skipping` - ); - return; - } - - const [promise, resolve, reject] = createPromise(); - - document = await this.database.getResolvedDocumentByRelativePath( - relativePath, - promise - ); + /** Reload an uncertain metadata save before starting another attempt. */ + public async reloadLocalState(): Promise { + await this.flushLocalChanges(); + await this.database.recoverPersistence(); + } + private async recordLocalChange(change: LocalChange): Promise { + change.changeId = uuid(); + const previous = [...this.changes]; + this.changes.push(change); + const saving = this.notificationSaving.then(async () => { + await this.files.captureChange(change, previous); + await this.localChangeStorage.save(structuredClone(this.changes)); + this.notificationSaveFailed = false; + }); + this.notificationSaving = saving.catch(() => { + this.notificationSaveFailed = true; + }); try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ - oldPath, - document - }) - ); - - resolve(); - } catch (e) { - reject(e); + await saving; } finally { - this.database.removeDocumentPromise(promise); + this.wake(); } } - public async scheduleSyncForOfflineChanges(): Promise { - if (this.runningScheduleSyncForOfflineChanges !== undefined) { - this.logger.debug("Uploading local changes is already in progress"); - return this.runningScheduleSyncForOfflineChanges; + public async flushLocalChanges(): Promise { + await this.notificationSaving; + if (this.notificationSaveFailed) { + await this.localChangeStorage.save(structuredClone(this.changes)); + this.notificationSaveFailed = false; } + } + private async run(): Promise { + this.onRemainingOperationsCountChanged.trigger(1); + this.failure = undefined; try { - this.runningScheduleSyncForOfflineChanges = - this.internalScheduleSyncForOfflineChanges(); - await this.runningScheduleSyncForOfflineChanges; - this.logger.info(`All local changes have been applied remotely`); - } catch (e) { - if (e instanceof SyncResetError) { - this.logger.info( - "Failed to apply local changes remotely due to a reset" + await this.reloadLocalState(); + if ( + this.database.state.historyRecovery || + (this.database.state.initialized && + this.service.verifiesHistory && + !this.service.hasHistoryCheckpoint) + ) + await this.recoverServerHistory(); + await this.serverConfig.initialize(); + if (!this.database.state.initialized) { + await this.bootstrap(); + } + if (!this.stopped) this.onReadyForEvents.trigger(); + do { + this.dirty = false; + // An unknown accepted request must be resolved before observing a + // newer base, otherwise we can merge our own edit a second time. + if (this.database.state.pending) { + await this.finishPending(); + } + const batch = await this.service.events( + this.database.state.lastSeenUpdateId ); + await this.incorporateEventBatch(batch); + // Previously excluded content becomes eligible after a setting + // change. Its saved obligation survives advancing the cursor. + if (Object.keys(this.database.state.excluded ?? {}).length) + await this.incorporateFileManifest( + this.database.state.fileManifest + ); + else await this.scan(); + for (const head of Object.values( + this.database.state.remoteHeads + )) { + const doc = this.database.state.documents[head.documentId]; + if ( + this.database.state.local[head.documentId] !== + undefined && + (!doc?.materialized || + !doc.base || + doc.bootstrap || + doc.base.vaultUpdateId < head.vaultUpdateId) + ) { + await this.incorporateContent(head); + } + } + if (await this.preparePush()) { + await this.finishPending(); + this.dirty = true; + } + } while (this.dirty && !this.stopped); + if (!this.dirty && !this.database.state.pending) + this.pendingPaths.clear(); + } catch (error) { + if (error instanceof ServerHistoryChangedError && !this.stopped) { + try { + await this.recoverServerHistory(); + this.dirty = true; + return; + } catch (recoveryError) { + error = recoveryError; + } + } + if (error instanceof LocalChangesDuringReconciliation) { + this.dirty = true; return; } - this.logger.error( - `Not all local changes have been applied remotely: ${e}` - ); - throw e; - } finally { - this.runningScheduleSyncForOfflineChanges = undefined; - } - } - - public async waitUntilFinished(): Promise { - await this.runningScheduleSyncForOfflineChanges; - await this.syncQueue.onIdle(); // Wait for queue to be empty and running tasks to finish - } - - public async syncRemotelyUpdatedFile( - message: WebSocketVaultUpdate - ): Promise { - try { - const handlerPromise = awaitAll( - message.documents.map(async (document) => - this.internalSyncRemotelyUpdatedFile(document) - ) - ); - - await handlerPromise; - - if (message.isInitialSync && message.documents.length > 0) { - this.database.setLastSeenUpdateId( - message.documents - .map((document) => document.vaultUpdateId) - .reduce((a, b) => Math.max(a, b)) - ); - } - - this._isFirstSyncComplete = true; - } catch (e) { - this.logger.error(`Failed to sync remotely updated file: ${e}`); - } - } - - public reset(): void { - this._isFirstSyncComplete = false; - this.syncQueue.clear(); - this.remoteDocumentsLock.reset(); - this.runningScheduleSyncForOfflineChanges = undefined; - } - - private sendHandshakeMessage(): void { - const message: WebSocketClientMessage = { - type: "handshake", - deviceId: this.deviceId, - token: this.settings.getSettings().token, - lastSeenVaultUpdateId: this.database.getLastSeenUpdateId() - }; - this.webSocketManager.sendHandshakeMessage(message); - } - - private async internalSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent - ): Promise { - let document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - if (document === undefined) { - // Let's avoid the same documents getting created in parallel multiple times. - // There might be multiple tasks waiting for the lock - return this.remoteDocumentsLock.withLock( - remoteVersion.documentId, - async () => { - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile` - if (document === undefined) { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion - ) - ); - } else { - const [promise, resolve, reject] = createPromise(); - - document = - await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - } - - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); - } - ); - } - - // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile` - const [promise, resolve, reject] = createPromise(); - - document = await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); - } - - private async internalScheduleSyncForOfflineChanges(): Promise { - await this.createFakeDocumentsFromRemoteState(); - - const allLocalFiles = await this.operations.listFilesRecursively(); - this.logger.info( - `Scheduling sync for ${allLocalFiles.length} local files` - ); - - let locallyPossiblyDeletedFiles: DocumentRecord[] = []; - - for (const document of this.database.resolvedDocuments) { - if ( - !document.isDeleted && - !(await this.operations.exists(document.relativePath)) - ) { - locallyPossiblyDeletedFiles.push(document); + if (!(error instanceof SyncResetError && this.stopped)) { + this.failure = error; + this.serverConfig.reset(); + this.logger.error(`Sync paused for retry: ${String(error)}`); } } + } - await awaitAll( - allLocalFiles.map(async (relativePath) => { + private async recoverServerHistory(): Promise { + this.onServerHistoryChanged.trigger(); + this.contentCache.reset(); + this.hasScanned = false; + if (!this.database.state.historyRecovery) { + await this.scan(); + const guard = this.localGuard(); + const previous = this.database.state; + const next = emptyState(previous.vaultKey); + next.historyRecovery = true; + next.lastAppliedLocalChangeId = previous.lastAppliedLocalChangeId; + next.protectedPaths = previous.protectedPaths; + for (const [id, path] of Object.entries(previous.local)) { + const info = await this.files.fs.stat(path); + if (info?.kind !== "file") continue; + const doc = previous.documents[id]; if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.metadata !== undefined + this.settings.isIgnored(path) || + this.settings.isOversized(info.size) ) { - this.logger.debug( - `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` - ); - - return this.syncLocallyUpdatedFile({ - relativePath - }); + next.local[id] = path; + next.documents[id] = { + materialized: true, + bootstrap: true + }; + (next.excluded ??= {})[id] = + previous.fileManifest.entries[id] ?? null; + continue; } + const snapshot = await this.files.snapshot(path); + if (!snapshot) throw new LocalChangesDuringReconciliation(); + // An unsent edit from the abandoned history is a separate local + // document. It must never be diffed against a reused version ID. + const clean = doc?.base?.hash === snapshot.hash; + const localId = doc?.base && !clean ? uuid() : id; + next.local[localId] = path; + next.documents[localId] = { + materialized: true, + observedHash: snapshot.hash, + recoveryBase: clean ? snapshot : doc?.recoveryBase + }; + } + guard(); + // Persist the reset BEFORE releasing the old checkpoint. After a + // crash no old request can be retried against the replacement log. + await this.database.commit(next); + } + await this.service.resetHistory(); + const next = this.next(); + delete next.historyRecovery; + await this.database.commit(next); + } - // Perhaps the file has been moved; let's check by looking at the deleted files - const contentHash = await this.syncQueue.add(async () => { - const contentBytes = - await this.operations.read(relativePath); // this can throw FileNotFoundError - return hash(contentBytes); - }); + private localGuard(): () => void { + const generation = this.localGeneration; + return () => { + if (generation !== this.localGeneration) { + throw new LocalChangesDuringReconciliation(); + } + }; + } - if (contentHash == undefined) { - // The file was deleted before we had a chance to read it, no need to sync it here - return; - } + private async cacheContent( + updateId: number, + path: string | undefined, + content: Uint8Array + ): Promise { + if ( + path !== undefined && + !isBinary(content) && + isFileTypeMergable( + path, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) + ) { + this.contentCache.put(updateId, content); + } + } - const originalFile = findMatchingFile( - contentHash, - locallyPossiblyDeletedFiles - ); - if (originalFile !== undefined) { - // `originalFile` hasn't been deleted but it got moved instead - /* eslint-disable no-restricted-syntax -- Comparing by property, not direct equality */ - locallyPossiblyDeletedFiles = - locallyPossiblyDeletedFiles.filter( - (item) => - item.relativePath !== originalFile.relativePath - ); - /* eslint-enable no-restricted-syntax */ + private async pushContent( + path: string, + parentVersionId: number | undefined, + snapshot: StoredSnapshot + ): Promise { + const content = base64ToBytes(snapshot.contentBase64); + const parent = + parentVersionId === undefined + ? undefined + : this.contentCache.get(parentVersionId); + if ( + parent !== undefined && + !isBinary(content) && + !isBinary(parent) && + isFileTypeMergable( + path, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) + ) { + const decode = (bytes: Uint8Array): string => + new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true + }).decode(bytes); + const value = diff(decode(parent), decode(content)); + // Match the server's work bound; large edit sets remain syncable. + if (value.length <= 10_000) return { type: "Diff", value }; + } - this.logger.debug( - `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` - ); + return { type: "Snapshot", value: snapshot.contentBase64 }; + } - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyUpdatedFile({ - oldPath: originalFile.relativePath, - relativePath - }); - } + private next(): EngineState { + return structuredClone(this.database.state); + } - this.logger.debug( - `Document ${relativePath} not found in database, scheduling sync to create it` - ); - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyCreatedFile(relativePath); - }) - ); - - // this has to happen strictly after the previous awaitAll, as that one - // might have removed some of the documents from the list - await awaitAll( - locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { - this.logger.debug( - `Document ${relativePath} has been deleted locally, scheduling sync to delete it` - ); - - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyDeletedFile(relativePath); - }) + private async bootstrap(): Promise { + let vaultSnapshot = this.database.state.bootstrap; + if (!vaultSnapshot) { + vaultSnapshot = await this.service.vaultSnapshot(); + validateFileManifest(vaultSnapshot.fileManifest.entries); + await this.database.commit({ + ...this.next(), + bootstrap: vaultSnapshot + }); + } + await this.incorporateFileManifest( + vaultSnapshot.fileManifest, + vaultSnapshot.headEventId, + vaultSnapshot ); } - /** - * Create fake documents in the database for all files that are present locally - * and also exist remotely. This will stop the subequent syncs from duplicating - * the documents by creating the same documents from multiple clients. - */ - private async createFakeDocumentsFromRemoteState(): Promise { - if (this.database.getHasInitialSyncCompleted()) { + private async scan(initial?: VaultSnapshot): Promise { + await this.flushLocalChanges(); + const hadChanges = this.changes.length > 0; + this.unsyncablePaths = await scanLocalFiles( + { + next: this.next(), + guard: this.localGuard(), + changes: this.changes, + files: this.files, + ignored: (path) => this.settings.isIgnored(path), + oversized: (size) => this.settings.isOversized(size), + commit: async (next) => this.database.commit(next) + }, + initial + ); + this.hasScanned = true; + if (hadChanges) + await this.localChangeStorage.save(structuredClone(this.changes)); + } + + private async incorporateEventBatch(batch: EventBatch): Promise { + let cursor = this.database.state.lastSeenUpdateId; + let manifest = this.database.state.fileManifest; + const heads = { ...this.database.state.remoteHeads }; + const receipts: EventRecord[] = []; + // Fold every page before touching local files. If fetching is interrupted, + // discard this work and retry from the last committed cursor. + for (;;) { + const after = cursor; + for (const event of batch.events) { + if (event.eventId <= cursor) continue; + if (event.eventId !== cursor + 1) + throw new PermanentSyncError( + "Non-contiguous event history; refusing to skip changes" + ); + cursor = event.eventId; + if ( + this.database.state.unconfirmed?.some( + (pending) => + pending.request.requestId === event.requestId + ) === true + ) + receipts.push(event); + if (event.type === "fileManifest") { + if ( + event.fileManifest.fileManifestId > + manifest.fileManifestId + ) + manifest = event.fileManifest; + } else if ( + !heads[event.document.documentId] || + heads[event.document.documentId].vaultUpdateId < + event.document.vaultUpdateId + ) { + heads[event.document.documentId] = this.documentHead( + event.document + ); + } + } + const end = batch.endEventId ?? batch.headEventId; + if ( + end !== cursor || + end > batch.headEventId || + (end === after && end < batch.headEventId) + ) + throw new PermanentSyncError("Incomplete event batch"); + if (end === batch.headEventId) break; + batch = await this.service.events(cursor); + } + if (cursor === this.database.state.lastSeenUpdateId) return; + // Recover our submitted bases before folding later remote edits. A + // permanent retry failure cannot cancel an earlier in-flight attempt. + const next = this.next(); + let acknowledged = false; + for (const event of receipts) { + const pending = next.unconfirmed?.find( + (candidate) => candidate.request.requestId === event.requestId + ); + if (!pending) continue; + if (event.type === "content" && pending.type === "content") { + const doc = next.documents[pending.documentId]; + if (!doc.base || doc.base.vaultUpdateId < event.eventId) { + doc.base = { + ...event.document, + hash: pending.snapshot.hash + }; + doc.bootstrap = false; + delete doc.rejected; + await this.cacheContent( + event.eventId, + next.local[pending.documentId], + base64ToBytes(pending.snapshot.contentBase64) + ); + } + } else if ( + event.type === "fileManifest" && + pending.type === "fileManifest" + ) { + if (next.fileManifest.fileManifestId < event.eventId) { + next.fileManifest = event.fileManifest; + delete next.rejectedManifest; + } + } else + throw new PermanentSyncError( + "Request receipt type does not match submission" + ); + if (next.unconfirmed) removeFromArray(next.unconfirmed, pending); + acknowledged = true; + } + if (acknowledged) await this.database.commit(next); + // Three-way merging compares snapshots. Replaying intermediate remote + // edits can discard a local edit even when the remote ends at its base. + await this.incorporateFileManifest( + manifest, + cursor, + undefined, + false, + heads + ); + } + + private documentHead( + version: DocumentVersionWithoutContent + ): DocumentVersionWithoutContent { + return { + vaultUpdateId: version.vaultUpdateId, + documentId: version.documentId, + updatedDate: version.updatedDate, + userId: version.userId, + deviceId: version.deviceId, + contentSize: version.contentSize + }; + } + + private recordManifestApplication( + before: Readonly>, + writes: Readonly> + ): void { + const after = this.database.state.local; + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(after) + ])) { + const from = before[id]; + const to = after[id]; + if (from === to && !writes[id]) continue; + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Remote change applied", + details: + from && to && from !== to + ? { + type: SyncType.MOVE, + relativePath: to, + movedFrom: from + } + : !from && to + ? { type: SyncType.CREATE, relativePath: to } + : from && !to + ? { type: SyncType.DELETE, relativePath: from } + : { + type: SyncType.UPDATE, + relativePath: to ?? from + } + }); + } + } + + private async remoteSnapshot( + head: DocumentVersionWithoutContent, + path: string, + supplied?: DocumentVersion + ): Promise { + const content = + supplied !== undefined + ? base64ToBytes(supplied.contentBase64) + : await this.service.getDocumentVersionContent({ + documentId: head.documentId, + vaultUpdateId: head.vaultUpdateId + }); + await this.cacheContent(head.vaultUpdateId, path, content); + return toStoredSnapshot({ content }); + } + + private async contentWrite( + next: EngineState, + head: DocumentVersionWithoutContent, + path: string, + supplied?: DocumentVersion + ): Promise { + const document = (next.documents[head.documentId] ??= { + materialized: false + }); + if ( + next.excluded?.[head.documentId] !== undefined || + this.settings.isIgnored(path) || + this.settings.isOversized(head.contentSize) + ) { + return undefined; + } + const currentPath = this.database.state.local[head.documentId]; + const local = + currentPath && + (await this.files.fs.stat(currentPath))?.kind === "file" + ? await this.files.snapshot(currentPath) + : undefined; + if ( + local !== undefined && + this.settings.isOversized(base64ToBytes(local.contentBase64).length) + ) { + return undefined; + } + const base = + document.recoveryBase ?? + (local !== undefined && + document.base !== undefined && + document.bootstrap !== true + ? await this.remoteSnapshot(document.base, path) + : undefined); + // Cache the new head after the older merge base so it remains the most + // recently used version and can back the follow-up local diff. + const remote = await this.remoteSnapshot(head, path, supplied); + let result = remote; + if (local !== undefined) { + result = await mergeContent( + path, + base, + local, + remote, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ); + } + document.base = { ...this.documentHead(head), hash: remote.hash }; + delete document.recoveryBase; + document.bootstrap = false; + document.materialized = true; + document.observedHash = result.hash; + return local?.contentBase64 === result.contentBase64 + ? undefined + : { expected: local, replacement: result }; + } + + private async incorporateContent( + head: DocumentVersionWithoutContent, + cursor?: number, + supplied?: DocumentVersion, + clearPending = false + ): Promise { + await this.scan(); + const guard = this.localGuard(); + const next = this.next(); + if ( + !next.remoteHeads[head.documentId] || + next.remoteHeads[head.documentId].vaultUpdateId < head.vaultUpdateId + ) { + next.remoteHeads[head.documentId] = this.documentHead(head); + } + if (cursor !== undefined) { + next.lastSeenUpdateId = cursor; + } + if (clearPending) { + delete next.pending; + } + const path = next.local[head.documentId]; + const document = next.documents[head.documentId]; + const wasMaterialized = document?.materialized; + const writes: Record = {}; + if ( + path !== undefined && + (!document?.base || + !document.materialized || + document.bootstrap || + document.base.vaultUpdateId < head.vaultUpdateId) + ) { + const write = await this.contentWrite(next, head, path, supplied); + if (write) { + writes[head.documentId] = write; + } + } + await this.files.apply(next, writes, guard); + if (path !== undefined && writes[head.documentId]) { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Remote content applied", + details: { + type: wasMaterialized ? SyncType.UPDATE : SyncType.CREATE, + relativePath: path + }, + author: head.userId, + timestamp: new Date(head.updatedDate) + }); + } + } + + private async incorporateFileManifest( + remote: FileManifest, + cursor?: number, + initial?: VaultSnapshot, + clearPending = false, + heads?: EngineState["remoteHeads"] + ): Promise { + validateFileManifest(remote.entries); + await this.scan(initial); + const before = { ...this.database.state.local }; + const guard = this.localGuard(); + const next = this.next(); + if (cursor !== undefined) { + next.lastSeenUpdateId = cursor; + } + if (clearPending) { + delete next.pending; + } + if (remote.fileManifestId < next.fileManifest.fileManifestId) { + await this.database.commit(next); return; } - const [allLocalFiles, remote] = await awaitAll([ - this.operations.listFilesRecursively(), - this.syncQueue.add(async () => this.syncService.getAll()) - ]); - - if (remote !== undefined) { - remote.latestDocuments - .filter( - (remoteDocument) => - allLocalFiles.includes(remoteDocument.relativePath) && - !remoteDocument.isDeleted && - this.database.getDocumentByDocumentId( - remoteDocument.documentId - ) === undefined - ) - .forEach((remoteDocument) => { - this.database.createNewEmptyDocument( - remoteDocument.documentId, - remoteDocument.vaultUpdateId, - remoteDocument.relativePath + const base = { ...next.fileManifest.entries }; + for (const [id, path] of Object.entries(next.excluded ?? {})) { + if (path === null) delete base[id]; + else base[id] = path; + } + const previousManifest = next.fileManifest.entries; + next.local = mergeFileManifests(base, next.local, remote.entries); + next.fileManifest = remote; + if (heads) next.remoteHeads = heads; + for (const [id, oldPath] of Object.entries(before)) { + const path = next.local[id]; + if ( + this.settings.isIgnored(oldPath) || + this.unsyncablePaths.has(oldPath) || + (path !== undefined && this.settings.isIgnored(path)) + ) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = previousManifest[id] ?? null; + next.local[id] = oldPath; + } else if (next.excluded) delete next.excluded[id]; + } + for (const [id, path] of Object.entries(next.excluded ?? {})) { + if ( + before[id] === undefined && + (path === null || !this.settings.isIgnored(path)) + ) + delete next.excluded![id]; + } + next.local = resolvePaths( + next.local, + remote.entries, + Object.keys(next.excluded ?? {}), + next.protectedPaths + ); + if (initial) { + next.initialized = true; + delete next.bootstrap; + for (const head of initial.documents) { + next.remoteHeads[head.documentId] = this.documentHead(head); + } + } + const writes: Record = {}; + for (const [id, path] of Object.entries(next.local)) { + next.documents[id] ??= { materialized: false }; + if (before[id] === undefined) + next.documents[id].materialized = false; + if ( + next.excluded?.[id] === undefined && + !this.settings.isIgnored(path) && + remote.entries[id] !== undefined && + (!this.database.state.local[id] || + !next.documents[id].base || + next.documents[id].bootstrap || + !next.documents[id].materialized || + (next.remoteHeads[id]?.vaultUpdateId ?? 0) > + (next.documents[id].base?.vaultUpdateId ?? 0)) + ) { + const knownHead = + next.remoteHeads[id] ?? + initial?.documents.find((doc) => doc.documentId === id); + const head = knownHead ?? (await this.service.metadata(id)); + if (!head) { + throw new PermanentSyncError( + `File manifest references missing content: ${id}` ); - }); + } + next.remoteHeads[id] = this.documentHead(head); + const write = await this.contentWrite(next, head, path); + if (write) { + writes[id] = write; + } + } + } + await this.files.apply(next, writes, guard); + this.recordManifestApplication(before, writes); + } + + private async preparePush(): Promise { + if (this.stopped) { + return false; } - this.database.setHasInitialSyncCompleted(true); + let guard = this.localGuard(); + const canonical = resolvePaths( + this.database.state.local, + this.database.state.fileManifest.entries, + Object.keys(this.database.state.excluded ?? {}), + this.database.state.protectedPaths + ); + if (!sameFileManifest(canonical, this.database.state.local)) { + await this.files.apply( + { ...this.next(), local: canonical }, + {}, + guard + ); + await this.scan(); + guard = this.localGuard(); + } + + // Resolve the map after restoring excluded server paths, then make + // those allocations visible locally before submitting a manifest. + const submitted = this.manifestToSubmit(); + const local = { ...this.database.state.local }; + for (const [id, path] of Object.entries(submitted)) + if (this.database.state.excluded?.[id] === undefined) + local[id] = path; + if (!sameFileManifest(local, this.database.state.local)) { + await this.files.apply({ ...this.next(), local }, {}, guard); + await this.scan(); + guard = this.localGuard(); + } + for (const [id, path] of Object.entries(this.database.state.local)) { + if ( + this.settings.isIgnored(path) || + this.database.state.excluded?.[id] !== undefined || + !this.database.state.documents[id]?.materialized || + this.unsyncablePaths.has(path) || + this.settings.isOversized( + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime. + this.database.state.remoteHeads[id]?.contentSize ?? 0 + ) + ) { + continue; + } + const snapshot = await this.files.snapshot(path); + if ( + !snapshot || + this.settings.isOversized( + base64ToBytes(snapshot.contentBase64).length + ) + ) { + continue; + } + const { base, rejected } = this.database.state.documents[id]; + if ( + base?.hash === snapshot.hash || + rejected?.hash === snapshot.hash + ) { + continue; + } + guard(); + await this.database.commit({ + ...this.next(), + pending: { + type: "content", + documentId: id, + snapshot, + request: { + requestId: uuid(), + parentVersionId: base?.vaultUpdateId ?? null, + content: await this.pushContent( + path, + base?.vaultUpdateId, + snapshot + ) + } + } + }); + return true; + } + + const entries = this.manifestToSubmit(); + validateFileManifest(entries); + if ( + !sameFileManifest( + entries, + this.database.state.fileManifest.entries + ) && + (!this.database.state.rejectedManifest || + !sameFileManifest( + entries, + this.database.state.rejectedManifest.entries + )) + ) { + guard(); + await this.database.commit({ + ...this.next(), + pending: { + type: "fileManifest", + request: { + requestId: uuid(), + parentFileManifestId: + this.database.state.fileManifest.fileManifestId, + entries + } + } + }); + return true; + } + return false; + } + + private manifestToSubmit(): Record { + // Do not publish a new document until its initial content is acknowledged. + const entries = { ...this.database.state.local }; + for (const id of Object.keys(this.database.state.excluded ?? {})) { + const remotePath = this.database.state.fileManifest.entries[id]; + if (remotePath === undefined) delete entries[id]; + else entries[id] = remotePath; + } + for (const id of Object.keys(entries)) { + if ( + !this.database.state.documents[id]?.base && + this.database.state.fileManifest.entries[id] === undefined + ) { + delete entries[id]; + } + } + return resolvePaths( + entries, + this.database.state.fileManifest.entries, + Object.keys(this.database.state.excluded ?? {}), + [ + ...(this.database.state.protectedPaths ?? []), + ...Object.keys(this.database.state.excluded ?? {}).flatMap( + (id) => + this.database.state.local[id] + ? [this.database.state.local[id]] + : [] + ) + ] + ); + } + + private get rejectionMessage(): string | undefined { + const { state } = this.database; + for (const [id, path] of Object.entries(state.local ?? {})) { + const doc = state.documents[id]; + if ( + doc?.rejected && + doc.observedHash === doc.rejected.hash && + doc.base?.hash !== doc.rejected.hash && + !this.settings.isIgnored(path) && + state.excluded?.[id] === undefined + ) + return doc.rejected.message; + } + if ( + state.rejectedManifest && + sameFileManifest( + this.manifestToSubmit(), + state.rejectedManifest.entries + ) && + !sameFileManifest( + state.fileManifest.entries, + state.rejectedManifest.entries + ) + ) + return state.rejectedManifest.message; + return undefined; + } + + /** An explicit reset or settings change permits fresh attempts after rejection. */ + public async retryRejectedRequests(): Promise { + await this.database.recoverPersistence(); + const next = structuredClone(this.database.state); + let changed = false; + if (next.rejectedManifest) { + delete next.rejectedManifest; + changed = true; + } + for (const doc of Object.values(next.documents)) { + if (doc.rejected) { + delete doc.rejected; + changed = true; + } + } + if (changed) await this.database.commit(next); + } + + private async finishPending(): Promise { + let { pending } = this.database.state; + if (!pending) { + return; + } + + if (!pending.response && !pending.rejection) { + const copy = structuredClone(pending); + try { + if (copy.type === "content") { + copy.response = await this.service.putFileContent( + copy.documentId, + copy.request + ); + } else { + copy.response = await this.service.pushFileManifest( + copy.request + ); + } + } catch (error) { + if (!(error instanceof PermanentSyncError)) throw error; + copy.rejection = error.message; + } + await this.database.commit({ ...this.next(), pending: copy }); + pending = copy; + } + + if (pending.rejection && !pending.response) { + const next = this.next(); + if (pending.type === "content") { + next.documents[pending.documentId].rejected = { + hash: pending.snapshot.hash, + message: pending.rejection + }; + } else + next.rejectedManifest = { + entries: pending.request.entries, + message: pending.rejection + }; + (next.unconfirmed ??= []).push(pending); + delete next.pending; + await this.database.commit(next); + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + message: pending.rejection, + details: { + type: SyncType.SKIPPED, + relativePath: + pending.type === "content" + ? (next.local[pending.documentId] ?? + pending.documentId) + : "File manifest" + } + }); + return; + } + + if (pending.type === "fileManifest") { + const response = pending.response!; + if (response.type === "StaleBase") { + await this.incorporateFileManifest( + response, + undefined, + undefined, + true + ); + } else { + const next = this.next(); + delete next.rejectedManifest; + next.fileManifest = { + fileManifestId: response.fileManifestId, + entries: pending.request.entries + }; + delete next.pending; + const previous = this.database.state.fileManifest.entries; + await this.database.commit(next); + for (const id of new Set([ + ...Object.keys(previous), + ...Object.keys(next.fileManifest.entries) + ])) { + const before = previous[id], + after = next.fileManifest.entries[id]; + if (before === after) { + continue; + } + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "File manifest change committed", + details: + after === undefined + ? { + type: SyncType.DELETE, + relativePath: before + } + : before === undefined + ? { + type: SyncType.CREATE, + relativePath: after + } + : { + type: SyncType.MOVE, + relativePath: after, + movedFrom: before + } + }); + } + } + } else { + const response = pending.response!; + if (response.type === "StaleBase") { + await this.incorporateContent( + response, + undefined, + undefined, + true + ); + } else { + const next = this.next(); + const doc = (next.documents[pending.documentId] ??= { + materialized: false + }); + doc.base = { + ...this.documentHead(response), + hash: pending.snapshot.hash + }; + doc.bootstrap = false; + delete doc.rejected; + next.remoteHeads[pending.documentId] = + this.documentHead(response); + delete next.pending; + await this.database.commit(next); + const path = next.local[pending.documentId]; + await this.cacheContent( + response.vaultUpdateId, + path, + base64ToBytes(pending.snapshot.contentBase64) + ); + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Content committed", + details: { + type: SyncType.UPDATE, + relativePath: + next.local[pending.documentId] ?? pending.documentId + }, + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + } + } } } diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts deleted file mode 100644 index e3964d30..00000000 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ /dev/null @@ -1,596 +0,0 @@ -import type { - Database, - DocumentRecord, - RelativePath -} from "../persistence/database"; - -import { diff } from "reconcile-text"; -import type { SyncService } from "../services/sync-service"; -import type { Logger } from "../tracing/logger"; -import type { - CommonHistoryEntry, - SyncCreateDetails, - SyncDeleteDetails, - SyncDetails, - SyncHistory, - SyncMovedDetails, - SyncUpdateDetails -} from "../tracing/sync-history"; -import { SyncStatus, SyncType } from "../tracing/sync-history"; -import { EMPTY_HASH, hash } from "../utils/hash"; - -import { base64ToBytes } from "byte-base64"; -import type { Settings } from "../persistence/settings"; -import type { FileOperations } from "../file-operations/file-operations"; -import { createPromise } from "../utils/create-promise"; -import { FileNotFoundError } from "../file-operations/file-not-found-error"; -import { SyncResetError } from "../services/sync-reset-error"; -import { globsToRegexes } from "../utils/globs-to-regexes"; -import type { DocumentVersion } from "../services/types/DocumentVersion"; -import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; -import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; -import { isFileTypeMergable } from "../utils/is-file-type-mergable"; -import { isBinary } from "../utils/is-binary"; -import type { ServerConfig } from "../services/server-config"; - -export class UnrestrictedSyncer { - private ignorePatterns: RegExp[]; - - public constructor( - private readonly logger: Logger, - private readonly database: Database, - private readonly settings: Settings, - private readonly syncService: SyncService, - private readonly operations: FileOperations, - private readonly history: SyncHistory, - private readonly contentCache: FixedSizeDocumentCache, - private readonly serverConfig: ServerConfig - ) { - this.ignorePatterns = globsToRegexes( - this.settings.getSettings().ignorePatterns, - this.logger - ); - - this.settings.onSettingsChanged.add((newSettings) => { - this.ignorePatterns = globsToRegexes( - newSettings.ignorePatterns, - this.logger - ); - }); - } - - public async unrestrictedSyncLocallyCreatedFile( - document: DocumentRecord - ): Promise { - const updateDetails: SyncCreateDetails = { - type: SyncType.CREATE, - relativePath: document.relativePath - }; - - return this.executeSync(updateDetails, async () => { - const originalRelativePath = document.relativePath; - if (document.isDeleted) { - this.logger.debug( - `Document ${originalRelativePath} has been already deleted, no need to create it` - ); - return; - } - - const contentBytes = - await this.operations.read(originalRelativePath); // this can throw FileNotFoundError - const contentHash = hash(contentBytes); - - const response = await this.syncService.create({ - documentId: document.documentId, - relativePath: originalRelativePath, - contentBytes - }); - - // In case a document with the same name (but different ID) had existed remotely that we haven't known about - if (response.relativePath != originalRelativePath) { - this.logger.debug( - `Document ${originalRelativePath} has been created remotely at a different path: ${response.relativePath}, moving it locally` - ); - await this.operations.move( - document.relativePath, - response.relativePath - ); // this can throw FileNotFoundError - } - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - - this.database.addSeenUpdateId(response.vaultUpdateId); - await this.updateCache( - response.vaultUpdateId, - contentBytes, - response.relativePath - ); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully uploaded locally created file` - }); - }); - } - - public async unrestrictedSyncLocallyDeletedFile( - document: DocumentRecord - ): Promise { - const updateDetails: SyncDeleteDetails = { - type: SyncType.DELETE, - relativePath: document.relativePath - }; - - await this.executeSync(updateDetails, async () => { - const response = await this.syncService.delete({ - documentId: document.documentId, - relativePath: document.relativePath - }); - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: document.relativePath - }, - document - ); - - this.database.addSeenUpdateId(response.vaultUpdateId); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully deleted locally deleted file on the server`, - author: response.userId - }); - }); - } - - public async unrestrictedSyncLocallyUpdatedFile({ - oldPath, - document, - // We use the same code path for both local and remote updates. We need to force the update - // if there are no local changes but we know that the remote version is newer. - force = false - }: { - oldPath?: RelativePath; - force?: boolean; - document: DocumentRecord; - }): Promise { - const updateDetails: SyncUpdateDetails | SyncMovedDetails = - oldPath !== undefined - ? { - type: SyncType.MOVE, - relativePath: document.relativePath, - movedFrom: oldPath - } - : { - type: SyncType.UPDATE, - relativePath: document.relativePath - }; - - await this.executeSync(updateDetails, async () => { - const originalRelativePath = document.relativePath; - - if (document.isDeleted || document.metadata === undefined) { - this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to update it` - ); - return; - } - - const contentBytes = await this.operations.read( - document.relativePath - ); // this can throw FileNotFoundError - let contentHash = hash(contentBytes); - - const areThereLocalChanges = !( - document.metadata.hash === contentHash && oldPath === undefined - ); - - let response: DocumentVersion | DocumentUpdateResponse | undefined = - undefined; - - if (areThereLocalChanges) { - const isText = - !isBinary(contentBytes) && - isFileTypeMergable( - document.relativePath, - (await this.serverConfig.getConfig()) - .mergeableFileExtensions - ); - const cachedVersion = this.contentCache.get( - document.metadata.parentVersionId - ); - - response = - isText && cachedVersion !== undefined - ? await this.syncService.putText({ - documentId: document.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - content: diff( - new TextDecoder().decode(cachedVersion), - new TextDecoder().decode(contentBytes) - ) - }) - : await this.syncService.putBinary({ - documentId: document.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - contentBytes - }); - } else { - if (!force) { - this.logger.debug( - `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` - ); - return; - } - - response = await this.syncService.get({ - documentId: document.documentId - }); - } - - // `document` is mutable and reflects the latest state in the local database - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (document.isDeleted) { - this.logger.info( - `Document ${document.relativePath} has been deleted before we could finish updating it` - ); - this.database.addSeenUpdateId(response.vaultUpdateId); - return; - } - - if ( - // `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match - // the latest versions so we still need to update the local versions to turn the fakes into real metadata. - document.metadata.parentVersionId > response.vaultUpdateId - ) { - this.logger.debug( - `Document ${document.relativePath} is already more up to date than the fetched version` - ); - this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through - return; - } - - if (response.isDeleted) { - return this.applyRemoteDeleteLocally(document, response); - } - - let actualPath = document.relativePath; - - if (response.relativePath != originalRelativePath) { - actualPath = response.relativePath; - // Make sure to update the remote relative path to avoid uploading - // the file as a result of this filesystem event. - document.metadata.remoteRelativePath = response.relativePath; - await this.operations.move( - document.relativePath, - response.relativePath - ); // this can throw FileNotFoundError - } - - if (!("type" in response) || response.type === "MergingUpdate") { - const responseBytes = base64ToBytes(response.contentBase64); - contentHash = hash(responseBytes); - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - await this.operations.write( - actualPath, - contentBytes, - responseBytes - ); - await this.updateCache( - response.vaultUpdateId, - responseBytes, - actualPath - ); - - if (!force) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `The file we updated had been updated remotely, so we downloaded the merged version` - }); - } - } else { - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - await this.updateCache( - response.vaultUpdateId, - contentBytes, - actualPath - ); - } - - this.database.addSeenUpdateId(response.vaultUpdateId); - - const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails = - oldPath !== undefined || - response.relativePath != originalRelativePath - ? { - type: SyncType.MOVE, - relativePath: response.relativePath, - movedFrom: originalRelativePath - } - : { - type: SyncType.UPDATE, - relativePath: response.relativePath - }; - - if (areThereLocalChanges) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: actualUpdateDetails, - message: `Successfully uploaded locally updated file to the server`, - author: response.userId - }); - } else { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: actualUpdateDetails, - message: `Successfully downloaded remotely updated file from the server`, - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - } - }); - } - - public async unrestrictedSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent, - document?: DocumentRecord - ): Promise { - const updateDetails: SyncCreateDetails = { - type: SyncType.CREATE, - relativePath: remoteVersion.relativePath - }; - - await this.executeSync(updateDetails, async () => { - if (document?.metadata !== undefined) { - // If the file exists locally, let's pretend the user has updated it - // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` - if ( - document.metadata.parentVersionId >= - remoteVersion.vaultUpdateId - ) { - this.logger.debug( - `Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version` - ); - - return; - } - - return this.unrestrictedSyncLocallyUpdatedFile({ - document, - force: true - }); - } else if (remoteVersion.isDeleted) { - // Either the document hasn't made it to us before and therefore we don't need to delete it, - // or we already have it, in which case the preceeding if would've dealt with it - this.logger.debug( - `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` - ); - return; - } - - // Don't download oversized files - const historyEntryForSkippedOversizedFile = - this.getHistoryEntryForSkippedOversizedFile( - remoteVersion.contentSize, - remoteVersion.relativePath - ); - if (historyEntryForSkippedOversizedFile !== undefined) { - this.history.addHistoryEntry( - historyEntryForSkippedOversizedFile - ); - return; - } - - const contentBytes = - await this.syncService.getDocumentVersionContent({ - documentId: remoteVersion.documentId, - vaultUpdateId: remoteVersion.vaultUpdateId - }); - - // We're trying to create an entirely new document that didn't exist locally - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - // It can happen that a concurrent sync operation has already created the document, so we can bail here - if (document !== undefined) { - this.logger.debug( - `Document ${remoteVersion.relativePath} has already been created locally, no need to create it again` - ); - return; - } - - await this.operations.ensureClearPath(remoteVersion.relativePath); - - const [promise, resolve] = createPromise(); - this.database.updateDocumentMetadata( - { - parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes), - remoteRelativePath: remoteVersion.relativePath - }, - this.database.createNewPendingDocument( - remoteVersion.documentId, - remoteVersion.relativePath, - promise - ) - ); - - await this.operations.create( - remoteVersion.relativePath, - contentBytes - ); - await this.updateCache( - remoteVersion.vaultUpdateId, - contentBytes, - remoteVersion.relativePath - ); - - resolve(); - this.database.removeDocumentPromise(promise); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully downloaded remote file which hadn't existed locally`, - author: remoteVersion.userId, - timestamp: new Date(remoteVersion.updatedDate) - }); - }); - } - - public async executeSync( - details: SyncDetails, - fn: () => Promise - ): Promise { - for (const pattern of this.ignorePatterns) { - if (pattern.test(details.relativePath)) { - this.logger.debug( - `File '${details.relativePath}' is ignored by the ignore pattern: ${pattern}` - ); - return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history - } - } - - try { - // Only check the size of files which already exist locally. - if (await this.operations.exists(details.relativePath)) { - const sizeInBytes = await this.operations.getFileSize( - details.relativePath - ); - const historyEntryForSkippedOversizedFile = - this.getHistoryEntryForSkippedOversizedFile( - sizeInBytes, - details.relativePath - ); - if (historyEntryForSkippedOversizedFile !== undefined) { - this.history.addHistoryEntry( - historyEntryForSkippedOversizedFile - ); - return; - } - } - - return await fn(); - } catch (e) { - if (e instanceof FileNotFoundError) { - // A subsequent sync operation must have been creating to deal with this - this.logger.info( - `Skiping file '${details.relativePath}' because it no longer exists when trying to ${details.type.toLocaleLowerCase()} it` - ); - return; - } - if (e instanceof SyncResetError) { - this.logger.info( - `Interrupting sync operation because of a reset` - ); - return; - } else { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - details, - message: `Failed to sync file '${details.relativePath}' because of ${e} when trying to ${details.type.toLocaleLowerCase()} it` - }); - throw e; - } - } - } - - private getHistoryEntryForSkippedOversizedFile( - sizeInBytes: number, - relativePath: RelativePath - ): CommonHistoryEntry | undefined { - const sizeInMB = Math.round(sizeInBytes / 1024 / 1024); - const { maxFileSizeMB } = this.settings.getSettings(); - if (sizeInMB > maxFileSizeMB) { - return { - status: SyncStatus.SKIPPED, - details: { - type: SyncType.SKIPPED, - relativePath - }, - message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${ - maxFileSizeMB - } MB` - }; - } - } - - private async updateCache( - updateId: number, - contentBytes: Uint8Array, - filePath: RelativePath - ): Promise { - if ( - isFileTypeMergable( - filePath, - (await this.serverConfig.getConfig()).mergeableFileExtensions - ) && - !isBinary(contentBytes) - ) { - this.contentCache.put(updateId, contentBytes); - } - } - - private async applyRemoteDeleteLocally( - document: DocumentRecord, - response: DocumentVersion | DocumentUpdateResponse - ): Promise { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: { - type: SyncType.DELETE, - relativePath: document.relativePath - }, - message: "File has been deleted remotely, so we deleted it locally", - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - - this.database.delete(document.relativePath); - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: response.relativePath - }, - document - ); - - await this.operations.delete(document.relativePath); - - this.database.addSeenUpdateId(response.vaultUpdateId); - } -} diff --git a/frontend/sync-client/src/types/file-manifest-entries.ts b/frontend/sync-client/src/types/file-manifest-entries.ts new file mode 100644 index 00000000..1e23dd06 --- /dev/null +++ b/frontend/sync-client/src/types/file-manifest-entries.ts @@ -0,0 +1,3 @@ +import type { FileManifest } from "../services/types/FileManifest"; + +export type FileManifestEntries = FileManifest["entries"]; diff --git a/frontend/sync-client/src/utils/assert-set-contains-exactly.ts b/frontend/sync-client/src/utils/assert-set-contains-exactly.ts deleted file mode 100644 index 5baf965e..00000000 --- a/frontend/sync-client/src/utils/assert-set-contains-exactly.ts +++ /dev/null @@ -1,13 +0,0 @@ -import assert from "node:assert"; - -export function assertSetContainsExactly(set: Set, ...values: T[]): void { - assert.ok( - set.size === values.length && - Array.from(set).every((value) => values.includes(value)), - `Expected set to contain only ${values.map((v) => '"' + v + '"').join(", ")}, but it contained ${Array.from( - set - ) - .map((v) => '"' + v + '"') - .join(", ")}` - ); -} diff --git a/frontend/sync-client/src/utils/create-client-id.ts b/frontend/sync-client/src/utils/create-client-id.ts index cfa132da..804efa9f 100644 --- a/frontend/sync-client/src/utils/create-client-id.ts +++ b/frontend/sync-client/src/utils/create-client-id.ts @@ -1,15 +1,13 @@ import { v4 as uuidv4 } from "uuid"; +import { packageVersion } from "./package-version"; export function createClientId(): string { - // @ts-expect-error, injected by webpack - const packageVersion = __CURRENT_VERSION__; // eslint-disable-line - const platform = typeof navigator !== "undefined" ? navigator.platform // eslint-disable-line @typescript-eslint/no-deprecated : typeof process !== "undefined" - ? process.platform - : "unknown"; + ? process.platform + : "unknown"; return `vault-link/${packageVersion} (${uuidv4()}; ${platform})`; } diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts index a5f0cc7c..47c850f9 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts @@ -3,6 +3,30 @@ import assert from "node:assert"; import { EventListeners } from "./event-listeners"; describe("EventListeners", () => { + it("isolates throwing and rejecting observers and still notifies later listeners", async () => { + const failures: unknown[] = []; + const listeners = new EventListeners<() => unknown>((error) => + failures.push(error) + ); + const syncError = new Error("sync observer"); + const asyncError = new Error("async observer"); + let called = 0; + listeners.add(() => { + throw syncError; + }); + listeners.add(async () => { + throw asyncError; + }); + listeners.add(() => { + called++; + }); + assert.doesNotThrow(() => { + listeners.trigger(); + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(called, 1); + assert.deepEqual(failures, [syncError, asyncError]); + }); it("should add & remove listeners", () => { const listeners = new EventListeners<() => void>(); // eslint-disable-next-line @typescript-eslint/no-empty-function diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.ts index e08ca65e..0c7b0e0b 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.ts @@ -8,50 +8,65 @@ import { awaitAll } from "../await-all"; export class EventListeners any> { private readonly listeners: TListener[] = []; + public constructor( + private readonly onError: (error: unknown) => void = (error) => { + console.error("Event observer failed", error); + } + ) {} + public get count(): number { return this.listeners.length; } /** - * Adds a new listener to the collection. - * - * @param listener The listener callback to add - * @returns An unsubscribe function that removes this listener when called - */ + * Adds a new listener to the collection. + * + * @param listener The listener callback to add + * @returns An unsubscribe function that removes this listener when called + */ public add(listener: TListener): () => void { this.listeners.push(listener); return () => this.remove(listener); } /** - * Removes a listener from the collection. - * - * @param listener The listener callback to remove - * @returns true if the listener was found and removed, false otherwise - */ + * Removes a listener from the collection. + * + * @param listener The listener callback to remove + * @returns true if the listener was found and removed, false otherwise + */ public remove(listener: TListener): boolean { return removeFromArray(this.listeners, listener); } /** - * Triggers all listeners synchronously with the provided arguments. - * Any returned promises are ignored. Use triggerAsync() to await them. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners synchronously with the provided arguments. + * Observer failures are reported without interrupting delivery or background + * work. Returned promises are observed; use triggerAsync() to await them. + * + * @param args The arguments to pass to each listener + */ public trigger(...args: Parameters): void { - this.listeners.forEach((listener) => { - listener(...args); - }); + for (const listener of [...this.listeners]) { + try { + void Promise.resolve(listener(...args)).catch( + (error: unknown) => { + this.reportFailure(error); + } + ); + } catch (error) { + this.reportFailure(error); + } + } } /** - * Triggers all listeners and awaits any promises they return. - * Synchronous listeners are called immediately, and any async listeners - * are awaited in parallel. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners and awaits any promises they return. + * Synchronous listeners are called immediately, and any async listeners + * are awaited in parallel. + * + * @param args The arguments to pass to each listener + */ public async triggerAsync(...args: Parameters): Promise { await awaitAll( this.listeners @@ -68,4 +83,11 @@ export class EventListeners any> { public clear(): void { this.listeners.length = 0; } + private reportFailure(error: unknown): void { + try { + this.onError(error); + } catch { + // A diagnostic callback must not break background work either. + } + } } diff --git a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts index 51ad41c1..7429cdf8 100644 --- a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts +++ b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts @@ -1,66 +1,32 @@ -// Implements an in-memory fixed-size cache for document contents, - import type { VaultUpdateId } from "../../persistence/database"; -// Doubly-linked list node for O(1) LRU operations -class LRUNode { - public constructor( - public key: VaultUpdateId, - public value: Uint8Array, - public prev: LRUNode | null = null, - public next: LRUNode | null = null - ) {} -} - -// evicting the least recently used documents when the size limit is exceeded. +/** Map insertion order tracks the least recently used document first. */ export class FixedSizeDocumentCache { - private currentSizeInBytes: number; - private readonly cache: Map; - private head: LRUNode | null; // Least recently used - private tail: LRUNode | null; // Most recently used + private currentSizeInBytes = 0; + private readonly cache = new Map(); - public constructor(private maxSizeInBytes: number) { - this.currentSizeInBytes = 0; - this.cache = new Map(); - this.head = null; - this.tail = null; - } + public constructor(private maxSizeInBytes: number) {} public get(updateId: VaultUpdateId): Uint8Array | undefined { - const node = this.cache.get(updateId); - if (node) { - this.moveToTail(node); - return node.value; + const content = this.cache.get(updateId); + if (content !== undefined) { + this.cache.delete(updateId); + this.cache.set(updateId, content); } - - return undefined; + return content; } public put(updateId: VaultUpdateId, content: Uint8Array): void { - if (content.byteLength > this.maxSizeInBytes) { - // Document is too large to fit in the cache - return; - } - - // If the document is already in the cache, update it - const existingNode = this.cache.get(updateId); - if (existingNode != null) { - this.currentSizeInBytes -= existingNode.value.byteLength; - this.removeNode(existingNode); - this.cache.delete(updateId); - } - - const newNode = new LRUNode(updateId, content); - this.cache.set(updateId, newNode); - this.addToTail(newNode); + if (content.byteLength > this.maxSizeInBytes) return; + this.currentSizeInBytes -= this.cache.get(updateId)?.byteLength ?? 0; + this.cache.delete(updateId); + this.cache.set(updateId, content); this.currentSizeInBytes += content.byteLength; this.fitBelowMaxSize(); } public reset(): void { this.cache.clear(); - this.head = null; - this.tail = null; this.currentSizeInBytes = 0; } @@ -70,50 +36,10 @@ export class FixedSizeDocumentCache { } private fitBelowMaxSize(): void { - // Evict least recently used documents if over size limit - while (this.currentSizeInBytes > this.maxSizeInBytes && this.head) { - const lruNode = this.head; - this.removeNode(lruNode); - this.cache.delete(lruNode.key); - this.currentSizeInBytes -= lruNode.value.byteLength; + for (const [id, content] of this.cache) { + if (this.currentSizeInBytes <= this.maxSizeInBytes) break; + this.cache.delete(id); + this.currentSizeInBytes -= content.byteLength; } } - - private removeNode(node: LRUNode): void { - if (node.prev) { - node.prev.next = node.next; - } else { - this.head = node.next; - } - - if (node.next) { - node.next.prev = node.prev; - } else { - this.tail = node.prev; - } - - node.prev = null; - node.next = null; - } - - private addToTail(node: LRUNode): void { - node.prev = this.tail; - node.next = null; - - if (this.tail) { - this.tail.next = node; - } - - this.tail = node; - - this.head ??= node; - } - - private moveToTail(node: LRUNode): void { - if (node === this.tail) { - return; - } - this.removeNode(node); - this.addToTail(node); - } } diff --git a/frontend/sync-client/src/utils/data-structures/locks.test.ts b/frontend/sync-client/src/utils/data-structures/locks.test.ts index 9beb867a..e325496b 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.test.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.test.ts @@ -1,292 +1,46 @@ -import { describe, it, beforeEach } from "node:test"; -import assert from "node:assert"; -import { Logger } from "../../tracing/logger"; -import type { RelativePath } from "../../persistence/database"; -import { Locks } from "./locks"; -import { awaitAll } from "../await-all"; -import { sleep } from "../sleep"; -import { SyncResetError } from "../../services/sync-reset-error"; +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Lock } from "./locks"; -describe("withLock", () => { - const testPath: RelativePath = "test/document/path"; - const testPath2: RelativePath = "test/document/path2"; - const logger = new Logger(); - - // eslint-disable-next-line @typescript-eslint/init-declarations - let locks: Locks; - - beforeEach(() => { - locks = new Locks(logger); +test("queued operations run in order and retain their results", async () => { + const lock = new Lock(); + const held = Promise.withResolvers(); + const started = Promise.withResolvers(); + const calls: number[] = []; + const first = lock.withLock(async () => { + calls.push(1); + started.resolve(); + await held.promise; + calls.push(2); + return "first"; }); - - it("should execute function with single key lock", async () => { - let executionCount = 0; - const result = await locks.withLock(testPath, () => { - executionCount++; - return "success"; - }); - - assert.strictEqual(result, "success"); - assert.strictEqual(executionCount, 1); + const second = lock.withLock(() => { + calls.push(3); + return "second"; }); - - it("should execute async function with single key lock", async () => { - let executionCount = 0; - const result = await locks.withLock(testPath, async () => { - executionCount++; - await sleep(10); - return "async-success"; - }); - - assert.strictEqual(result, "async-success"); - assert.strictEqual(executionCount, 1); - }); - - it("should execute function with multiple key locks", async () => { - let executionCount = 0; - const result = await locks.withLock([testPath, testPath2], () => { - executionCount++; - return "multi-success"; - }); - - assert.strictEqual(result, "multi-success"); - assert.strictEqual(executionCount, 1); - }); - - it("should sort multiple keys to prevent deadlocks", async () => { - const executionOrder: string[] = []; - - // Start two concurrent operations with keys in different orders - const promise1 = locks.withLock([testPath2, testPath], async () => { - executionOrder.push("operation1-start"); - await sleep(50); - executionOrder.push("operation1-end"); - return "result1"; - }); - - const promise2 = locks.withLock([testPath, testPath2], async () => { - executionOrder.push("operation2-start"); - await sleep(50); - executionOrder.push("operation2-end"); - return "result2"; - }); - - const [result1, result2] = await awaitAll([promise1, promise2]); - - assert.strictEqual(result1, "result1"); - assert.strictEqual(result2, "result2"); - // One operation should complete entirely before the other starts - assert.deepStrictEqual(executionOrder, [ - "operation1-start", - "operation1-end", - "operation2-start", - "operation2-end" - ]); - }); - - it("should serialize access to same key", async () => { - const executionOrder: string[] = []; - - const promise1 = locks.withLock(testPath, async () => { - executionOrder.push("operation1-start"); - await sleep(50); - executionOrder.push("operation1-end"); - return "result1"; - }); - - const promise2 = locks.withLock(testPath, async () => { - executionOrder.push("operation2-start"); - await sleep(30); - executionOrder.push("operation2-end"); - return "result2"; - }); - - const [result1, result2] = await awaitAll([promise1, promise2]); - - assert.strictEqual(result1, "result1"); - assert.strictEqual(result2, "result2"); - assert.deepStrictEqual(executionOrder, [ - "operation1-start", - "operation1-end", - "operation2-start", - "operation2-end" - ]); - }); - - it("should allow concurrent access to different keys", async () => { - const executionOrder: string[] = []; - - const promise1 = locks.withLock(testPath, async () => { - executionOrder.push("operation1-start"); - await sleep(50); - - executionOrder.push("operation1-end"); - return "result1"; - }); - - const promise2 = locks.withLock(testPath2, async () => { - executionOrder.push("operation2-start"); - await sleep(30); - executionOrder.push("operation2-end"); - return "result2"; - }); - - const [result1, result2] = await awaitAll([promise1, promise2]); - - assert.strictEqual(result1, "result1"); - assert.strictEqual(result2, "result2"); - // Both operations should run concurrently - assert.strictEqual(executionOrder[0], "operation1-start"); - assert.strictEqual(executionOrder[1], "operation2-start"); - }); - - it("should release locks even if function throws", async () => { - const error = new Error("test error"); - - await assert.rejects( - locks.withLock(testPath, () => { - throw error; - }), - { message: "test error" } - ); - - // Lock should be released, allowing another operation - const result = await locks.withLock( - testPath, - () => "success-after-error" - ); - assert.strictEqual(result, "success-after-error"); - }); - - it("should release locks even if async function throws", async () => { - const error = new Error("async test error"); - - await assert.rejects( - locks.withLock(testPath, async () => { - await sleep(10); - - throw error; - }), - { message: "async test error" } - ); - - // Lock should be released, allowing another operation - const result = await locks.withLock( - testPath, - () => "success-after-async-error" - ); - assert.strictEqual(result, "success-after-async-error"); - }); - - it("should handle empty array of keys", async () => { - const result = await locks.withLock([], () => "empty-keys"); - assert.strictEqual(result, "empty-keys"); - }); - - it("should maintain FIFO order for multiple waiters", async () => { - const executionOrder: string[] = []; - - // Start first operation that holds the lock - const firstPromise = locks.withLock(testPath, async () => { - executionOrder.push("first-start"); - await sleep(100); - executionOrder.push("first-end"); - return "first"; - }); - - // Small delay to ensure first operation starts - await sleep(10); - - // Queue second and third operations - const secondPromise = locks.withLock(testPath, async () => { - executionOrder.push("second-start"); - await sleep(50); - executionOrder.push("second-end"); - return "second"; - }); - - const thirdPromise = locks.withLock(testPath, async () => { - executionOrder.push("third-start"); - await sleep(20); - executionOrder.push("third-end"); - return "third"; - }); - - const [first, second, third] = await awaitAll([ - firstPromise, - secondPromise, - thirdPromise - ]); - - assert.strictEqual(first, "first"); - assert.strictEqual(second, "second"); - assert.strictEqual(third, "third"); - assert.deepStrictEqual(executionOrder, [ - "first-start", - "first-end", - "second-start", - "second-end", - "third-start", - "third-end" - ]); + const third = lock.withLock(() => { + calls.push(4); + return "third"; }); + await started.promise; + assert.deepEqual(calls, [1]); + held.resolve(); + assert.equal(await first, "first"); + assert.equal(await second, "second"); + assert.equal(await third, "third"); + assert.deepEqual(calls, [1, 2, 3, 4]); }); -describe("reset", () => { - const testPath: RelativePath = "test/document/path"; - const logger = new Logger(); - - // eslint-disable-next-line @typescript-eslint/init-declarations - let locks: Locks; - - beforeEach(() => { - locks = new Locks(logger); - }); - - it("should reject pending waiters with SyncResetError while running operation completes", async () => { - const firstPromise = locks.withLock(testPath, async () => { - await sleep(2); - return "first"; - }); - - await sleep(1); - - const secondPromise = locks.withLock(testPath, async () => "second"); - void secondPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function - - locks.reset(); - - assert.strictEqual(await firstPromise, "first"); - - await assert.rejects(secondPromise, (err: Error) => { - assert.ok(err instanceof SyncResetError); - return true; +for (const asynchronous of [false, true]) { + test(`a ${asynchronous ? "rejected promise" : "thrown error"} releases the lock`, async () => { + const lock = new Lock(); + const error = new Error("failed operation"); + const failed = lock.withLock(() => { + if (asynchronous) return Promise.reject(error); + throw error; }); + const next = lock.withLock(() => "continued"); + await assert.rejects(failed, (caught) => caught === error); + assert.equal(await next, "continued"); }); - - it("should allow locks to work normally after reset", async () => { - const firstPromise = locks.withLock(testPath, async () => { - await sleep(1); - return "first"; - }); - - await sleep(1); - - const secondPromise = locks.withLock(testPath, async () => "second"); - void secondPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function - - locks.reset(); - - await firstPromise; - - const result = await locks.withLock(testPath, () => "after-reset"); - assert.strictEqual(result, "after-reset"); - }); - - it("should handle reset with no pending operations", async () => { - locks.reset(); - - const result = await locks.withLock(testPath, () => "success"); - assert.strictEqual(result, "success"); - }); -}); +} diff --git a/frontend/sync-client/src/utils/data-structures/locks.ts b/frontend/sync-client/src/utils/data-structures/locks.ts index e55c76b0..34bf2e60 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.ts @@ -1,166 +1,11 @@ -import { SyncResetError } from "../../services/sync-reset-error"; -import type { Logger } from "../../tracing/logger"; -import { awaitAll } from "../await-all"; - -/** - * Manages exclusive locks on items to prevent concurrent modifications. - * Locks are granted in FIFO order. - * - * @template T The type of the key used for locking - */ -export class Locks { - /** Currently locked keys */ - private readonly locked = new Set(); - - /** Queue of resolve functions waiting for each key */ - private readonly waiters = new Map< - T, - [() => unknown, (err: unknown) => unknown][] - >(); - - public constructor(private readonly logger?: Logger) {} - - /** - * Executes a function while holding exclusive locks on one or more keys. - * - * This method ensures that the provided function runs with exclusive access to the - * specified key(s). Multiple keys are sorted to prevent deadlocks when different - * operations request the same keys in different orders. - * - * @template R The return type of the function to execute - * @param keyOrKeys A single key or array of keys to lock during function execution - * @param fn The function to execute while holding the lock(s). Can be sync or async. - * @returns A Promise that resolves to the return value of the executed function - * - * @example - * ```typescript - * // Lock a single key - * const result = await locks.withLock('file1', () => { - * // Critical section - only one operation can access 'file1' at a time - * return processFile('file1'); - * }); - * - * // Lock multiple keys (prevents deadlocks through consistent ordering) - * await locks.withLock(['file1', 'file2'], async () => { - * // Critical section - exclusive access to both files - * await moveFile('file1', 'file2'); - * }); - * ``` - * - * @throws Any error thrown by the provided function will be propagated after locks are released - */ - public async withLock( - keyOrKeys: T | T[], - fn: () => R | Promise - ): Promise { - const keys = Array.isArray(keyOrKeys) ? keyOrKeys : [keyOrKeys]; - - // Deduplicate keys to prevent deadlock from acquiring same lock twice - const uniqueKeys = Array.from(new Set(keys)); - uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks - - await awaitAll(uniqueKeys.map(async (key) => this.waitForLock(key))); - - try { - return await fn(); - } finally { - uniqueKeys.forEach((key) => { - this.unlock(key); - }); - } - } - - public reset(): void { - // Resolve all waiting promises before clearing to prevent deadlock - // Any operation waiting for a lock will be granted access immediately - for (const waiting of this.waiters.values()) { - for (const [_, reject] of waiting) { - reject(new SyncResetError()); - } - } - this.locked.clear(); - this.waiters.clear(); - } - - /** - * Attempts to acquire a lock immediately without waiting. - * Must call `unlock()` if successful. - * - * @param key The key to lock - * @returns `true` if lock acquired, `false` if already locked - */ - public tryLock(key: T): boolean { - if (this.locked.has(key)) { - return false; - } - - this.locked.add(key); - - return true; - } - - /** - * Waits to acquire a lock, blocking until available. - * Operations are queued in FIFO order. Must call `unlock()` when done. - * - * @param key The key to wait for and lock - * @returns Promise that resolves when lock is acquired - */ - public async waitForLock(key: T): Promise { - if (this.tryLock(key)) { - return Promise.resolve(); - } - - this.logger?.debug(`Waiting for lock on ${key}`); - - return new Promise((resolve, reject) => { - // DefaultDict behavior - let waiting = this.waiters.get(key); - if (!waiting) { - waiting = []; - this.waiters.set(key, waiting); - } - - waiting.push([resolve, reject]); - }); - } - - /** - * Releases a lock and grants access to the next waiting operation in FIFO order. - * Removes the key from locked set if no waiters. - * - * @param key The key to unlock - * @throws {Error} If key is not currently locked - */ - public unlock(key: T): void { - if (!this.locked.has(key)) { - return; - } - - // Remove first waiter to ensure FIFO order - const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? []; - - if (resolveNextWaiting) { - this.logger?.debug(`Granted lock on ${key}`); - resolveNextWaiting(); - } else { - this.locked.delete(key); - } - } -} - +/** Serializes operations in arrival order; failures do not block the queue. */ export class Lock { - private readonly locks: Locks; + private tail: Promise = Promise.resolve(); - public constructor(logger?: Logger) { - this.locks = new Locks(logger); - } - - public async withLock(fn: () => R | Promise): Promise { - return this.locks.withLock(true, fn); - } - - public reset(): void { - this.locks.reset(); + // eslint-disable-next-line @typescript-eslint/promise-function-async -- Preserve the queued operation promise without an extra async wrapper. + public withLock(fn: () => R | Promise): Promise { + const operation = this.tail.then(fn); + this.tail = operation.catch(() => undefined); + return operation; } } diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts deleted file mode 100644 index 7b7271d7..00000000 --- a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { CoveredValues } from "./min-covered"; - -describe("CoveredValues", () => { - it("should initialize with the given min value", () => { - const covered = new CoveredValues(5); - assert.strictEqual(covered.min, 5); - }); - - it("should add values greater than min", () => { - const covered = new CoveredValues(0); - covered.add(3); - assert.strictEqual(covered.min, 0); - covered.add(1); - assert.strictEqual(covered.min, 1); - covered.add(4); - assert.strictEqual(covered.min, 1); - covered.add(2); - assert.strictEqual(covered.min, 4); - }); - - it("should ignore duplicate values", () => { - const covered = new CoveredValues(0); - covered.add(3); - covered.add(3); - covered.add(3); - assert.strictEqual(covered.min, 0); - covered.add(1); - covered.add(2); - assert.strictEqual(covered.min, 3); - }); - - it("should handle multiple consecutive values", () => { - const covered = new CoveredValues(132); - for (let i = 250; i > 132; i--) { - assert.strictEqual(covered.min, 132); - covered.add(i); - } - assert.strictEqual(covered.min, 250); - }); - - it("should handle adding values lower than current min", () => { - const covered = new CoveredValues(5); - covered.add(3); - assert.strictEqual(covered.min, 5); - covered.add(6); - assert.strictEqual(covered.min, 6); - }); - - it("should auto-advance when setting min value", () => { - const covered = new CoveredValues(5); - covered.add(7); - covered.add(8); - covered.add(9); - assert.strictEqual(covered.min, 5); - // Setting min to 6 should auto-advance through 7, 8, 9 - covered.min = 6; - assert.strictEqual(covered.min, 9); - covered.add(10); - assert.strictEqual(covered.min, 10); - }); - - it("should handle setting min value with no consecutive values", () => { - const covered = new CoveredValues(5); - covered.add(10); - covered.add(15); - assert.strictEqual(covered.min, 5); - // Setting min to 8 should not auto-advance (no consecutive values) - covered.min = 8; - assert.strictEqual(covered.min, 8); - // Add 9 to trigger auto-advance to 10 - covered.add(9); - assert.strictEqual(covered.min, 10); - }); -}); diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.ts b/frontend/sync-client/src/utils/data-structures/min-covered.ts deleted file mode 100644 index 8b38822f..00000000 --- a/frontend/sync-client/src/utils/data-structures/min-covered.ts +++ /dev/null @@ -1,61 +0,0 @@ -/** - * A class that tracks the minimum covered value in a sequence of numbers. - * It keeps track of a minimum value based on the seen values. - * - * It expects integers slightly out of order and makes sure that the value of `min` is - * always the minimum of the seen values. This is done with bounded memory usage. - * - * @example - * ```typescript - * const covered = new CoveredValues(0); - * covered.add(2); // seenValues = [2], min = 0 - * covered.add(1); // seenValues = [], min = 2 - * covered.min; // returns 2 - * ``` - */ -export class CoveredValues { - private seenValues: number[] = []; - - public constructor(private minValue: number) {} - - public get min(): number { - return this.minValue; - } - - public set min(value: number) { - this.minValue = Math.max(value, this.minValue); - this.seenValues = this.seenValues.filter((v) => v > this.minValue); - this.advanceMinWhilePossible(); - } - - public add(value: number | undefined): void { - if (value === undefined || value < this.minValue) { - return; - } - - let i = 0; - while (i < this.seenValues.length && this.seenValues[i] < value) { - i++; - } - - if (i === this.seenValues.length) { - this.seenValues.push(value); - } else if (this.seenValues[i] === value) { - return; - } else { - this.seenValues.splice(i, 0, value); - } - - this.advanceMinWhilePossible(); - } - - private advanceMinWhilePossible(): void { - while ( - this.seenValues.length > 0 && - this.seenValues[0] === this.minValue + 1 - ) { - this.seenValues.shift(); - this.minValue++; - } - } -} diff --git a/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts index c64bff18..258db398 100644 --- a/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts +++ b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts @@ -1,5 +1,5 @@ import { sleep } from "../sleep"; -import { Locks } from "../data-structures/locks"; +import { Lock } from "../data-structures/locks"; import type { Logger } from "../../tracing/logger"; export function slowWebSocketFactory( @@ -8,10 +8,8 @@ export function slowWebSocketFactory( ): typeof WebSocket { // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion return class FlakyWebSocket extends WebSocket { - private static readonly RECEIVE_KEY = "websocket-receive"; - private static readonly SEND_KEY = "websocket-send"; - - private readonly locks = new Locks(logger); + private readonly receiving = new Lock(); + private readonly sending = new Lock(); public set onopen(callback: ((event: Event) => void) | null) { super.onopen = async (event: Event): Promise => { @@ -25,18 +23,15 @@ export function slowWebSocketFactory( public set onmessage(callback: ((event: MessageEvent) => void) | null) { super.onmessage = async (event: MessageEvent): Promise => { - await this.locks.withLock( - FlakyWebSocket.RECEIVE_KEY, - async () => { - if (jitterScaleInSeconds > 0) { - await sleep( - Math.random() * jitterScaleInSeconds * 1000 - ); - } - - callback?.(event); + await this.receiving.withLock(async () => { + if (jitterScaleInSeconds > 0) { + await sleep( + Math.random() * jitterScaleInSeconds * 1000 + ); } - ); + + callback?.(event); + }); }; } @@ -70,7 +65,7 @@ export function slowWebSocketFactory( data: string | ArrayBufferLike | Blob | ArrayBufferView ): Promise { // maintain message order - await this.locks.withLock(FlakyWebSocket.SEND_KEY, async () => { + await this.sending.withLock(async () => { if (jitterScaleInSeconds > 0) { await sleep(Math.random() * jitterScaleInSeconds * 1000); } diff --git a/frontend/sync-client/src/utils/find-matching-file.ts b/frontend/sync-client/src/utils/find-matching-file.ts deleted file mode 100644 index c3d323d3..00000000 --- a/frontend/sync-client/src/utils/find-matching-file.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { DocumentRecord } from "../persistence/database"; -import { EMPTY_HASH } from "./hash"; - -// TODO: make this smarter so that offline files can be renamed & edited at the same time -export function findMatchingFile( - contentHash: string, - candidates: DocumentRecord[] -): DocumentRecord | undefined { - if (contentHash === EMPTY_HASH) { - return undefined; - } - - return candidates.find(({ metadata }) => metadata?.hash === contentHash); -} diff --git a/frontend/sync-client/src/utils/hash.test.ts b/frontend/sync-client/src/utils/hash.test.ts new file mode 100644 index 00000000..9cf80545 --- /dev/null +++ b/frontend/sync-client/src/utils/hash.test.ts @@ -0,0 +1,7 @@ +import assert from "node:assert"; +import { test } from "node:test"; +import { EMPTY_HASH, hash } from "./hash"; + +void test("EMPTY_HASH is the SHA-256 hash of empty content", async () => { + assert.strictEqual(await hash(new Uint8Array()), EMPTY_HASH); +}); diff --git a/frontend/sync-client/src/utils/hash.ts b/frontend/sync-client/src/utils/hash.ts index 906b6fad..508001a8 100644 --- a/frontend/sync-client/src/utils/hash.ts +++ b/frontend/sync-client/src/utils/hash.ts @@ -1,12 +1,13 @@ -// https://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript -export function hash(content: Uint8Array): string { - let result = 0; - // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < content.length; i++) { - result = (result << 5) - result + content[i]; - result |= 0; // Convert to 32bit integer - } - return Math.abs(result).toString(16).padStart(8, "0"); +export async function hash(content: Uint8Array): Promise { + const digest = await globalThis.crypto.subtle.digest( + "SHA-256", + new Uint8Array(content) + ); + + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); } -export const EMPTY_HASH = hash(new Uint8Array(0)); +export const EMPTY_HASH = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts index fd316588..08972b22 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts @@ -71,3 +71,17 @@ describe("isFileTypeMergable", () => { ); }); }); + +for (const path of [ + "md", + "txt", + "nested/md", + "nested/txt", + "a.md/txt", + "a.txt/md", + "note." +]) { + it(`does not merge an extensionless file: ${path}`, () => { + assert.equal(isFileTypeMergable(path, ["md", "txt"]), false); + }); +} diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.ts b/frontend/sync-client/src/utils/is-file-type-mergable.ts index a895b3e2..c09deaf6 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.ts @@ -2,8 +2,10 @@ export function isFileTypeMergable( pathOrFileName: string, mergeableExtensions: string[] ): boolean { - const parts = pathOrFileName.split("."); - const fileExtension = parts.at(-1) ?? ""; + const name = pathOrFileName.split("/").at(-1) ?? ""; + const separator = name.lastIndexOf("."); + if (separator < 0) return false; + const fileExtension = name.slice(separator + 1); return mergeableExtensions.includes(fileExtension.toLowerCase()); } diff --git a/frontend/sync-client/src/utils/package-version.test.ts b/frontend/sync-client/src/utils/package-version.test.ts new file mode 100644 index 00000000..64520e22 --- /dev/null +++ b/frontend/sync-client/src/utils/package-version.test.ts @@ -0,0 +1,9 @@ +import assert from "node:assert/strict"; +import { it } from "node:test"; +import { packageVersion } from "./package-version"; + +it("supports direct source imports without webpack globals", async () => { + assert.strictEqual(packageVersion, "development"); + const { SyncClient } = await import("../sync-client"); + assert.strictEqual(typeof SyncClient.create, "function"); +}); diff --git a/frontend/sync-client/src/utils/package-version.ts b/frontend/sync-client/src/utils/package-version.ts new file mode 100644 index 00000000..92d8c0b4 --- /dev/null +++ b/frontend/sync-client/src/utils/package-version.ts @@ -0,0 +1,7 @@ +declare const __CURRENT_VERSION__: string; + +/** Webpack replaces this identifier in bundles; source imports use a stable fallback. */ +export const packageVersion = + typeof __CURRENT_VERSION__ === "undefined" + ? "development" + : __CURRENT_VERSION__; diff --git a/frontend/sync-client/src/utils/portable-path.test.ts b/frontend/sync-client/src/utils/portable-path.test.ts new file mode 100644 index 00000000..1ada24d5 --- /dev/null +++ b/frontend/sync-client/src/utils/portable-path.test.ts @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { test } from "node:test"; +import { validatePortablePaths } from "./portable-path"; + +test("a small deeply nested manifest fits within a bounded validation heap", () => { + const source = pathToFileURL(resolve(__dirname, "portable-path.ts")).href; + const result = spawnSync( + process.execPath, + [ + "--max-old-space-size=64", + "--import", + require.resolve("tsx"), + "--input-type=module", + "--eval", + ` + import { validatePortablePaths } from ${JSON.stringify(source)}; + const directory = Array(20000).fill("a").join("/"); + validatePortablePaths([directory + "/one.md", directory + "/two.md"]); + ` + ], + { encoding: "utf8", timeout: 10000 } + ); + assert.equal(result.status, 0, result.stderr || String(result.error)); +}); + +test("shared directories remain distinct from aliases and file ancestors", () => { + validatePortablePaths(["Notes/one.md", "Notes/two.md", "Other/one.md"]); + for (const paths of [ + ["Notes/one.md", "notes/two.md"], + ["Straße/one.md", "STRASSE/two.md"], + ["Notes", "Notes/one.md"], + ["Notes/one.md", "Notes"], + ["Notes/one.md", "Notes/one.md"] + ]) + assert.throws(() => validatePortablePaths(paths), /Conflicting path/); +}); diff --git a/frontend/sync-client/src/utils/portable-path.ts b/frontend/sync-client/src/utils/portable-path.ts new file mode 100644 index 00000000..f794cae7 --- /dev/null +++ b/frontend/sync-client/src/utils/portable-path.ts @@ -0,0 +1,190 @@ +export const INTERNAL_DIRECTORY = ".vault-link-sync"; +const MAX_COMPONENT_BYTES = 255; +const byteLength = (value: string): number => + new TextEncoder().encode(value).length; +function truncate(value: string, limit: number): string { + let result = ""; + for (const character of value) { + if (byteLength(result + character) > limit) break; + result += character; + } + return result; +} + +const reservedDeviceName = + /^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$/u; + +// NFC -> uppercase -> NFC matches the server's locale-independent alias rules. +const pathKey = (path: string): string => + path.normalize("NFC").toUpperCase().normalize("NFC"); + +export const arePathAliases = (left: string, right: string): boolean => + pathKey(left) === pathKey(right); + +export function findPathWithSameSpelling( + paths: Iterable, + wanted: string +): string | undefined { + const normalized = wanted.normalize("NFC"); + for (const path of paths) { + if (path !== wanted && path.normalize("NFC") === normalized) + return path; + } + return undefined; +} + +export const isInternalPath = (path: string): boolean => + arePathAliases(path.split("/")[0], INTERNAL_DIRECTORY); + +function validatePath(path: string): void { + if (!path || path !== path.normalize("NFC") || isInternalPath(path)) + throw new Error(`Invalid portable path: ${path}`); + + for (const part of path.split("/")) { + if ( + !part || + part === "." || + part === ".." || + /[<>:"\\|?*\p{Cc}]|[. ]$/u.test(part) || + reservedDeviceName.test( + part.split(".")[0].replace(/ +$/u, "").toUpperCase() + ) + ) + throw new Error(`Invalid portable path: ${path}`); + } +} + +export function validatePortablePaths(paths: Iterable): void { + // Component keys keep storage proportional to the input, even when a + // manifest contains thousands of directory levels. + const nodes = new Map< + string, + { id: number; spelling: string; file: boolean } + >(); + for (const path of paths) { + validatePath(path); + const parts = path.split("/"); + let parent = 0; + parts.forEach((spelling, index) => { + const file = index === parts.length - 1; + const key = `${parent}/${pathKey(spelling)}`; + const previous = nodes.get(key); + if ( + previous && + (previous.spelling !== spelling || previous.file || file) + ) + throw new Error(`Conflicting path: ${path}`); + const node = previous ?? { id: nodes.size + 1, spelling, file }; + nodes.set(key, node); + parent = node.id; + }); + } +} + +function portableName(part: string): string { + let result = part + .normalize("NFC") + .replace(/[<>:"\\|?*\p{Cc}]/gu, "_") + .replace(/[. ]+$/u, "_"); + if (!result || result === "." || result === "..") result = "_"; + if ( + reservedDeviceName.test( + result.split(".")[0].replace(/ +$/u, "").toUpperCase() + ) + ) + result = "_" + result; + return result; +} + +function fitName(name: string, suffix: string, file: boolean): string { + const dot = file ? name.lastIndexOf(".") : -1; + const extension = dot > 0 ? truncate(name.slice(dot), 64) : ""; + const stem = dot > 0 ? name.slice(0, dot) : name; + return portableName( + truncate(stem, MAX_COMPONENT_BYTES - byteLength(suffix + extension)) + + suffix + + extension + ); +} + +function pathsConflict(left: string, right: string): boolean { + const a = left.split("/"), + b = right.split("/"); + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (!arePathAliases(a[i], b[i])) return false; + if (a[i] !== b[i]) return true; + } + return true; // Equal names, or a file occupies an ancestor directory. +} + +function isAvailablePath( + candidate: string, + id: string, + occupied: Readonly> +): boolean { + validatePath(candidate); + return Object.entries(occupied).every( + ([otherId, path]) => otherId === id || !pathsConflict(candidate, path) + ); +} + +/** Deterministic allocation also handles a file blocking an ancestor directory. */ +export function allocatePortablePath( + wanted: string, + id: string, + occupied: Readonly> +): string { + id = portableName(truncate(id, 64)); + const originals = wanted.split("/"); + const parts = originals.map((part, index) => { + const portable = portableName(part); + return byteLength(portable) > MAX_COMPONENT_BYTES + ? fitName( + portable, + ` (conflict ${id})`, + index === originals.length - 1 + ) + : portable; + }); + if (isInternalPath(parts.join("/"))) parts[0] = "_" + parts[0]; + let candidate = parts.join("/"); + const suffix = ` (conflict ${id})`; + const limit = (Object.keys(occupied).length + 1) * (parts.length + 1); + for (let attempt = 0; attempt <= limit; attempt++) { + try { + if (isAvailablePath(candidate, id, occupied)) return candidate; + } catch { + /* allocate below */ + } + const current = candidate.split("/"); + let conflictAt = current.length - 1; + for (let i = 0; i < current.length; i++) { + const prefix = current.slice(0, i + 1).join("/"); + if ( + Object.values(occupied).some((path) => { + const other = path + .split("/") + .slice(0, i + 1) + .join("/"); + return ( + arePathAliases(prefix, other) && + (prefix !== other || + path.split("/").length === i + 1 || + i === current.length - 1) + ); + }) + ) { + conflictAt = i; + break; + } + } + const original = parts[conflictAt] ?? current[conflictAt]; + current[conflictAt] = fitName( + original, + suffix + (attempt ? ` (${attempt})` : ""), + conflictAt === current.length - 1 + ); + candidate = current.join("/"); + } + throw new Error(`Unable to allocate a portable path: ${wanted}`); +} diff --git a/frontend/sync-client/src/utils/set-up-telemetry.ts b/frontend/sync-client/src/utils/set-up-telemetry.ts index d9e73a79..bcd81dc2 100644 --- a/frontend/sync-client/src/utils/set-up-telemetry.ts +++ b/frontend/sync-client/src/utils/set-up-telemetry.ts @@ -1,7 +1,5 @@ import * as Sentry from "@sentry/browser"; - -// @ts-expect-error, injected by webpack -const packageVersion = __CURRENT_VERSION__; // eslint-disable-line +import { packageVersion } from "./package-version"; export const setUpTelemetry = (): (() => void) => { Sentry.init({ @@ -24,11 +22,13 @@ export const setUpTelemetry = (): (() => void) => { } }); }; + window.addEventListener("error", onError); const onUnhandledRejection = (event: PromiseRejectionEvent): void => { Sentry.captureException(event.reason); }; + window.addEventListener("unhandledrejection", onUnhandledRejection); return (): void => { diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index b7c3a3fd..899790b8 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -9,7 +9,12 @@ const common = { rules: [ { test: /\.ts$/, - use: ["ts-loader"] + use: [ + { + loader: "ts-loader", + options: { onlyCompileBundledFiles: true } + } + ] }, { test: /\.wasm$/, diff --git a/frontend/test-client/README.md b/frontend/test-client/README.md new file mode 100644 index 00000000..230f42d7 --- /dev/null +++ b/frontend/test-client/README.md @@ -0,0 +1,36 @@ +# Seeded sync workloads + +The old random mock filesystem/agent have been replaced by the same v4 adapter, +client wrapper, and canonical-state assertions as the scripted suite. +Periodic polling is explicitly disabled so it cannot repair missed notifications +and hide failures in these workloads. Polling has separate fake-clock tests. + +From `frontend`: + +```sh +npm run build --workspace sync-client --workspace test-client +node test-client/dist/cli.js --seed 42 --iterations 25 +node test-client/dist/cli.js --replay /path/to/trace-42.json +``` + +The server binary must already be built. Each invocation starts and cleans up its +own server/database, and prints the replay command before executing any actions. +`--artifacts` chooses the trace/result directory. Failures include database, +visible/durable disk images, and CAS request records. The default 25 iterations +are scenarios selected in shuffled cycles of nine operations, using two or three +clients. Existing documents are repeatedly edited, moved, swapped, deleted and +recreated. Concurrent edits share an established base; delayed editor events stay +queued through reconnection and a subsequent remote edit. Create, content-update +and manifest requests receive before/after network faults. + +Each checkpoint serializes a complete expected file set with exact bytes, paths +and persistent identity keys. Independent creates must remain distinct, and a +recreated file must obtain a new UUID. Later checkpoints reassert earlier edits +and moves, rather than retaining only their markers. Old traces without a client +count still replay with two clients. + +All preservation assertions run even with delayed editor events. Explicit deletes +and overwrites exempt only their specific markers; unrelated content still must +appear exactly once. UUID continuity and canonical server state are checked too. +Seeds reproduce choices, not OS scheduling. See the +[full harness documentation](../deterministic-tests/README.md). diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts deleted file mode 100644 index 1640c2ec..00000000 --- a/frontend/test-client/src/agent/mock-agent.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { choose } from "../utils/choose"; -import { v4 as uuidv4 } from "uuid"; -import { assert } from "../utils/assert"; -import type { RelativePath, SyncSettings } from "sync-client"; -import { debugging, Logger, LogLevel, utils } from "sync-client"; -import { MockClient } from "./mock-client"; -import { sleep } from "../utils/sleep"; -import type { LogLine } from "sync-client"; -import { withTimeout } from "../utils/with-timeout"; - -const TIMEOUT_MS = 10 * 60 * 1000; - -export class MockAgent extends MockClient { - private readonly writtenContents: string[] = []; - private readonly pendingActions: Promise[] = []; - - // The renamed file finding algorithm isn't too smart so we can't both update and rename the same file - private readonly doNotTouchWhileOffline: string[] = []; - - public constructor( - initialSettings: Partial, - public readonly name: string, - private readonly doDeletes: boolean, - private readonly doResets: boolean, - useSlowFileEvents: boolean, - private readonly jitterScaleInSeconds: number - ) { - super(initialSettings, useSlowFileEvents); - } - - public async init(): Promise { - await super.init( - debugging.slowFetchFactory(this.jitterScaleInSeconds), - debugging.slowWebSocketFactory( - this.jitterScaleInSeconds, - new Logger() // this logger isn't wired anywhere, so messages to it will be ignored - ) - ); - - assert( - (await this.client.checkConnection()).isSuccessful, - "Connection check failed" - ); - - this.client.logger.onLogEmitted.add((logLine: LogLine) => { - const state = this.client.getSettings().isSyncEnabled - ? "(online) " - : "(offline)"; - const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; - - // HACK: we have to ensure the file has been synced if we want to change it offline without data loss - const historyEntry = /.*History entry: (.*.md).*/.exec( - logLine.message - ); - - if (historyEntry) { - utils.removeFromArray( - this.doNotTouchWhileOffline, - historyEntry[1] - ); - } - switch (logLine.level) { - case LogLevel.ERROR: - console.error(formatted); - - if (!this.useSlowFileEvents) { - // Let's wait for the error to be caught if there was one - // eslint-disable-next-line @typescript-eslint/no-floating-promises - sleep(100).then(() => process.exit(1)); - } - - break; - case LogLevel.WARNING: - console.warn(formatted); - break; - case LogLevel.INFO: - console.info(formatted); - break; - case LogLevel.DEBUG: - console.debug(formatted); - break; - } - }); - - this.client.logger.info("Agent initialized"); - } - - public async act(): Promise { - const options: (() => Promise)[] = [ - this.createFileAction.bind(this) - ]; - - if ( - this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.length === 0 - ) { - options.push(this.disableSyncAction.bind(this)); - } else { - options.push(this.enableSyncAction.bind(this)); - } - - const files = await this.listFilesRecursively(); - - if (files.length > 0) { - options.push( - this.renameFileAction.bind(this, files), - this.updateFileAction.bind(this, files) - ); - - if (this.doDeletes) { - options.push(this.deleteFileAction.bind(this, files)); - } - } - - if (Math.random() < 0.015 && this.doResets) { - // we can't just queue this up as once it's destroyed, no more method calls can go to SyncClient - await this.resetClient(); - } else { - this.pendingActions.push( - (async (): Promise => { - try { - return await choose(options)(); - } catch (error) { - this.client.logger.error( - `Failed to perform an action: ${error}` - ); - this.client.logger.info( - JSON.stringify(this.data, null, 2) - ); - this.client.logger.info( - JSON.stringify(this.localFiles, null, 2) - ); - throw error; - } - })() - ); - } - } - - public async finish(): Promise { - await withTimeout( - (async (): Promise => { - await this.client.setSetting("isSyncEnabled", true); - await utils.awaitAll(this.pendingActions); - await this.client.waitUntilFinished(); - })(), - TIMEOUT_MS, - "finish()" - ); - } - - public async destroy(): Promise { - await withTimeout( - (async (): Promise => { - await this.client.waitUntilFinished(); - await this.client.destroy(); - })(), - TIMEOUT_MS, - "destroy()" - ); - } - - public assertFileSystemsAreConsistent(otherAgent: MockAgent): void { - const globalFiles = Array.from(otherAgent.localFiles.keys()); - const localFiles = Array.from(this.localFiles.keys()); - - const missingInOther = localFiles.filter( - (file) => !otherAgent.localFiles.has(file) - ); - const missingInLocal = globalFiles.filter( - (file) => !this.localFiles.has(file) - ); - - try { - assert( - missingInOther.length === 0, - `Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}` - ); - assert( - missingInLocal.length === 0, - `Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}` - ); - - for (const file of globalFiles) { - const localContent = new TextDecoder().decode( - this.localFiles.get(file) - ); - const otherContent = new TextDecoder().decode( - otherAgent.localFiles.get(file) - ); - assert( - localContent === otherContent, - `Content mismatch for file ${file}:\n${localContent}\n${otherContent}` - ); - } - } catch (e) { - this.client.logger.info( - "Local data: " + JSON.stringify(this.data, null, 2) - ); - this.client.logger.info( - "Local files: " + - Array.from(otherAgent.localFiles.keys()).join(", ") - ); - otherAgent.client.logger.info( - "Local data: " + JSON.stringify(otherAgent.data, null, 2) - ); - otherAgent.client.logger.info( - "Local files: " + - Array.from(otherAgent.localFiles.keys()).join(", ") - ); - - throw e; - } - } - - public assertAllContentIsPresentOnce(): void { - if (this.useSlowFileEvents) { - this.client.logger.info( - // We can't ensure that we have seen every single update - `Skipping content check for ${this.name} because slow file events are enabled` - ); - return; - } - - for (const content of this.writtenContents) { - const found = Array.from(this.localFiles.keys()).filter((key) => { - return new TextDecoder() - .decode(this.localFiles.get(key)) - .includes(content); - }); - - if (this.doDeletes) { - assert( - found.length <= 1, - `[${this.name}] Content ${content} found in ${found.join(", ")}` - ); - } else { - assert( - found.length >= 1, - `[${this.name}] Content ${content} not found in any files` - ); - - assert( - found.length <= 1, - `[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}` - ); - - const [file] = found; - const fileContent = new TextDecoder().decode( - this.localFiles.get(file) - ); - assert( - fileContent.split(content).length == 2, - `Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}` - ); - } - } - } - - private async resetClient(): Promise { - this.client.logger.info(`Resetting client ${this.name}`); - await this.client.destroy(); - await this.init(); - } - - private async createFileAction(): Promise { - const file = this.getFileName(); - - if ( - (!this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file)) || - (await this.exists(file)) - ) { - return; - } - - const content = this.getContent(); - this.client.logger.info( - `Decided to create file ${file} with content ${content}` - ); - - return this.create(file, new TextEncoder().encode(` ${content} `)); - } - - private async disableSyncAction(): Promise { - this.client.logger.info(`Decided to disable sync`); - await this.client.setSetting("isSyncEnabled", false); - } - - private async enableSyncAction(): Promise { - this.client.logger.info(`Decided to enable sync`); - await this.client.setSetting("isSyncEnabled", true); - } - - private async renameFileAction(files: RelativePath[]): Promise { - const file = choose(files); - - // We can't edit files offline that have been updated while offline. - // Otherwise, the resolution logic couldn't handle it. - if ( - !this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file) - ) { - this.client.logger.info( - `Skipping file ${file} because it has been updated while offline` - ); - return; - } - - const newName = this.getFileName(); - - if ( - (!this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(newName)) || - (await this.exists(newName)) - ) { - return; - } - - this.client.logger.info(`Decided to rename file ${file} to ${newName}`); - this.doNotTouchWhileOffline.push(file, newName); - - return this.rename(file, newName); - } - - private async updateFileAction(files: RelativePath[]): Promise { - const file = choose(files); - - // We can't edit files offline that have been updated while offline. - // Otherwise, the resolution logic couldn't handle it. - if ( - !this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file) - ) { - this.client.logger.info( - `Skipping file ${file} because it has been updated while offline` - ); - return; - } - - const content = this.getContent(); - this.client.logger.info( - `Decided to update file ${file} with ${content}` - ); - this.doNotTouchWhileOffline.push(file); - await this.atomicUpdateText(file, (old) => ({ - text: old.text + ` ${content} `, - cursors: [] - })); - } - - private async deleteFileAction(files: RelativePath[]): Promise { - const file = choose(files); - this.client.logger.info(`Decided to delete file ${file}`); - return this.delete(file); - } - - private getContent(): string { - const uuid = uuidv4(); - this.writtenContents.push(uuid); - return uuid; - } - - private getFileName(): string { - // Simulate name collisions between the clients - return `file-${Math.floor(Math.random() * 64)}.md`; - } -} diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts deleted file mode 100644 index c814879a..00000000 --- a/frontend/test-client/src/agent/mock-client.ts +++ /dev/null @@ -1,197 +0,0 @@ -import type { StoredDatabase, TextWithCursors } from "sync-client"; -import { assert } from "../utils/assert"; -import { - type RelativePath, - type FileSystemOperations, - type SyncSettings, - SyncClient -} from "sync-client"; - -export class MockClient implements FileSystemOperations { - protected readonly localFiles = new Map(); - protected client!: SyncClient; - - protected data: Partial<{ - settings: Partial; - database: Partial; - }> = { - database: { - // Assume all clients start at the same time so there's no need to fetch - // any shared state. - hasInitialSyncCompleted: true - } - }; - - public constructor( - initialSettings: Partial, - protected readonly useSlowFileEvents: boolean - ) { - this.data.settings = initialSettings; - } - - public async init( - fetchImplementation: typeof globalThis.fetch, - webSocketImplementation: typeof globalThis.WebSocket - ): Promise { - this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: fetchImplementation, - webSocket: webSocketImplementation - }); - - await this.client.start(); - } - - public async listFilesRecursively( - _root: RelativePath | undefined = undefined // we don't use multi-level paths during tests - ): Promise { - return Array.from(this.localFiles.keys()); - } - - public async read(path: RelativePath): Promise { - const file = this.localFiles.get(path); - if (!file) { - throw new Error(`File ${path} does not exist`); - } - return file; - } - - public async getFileSize(path: RelativePath): Promise { - return (await this.read(path)).length; - } - - public async exists(path: RelativePath): Promise { - return this.localFiles.has(path); - } - - public async create( - path: RelativePath, - newContent: Uint8Array - ): Promise { - if (this.localFiles.has(path)) { - throw new Error(`File ${path} already exists`); - } - this.client.logger.info( - `Creating file ${path} with content ${new TextDecoder().decode(newContent)}` - ); - this.localFiles.set(path, newContent); - - this.executeFileOperation(async () => - this.client.syncLocallyCreatedFile(path) - ); - } - - public async createDirectory(_path: RelativePath): Promise { - // This doesn't mean anything in our virtual FS representation - } - - public async atomicUpdateText( - path: RelativePath, - updater: (currentContent: TextWithCursors) => TextWithCursors - ): Promise { - const file = this.localFiles.get(path); - if (!file) { - throw new Error(`File ${path} does not exist`); - } - const currentContent = new TextDecoder().decode(file); - const newContent = updater({ text: currentContent, cursors: [] }).text; - const newContentUint8Array = new TextEncoder().encode(newContent); - this.localFiles.set(path, newContentUint8Array); - - if (!this.useSlowFileEvents) { - const existingParts = currentContent - .split(" ") - .map((part) => part.trim()); - const newParts = newContent.split(" ").map((part) => part.trim()); - existingParts.forEach((part) => - // all changes should be additive - { - assert( - newParts.includes(part), - `Part ${part} not found in new content: ${newContent}` - ); - } - ); - } - - this.client.logger.info( - `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` - ); - - this.executeFileOperation(async () => - this.client.syncLocallyUpdatedFile({ - relativePath: path - }) - ); - - return newContent; - } - - public async write(path: RelativePath, content: Uint8Array): Promise { - const hasExisted = this.localFiles.has(path); - this.localFiles.set(path, content); - - this.client.logger.info( - `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` - ); - - this.executeFileOperation(async () => { - if (hasExisted) { - return this.client.syncLocallyUpdatedFile({ - relativePath: path - }); - } else { - return this.client.syncLocallyCreatedFile(path); - } - }); - } - - public async delete(path: RelativePath): Promise { - this.client.logger.info( - `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` - ); - this.localFiles.delete(path); - - this.executeFileOperation(async () => - this.client.syncLocallyDeletedFile(path) - ); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - const file = this.localFiles.get(oldPath); - if (!file) { - throw new Error(`File ${oldPath} does not exist`); - } - this.localFiles.set(newPath, file); - if (oldPath !== newPath) { - this.localFiles.delete(oldPath); - } - - this.client.logger.info( - `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` - ); - - this.executeFileOperation(async () => - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }) - ); - } - - private executeFileOperation(callback: () => unknown): void { - if (this.useSlowFileEvents) { - // we aren't the best client and it takes some time to notice changes - setTimeout(callback, Math.random() * 100); - } else { - callback(); - } - } -} diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 3af547e7..eb407615 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -1,205 +1,90 @@ -import type { SyncSettings } from "sync-client"; -import { utils } from "sync-client"; -import { MockAgent } from "./agent/mock-agent"; -import { sleep } from "./utils/sleep"; -import { v4 as uuidv4 } from "uuid"; -import { randomCasing } from "./utils/random-casing"; +import { parseArgs } from "node:util"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { Logger } from "sync-client"; +import { ServerControl } from "../../deterministic-tests/src/server-control"; +import { ServerManager } from "../../deterministic-tests/src/server-manager"; +import { TestRunner } from "../../deterministic-tests/src/test-runner"; +import { generateTrace, traceTest, type Trace } from "./workload"; -const TEST_ITERATIONS = 5; - -// Simulate async file access by injecting waiting time before returning from file operations. -let slowFileEvents = false; - -// Whether to do resets in the test runs -let doResets = false; - -async function runTest({ - agentCount, - concurrency, - iterations, - doDeletes, - useResets, - useSlowFileEvents, - jitterScaleInSeconds -}: { - agentCount: number; - concurrency: number; - iterations: number; - doDeletes: boolean; - useResets: boolean; - useSlowFileEvents: boolean; - jitterScaleInSeconds: number; -}): Promise { - slowFileEvents = useSlowFileEvents; - doResets = useResets; - - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; - console.info(`Running test ${settings}`); - - const vaultName = uuidv4(); - console.info(`Using vault name: ${vaultName}`); - const initialSettings: Partial = { - isSyncEnabled: true, - token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces - vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter - syncConcurrency: concurrency, - remoteUri: "http://localhost:3000" - }; - - const clients: MockAgent[] = []; - for (let i = 0; i < agentCount; i++) { - clients.push( - new MockAgent( - initialSettings, - `agent-${i}`, - doDeletes, - useResets, - useSlowFileEvents, - jitterScaleInSeconds - ) - ); - } - - try { - await utils.awaitAll(clients.map(async (client) => client.init())); - - for (let i = 0; i < iterations; i++) { - console.info(`Iteration ${i + 1}/${iterations}`); - await utils.awaitAll(clients.map(async (client) => client.act())); - await sleep(Math.random() * 200); - } - - console.info("Stopping agents"); - - // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and - for (const client of clients) { - try { - console.info(`Finishing up ${client.name}`); - await client.finish(); - } catch (err) { - if (!slowFileEvents) { - throw err; - } - } - } - - // then we need a second pass to ensure that all agents pull the same state. - for (const client of clients) { - try { - console.info(`Destroying ${client.name}`); - await client.destroy(); - } catch (err) { - if (!slowFileEvents) { - throw err; - } - } - } - - console.info("Agents finished successfully"); - - clients.slice(0, -1).forEach((client, i) => { - console.info( - `Checking consistency between ${client.name} and ${clients[i + 1].name}` - ); - client.assertFileSystemsAreConsistent(clients[i]); - console.info(`Consistency check for ${client.name} passed`); - }); - - console.info("File systems found to be consistent"); - - clients.forEach((client) => { - console.info(`Checking content for ${client.name}`); - client.assertAllContentIsPresentOnce(); - console.info(`Content check for ${client.name} passed`); - }); - - console.info(`Test passed ${settings}`); - } catch (err) { - console.error(`Test failed ${settings}`); - throw err; - } +function integer(value: string, zero = false): number { + const number = Number(value); + if ( + !/^\d+$/.test(value) || + !Number.isSafeInteger(number) || + number < (zero ? 0 : 1) + ) + throw new Error(`Invalid integer: ${value}`); + return number; } -async function runTests(): Promise { - for (let i = 0; i < TEST_ITERATIONS; i++) { - await runTest({ - agentCount: 2, - concurrency: 16, - iterations: 100, - doDeletes: true, - useResets: true, - useSlowFileEvents: true, - jitterScaleInSeconds: 0.75 - }); - - for (const useSlowFileEvents of [true, false]) { - for (const concurrency of [ - 16, - 1 // test with concurrency 1 to check for deadlocks - ]) { - for (const doDeletes of [false, true]) { - await runTest({ - agentCount: 2, - concurrency, - iterations: 100, - doDeletes, - useResets: false, - useSlowFileEvents, - jitterScaleInSeconds: 0.75 - }); - } - } +async function main(): Promise { + const { values } = parseArgs({ + options: { + seed: { type: "string", default: "1" }, + iterations: { type: "string", default: "25" }, + replay: { type: "string" }, + artifacts: { type: "string" }, + "server-binary": { type: "string" }, + config: { type: "string" } } - } -} - -process.on("uncaughtException", (error) => { - if ( - error instanceof Error && - error.message.includes( - "WebSocket was closed before the connection was established" - ) - ) { - return; - } - - console.error("Uncaught exception:", error); - process.exit(1); -}); - -process.on("unhandledRejection", (error, _promise) => { - if (error instanceof Error && error.message === "Sync was reset") { - return; - } - - if ( - slowFileEvents && - error instanceof Error && - (error.message.includes("Document not found") || - error.message.includes("Document already exists at new location")) - ) { - return; - } - - if ( - doResets && - error instanceof Error && - error.message.includes( - "SyncClient has been destroyed and can no longer be used" - ) - ) { - return; - } - - console.error("Unhandled rejection:", error); - process.exit(1); -}); - -runTests() - .then(() => { - process.exit(0); - }) - .catch((err: unknown) => { - console.error(err); - process.exit(1); }); + const seed = integer(values.seed!, true), + iterations = integer(values.iterations!); + const trace: Trace = values.replay + ? (JSON.parse(fs.readFileSync(values.replay, "utf8")) as Trace) + : generateTrace(seed, iterations); + const artifacts = values.artifacts + ? path.resolve(values.artifacts) + : fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-fuzz-")); + fs.mkdirSync(artifacts, { recursive: true }); + const tracePath = path.join(artifacts, `trace-${trace.seed}.json`); + fs.writeFileSync(tracePath, JSON.stringify(trace, null, 2)); + console.log(`Replay: node ${__filename} --replay ${tracePath}`); + const logger = new Logger(); + logger.onLogEmitted.add((line) => console.log(line.message)); + const root = path.resolve(__dirname, "../../.."); + const server = new ServerControl( + values["server-binary"] ?? + path.join(root, "sync-server/target/release/sync_server"), + values.config ?? path.join(root, "sync-server/config-e2e.yml"), + logger + ); + const manager = new ServerManager(logger); + manager.track(server); + manager.installSignalHandlers(); + try { + await server.start(); + const result = await new TestRunner( + server, + logger, + "test-token-change-me", + server.remoteUri + ).runTest(`seed-${trace.seed}`, traceTest(trace)); + fs.writeFileSync( + path.join(artifacts, `result-${trace.seed}.json`), + JSON.stringify(result, null, 2) + ); + if (!result.success) throw new Error(result.error); + } finally { + await server.stop(); + manager.untrack(server); + } +} + +process.on("unhandledRejection", (error) => { + console.error(error); + process.exit(1); +}); +process.on("uncaughtException", (error) => { + console.error(error); + process.exit(1); +}); +main().then( + () => process.exit(0), + (error: unknown) => { + console.error(error); + process.exit(1); + } +); diff --git a/frontend/test-client/src/utils/random-casing.test.ts b/frontend/test-client/src/utils/random-casing.test.ts index 33217525..34d36a37 100644 --- a/frontend/test-client/src/utils/random-casing.test.ts +++ b/frontend/test-client/src/utils/random-casing.test.ts @@ -6,7 +6,7 @@ describe("randomCasing", () => { it("simple test", () => { const input = "hello, this is a really long string with a lot of characters"; - const result = randomCasing(input); + const result = randomCasing(input, () => 0); assert.strictEqual(result.toLowerCase(), input.toLowerCase()); assert.notStrictEqual(result, input); }); diff --git a/frontend/test-client/src/utils/random-casing.ts b/frontend/test-client/src/utils/random-casing.ts index ba09dace..96d7d25a 100644 --- a/frontend/test-client/src/utils/random-casing.ts +++ b/frontend/test-client/src/utils/random-casing.ts @@ -1,7 +1,7 @@ -export function randomCasing(str: string): string { +export function randomCasing(str: string, random: () => number = Math.random): string { const chars = str.split(""); const randomCasedChars = chars.map((char) => { - if (Math.random() < 0.5) { + if (random() < 0.5) { return char.toUpperCase(); } return char.toLowerCase(); diff --git a/frontend/test-client/src/workload.test.ts b/frontend/test-client/src/workload.test.ts new file mode 100644 index 00000000..8d9814f7 --- /dev/null +++ b/frontend/test-client/src/workload.test.ts @@ -0,0 +1,71 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { generateTrace, traceTest } from "./workload"; + +test("concrete traces replay all choices and retain preservation assertions with delayed events", () => { + const trace = generateTrace(42, 12); + assert.deepEqual(trace, generateTrace(42, 12)); + assert.notDeepEqual(trace, generateTrace(43, 12)); + assert.deepEqual( + traceTest(trace), + traceTest(JSON.parse(JSON.stringify(trace))) + ); + assert.equal( + trace.steps.filter((step) => step.type === "assert-markers").length, + 12 + ); + assert(trace.steps.some((step) => step.type === "delay-notifications")); + assert(trace.steps.some((step) => step.type === "delete")); + assert(trace.steps.some((step) => step.type === "assert-documents")); + const assertions = trace.steps.filter( + (step) => step.type === "assert-documents" + ); + assert( + assertions.at(-1)!.expected.some((doc) => doc.conflict), + "Independent creates need two expected documents" + ); + assert( + assertions + .at(-1)! + .expected.some((doc) => + assertions[0].expected.some( + (initial) => initial.key === doc.key + ) + ), + "Later assertions must retain earlier identities" + ); + for (const kind of ["create", "manifest", "content"]) { + assert( + Array.from({ length: 10 }, (_, seed) => + generateTrace(seed, 9) + ).some((generated) => + generated.steps.some( + (step) => + step.type === "drop-response" && step.kind === kind + ) + ) + ); + } + const delayed = trace.steps.findIndex( + (step) => step.type === "delay-notifications" + ); + const flushed = trace.steps.findIndex( + (step, i) => i > delayed && step.type === "flush-notifications" + ); + assert( + trace.steps + .slice(delayed, flushed) + .some((step) => step.type === "enable-sync") + ); + assert( + trace.steps + .slice(delayed, flushed) + .some((step) => step.type === "assert-documents") + ); + assert.equal(traceTest({ ...trace, clients: undefined }).clients, 2); + assert( + Array.from({ length: 10 }, (_, seed) => generateTrace(seed, 1)).some( + (generated) => generated.clients === 3 + ) + ); +}); diff --git a/frontend/test-client/src/workload.ts b/frontend/test-client/src/workload.ts new file mode 100644 index 00000000..4f199f46 --- /dev/null +++ b/frontend/test-client/src/workload.ts @@ -0,0 +1,310 @@ +import type { + ExpectedDocument, + TestDefinition, + TestStep +} from "../../deterministic-tests/src/test-definition"; +import { Random } from "../../test-support/random"; + +export interface Trace { + version: 1; + seed: number; + iterations: number; + /** Old traces omit this and retain their original two-client behavior. */ + clients?: number; + steps: TestStep[]; +} + +type Document = ExpectedDocument & { content: string }; + +/** Concrete actions and independent expected states are serialized together. + * Every round checks the whole history, including earlier bytes and UUIDs. + * A shuffled cycle covers nine actions; most revisit existing documents. + */ +export function generateTrace(seed: number, iterations: number): Trace { + const random = new Random(seed); + const clients = 2 + random.int(2); + const steps: TestStep[] = []; + const documents: Document[] = []; + const markers: string[] = []; + const removed: string[] = []; + let serial = 0; + const newDocument = (path: string): Document => { + const key = `seed-${seed}-document-${serial++}`; + const marker = `[${key}]`; + markers.push(marker); + return { + key, + path, + content: `${marker}\n\nfirst paragraph\n\nlast paragraph\n` + }; + }; + const all = (type: "enable-sync" | "disable-sync") => { + for (let client = 0; client < clients; client++) + steps.push({ type, client }); + }; + const expected = () => + steps.push({ + type: "assert-documents", + expected: structuredClone(documents) + }); + const barrier = () => { + steps.push({ type: "barrier" }); + expected(); + }; + const forget = (doc: Document) => { + for (const marker of markers) + if (doc.content.includes(marker) && !removed.includes(marker)) + removed.push(marker); + documents.splice(documents.indexOf(doc), 1); + }; + all("enable-sync"); + steps.push({ type: "barrier" }); + // A lasting pool makes later actions interact with earlier versions/moves. + for (const path of ["shared/a.md", "日本語/b.md", "café/c.md"]) { + const doc = newDocument(path); + documents.push(doc); + steps.push({ type: "create", client: 0, path, content: doc.content }); + } + barrier(); + let choices: number[] = []; + for (let round = 0; round < iterations; round++) { + if (!choices.length) { + choices = Array.from({ length: 9 }, (_, i) => i); + for (let i = choices.length - 1; i > 0; i--) { + const j = random.int(i + 1); + [choices[i], choices[j]] = [choices[j], choices[i]]; + } + } + const kind = choices.pop()!; + const writer = random.int(clients); + const other = (writer + 1 + random.int(clients - 1)) % clients; + const doc = random.pick(documents.filter((entry) => !entry.conflict)); + const directory = `round-${round}/${random.pick(["nested", "日本語", "café", "space folder"])}`; + if (kind === 0) { + all("disable-sync"); + const first = newDocument(`${directory}/same.md`); + const second = { ...newDocument(first.path), conflict: true }; + steps.push( + { + type: "create", + client: writer, + path: first.path, + content: first.content + }, + { + type: "create", + client: other, + path: second.path, + content: second.content + }, + { + type: "drop-response", + client: writer, + kind: random.pick(["create", "manifest"] as const), + point: random.pick(["before", "after"] as const) + }, + { type: "enable-sync", client: writer }, + { type: "sync", client: writer }, + { type: "wait-for-response-drop", client: writer } + ); + all("enable-sync"); + documents.push(first, second); + } else if (kind === 1) { + // Both clients edit the same established version, at opposite ends. + all("disable-sync"); + const prefix = `remote prefix ${round}\n\n`; + const suffix = `\n\nlocal suffix ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + suffix + }, + { + type: "update", + client: other, + path: doc.path, + content: prefix + doc.content + }, + { type: "enable-sync", client: writer } + ); + all("enable-sync"); + doc.content = prefix + doc.content + suffix; + } else if (kind === 2) { + all("disable-sync"); + const target = `${directory}/moved.md`; + doc.content += `\nremote addition ${round}\n`; + steps.push( + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + }, + { + type: "update", + client: other, + path: doc.path, + content: doc.content + }, + { type: "enable-sync", client: writer } + ); + all("enable-sync"); + doc.path = target; + } else if (kind === 3) { + const second = random.pick( + documents.filter((entry) => !entry.conflict && entry !== doc) + ); + steps.push( + { type: "disable-sync", client: writer }, + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: `${directory}/temporary.md` + }, + { + type: "rename", + client: writer, + oldPath: second.path, + newPath: doc.path + }, + { + type: "rename", + client: writer, + oldPath: `${directory}/temporary.md`, + newPath: second.path + }, + { type: "enable-sync", client: writer } + ); + [doc.path, second.path] = [second.path, doc.path]; + } else if (kind === 4) { + // Reuse the same path but require a new UUID, even in later rounds. + steps.push({ type: "delete", client: writer, path: doc.path }); + forget(doc); + barrier(); + const replacement = newDocument(doc.path); + steps.push({ + type: "create", + client: other, + path: replacement.path, + content: replacement.content + }); + documents.push(replacement); + } else if (kind === 5) { + const target = `${directory}/rapid.md`; + steps.push( + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + }, + { + type: "rename", + client: writer, + oldPath: target, + newPath: doc.path + }, + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + } + ); + doc.path = target; + doc.content += `\nrapid edit ${round}\n`; + steps.push({ + type: "update", + client: writer, + path: doc.path, + content: doc.content + }); + } else if (kind === 6) { + steps.push({ + type: "drop-response", + client: writer, + kind: "content", + point: random.pick(["before", "after"] as const) + }); + doc.content += `\nretry edit ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + }, + { type: "wait-for-response-drop", client: writer } + ); + } else if (kind === 7) { + // Keep an editor event delayed across reconnection and another edit. + steps.push( + { type: "disable-sync", client: writer }, + { type: "delay-notifications", client: writer } + ); + doc.content += `\ndelayed local edit ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + }, + { type: "enable-sync", client: writer } + ); + barrier(); + doc.content += `\nlater remote edit ${round}\n`; + steps.push({ + type: "update", + client: other, + path: doc.path, + content: doc.content + }); + barrier(); + steps.push({ type: "flush-notifications", client: writer }); + } else { + // Explicit replacement retains the UUID. Only markers in the + // overwritten bytes are exempted from preservation. + for (const marker of markers) + if (doc.content.includes(marker) && !removed.includes(marker)) + removed.push(marker); + doc.content = newDocument(doc.path).content; + steps.push({ + type: "update", + client: writer, + path: doc.path, + content: doc.content + }); + } + barrier(); + steps.push({ + type: "assert-markers", + markers: [...markers], + removed: [...removed] + }); + if (random.int(4) === 0) { + steps.push({ type: "reset", client: random.int(clients) }); + barrier(); + } + } + return { version: 1, seed, iterations, clients, steps }; +} + +export function traceTest(trace: Trace): TestDefinition { + if ( + trace.version !== 1 || + !Array.isArray(trace.steps) || + !trace.steps.length || + (trace.clients !== undefined && + (!Number.isInteger(trace.clients) || trace.clients < 2)) + ) + throw new Error("Invalid/empty replay trace"); + return { + clients: trace.clients ?? 2, + description: `Seed ${trace.seed}; ${trace.iterations} generated conflict scenarios`, + steps: trace.steps + }; +} diff --git a/frontend/test-client/tsconfig.json b/frontend/test-client/tsconfig.json index e86df89d..7c59fb92 100644 --- a/frontend/test-client/tsconfig.json +++ b/frontend/test-client/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "baseUrl": ".", "strict": true, + "skipLibCheck": true, "target": "ES2022", "module": "CommonJS", "esModuleInterop": true, diff --git a/frontend/test-client/webpack.config.js b/frontend/test-client/webpack.config.js index 6aee1547..11c3e6dd 100644 --- a/frontend/test-client/webpack.config.js +++ b/frontend/test-client/webpack.config.js @@ -4,6 +4,7 @@ const webpack = require("webpack"); module.exports = { entry: "./src/cli.ts", target: "node", + externals: { "sync-client": "commonjs sync-client" }, mode: "production", optimization: { minimize: false diff --git a/frontend/test-support/README.md b/frontend/test-support/README.md new file mode 100644 index 00000000..6b5d3cb8 --- /dev/null +++ b/frontend/test-support/README.md @@ -0,0 +1,57 @@ +# Shared sync harness support + +Test-only code shared by the deterministic and seeded clients. + +| Suite | Coverage | +| --- | --- | +| `harness.test.ts`, `oracle-regressions.test.ts` | Independent content/identity oracles, exercised faults, exclusive operations, persistence ownership, cleanup and seeds | +| `file-application.test.ts` | Swaps, file/directory transitions, visible conflict files, live edits, exclusions, and one metadata save per application | +| `recovery.test.ts` | Interrupt every application filesystem/save boundary, reopen metadata, modify disk, and verify stable ordinary scans; manifest policy and notification identity | +| `engine-regressions.test.ts`, `sync-audit.test.ts` | Offline identities, unknown network outcomes, rejection/retry, exclusions, interrupted in-memory pagination, namespace races and lifecycle changes | +| `event-delivery.test.ts`, `polling.test.ts` | WebSocket hints with HTTP-only event reads, missed notifications, opt-in polling, shutdown and retry | +| `protocol.test.ts` | Abrupt worker termination after accepted requests, exact retries, real server SIGKILL, CAS and event ordering | +| `restart-reconciliation.test.ts` | Interrupted file application followed by user edits, two-client/server convergence and a second restart | +| `server-restore.test.ts` | History checkpoints, server backup restores and interrupted metadata resets | +| `content-limits.test.ts`, `protocol-audit.test.ts` | Content validation and API invariants | +| `real-disk.test.ts` | Production Node adapter exclusive create/move, unsafe links, and rename cycles | +| `production-adapters.test.ts` | Actual CLI processes, raw watchers, Obsidian adapter, complete metadata and restart after SIGKILL | + +The filesystem interruption tests assert the supported contract: complete metadata +remains usable, scans observe the current files, and a second scan is stable. +They do not demand the pre-interruption file contents, identities or a replay of +unfinished operations. Normal-operation tests still check exact bytes and identities. +There is no filesystem journal or hidden recovery archive. + +`MemoryDisk` exposes `before:`, `visible:` and `durable:` boundaries. A process +interruption retains visible changes; a simulated power loss rolls unflushed +changes back. This provides different partial-application states for testing; it +is not an APFS/ext4 writeback model or a production durability requirement. +`MemoryPersistence` atomically replaces complete objects and can reject before or +after replacement. Abandoned filesystem sessions are fenced against later writes. + +`crash-worker.ts` runs a complete client in a worker, which the parent terminates +at accepted-response checkpoints without graceful shutdown. A new runtime receives +the saved disk and metadata. Real-disk tests exercise the production Node adapter. +The production integration suite launches built CLI processes and an Obsidian +filesystem adapter against a real server. Obsidian unit tests exercise unsaved editor +text, selections and mobile storage contracts; they do not launch the Obsidian app. +Filesystem path checks are not a sandbox against hostile concurrent changes to +ancestor directories. + +Scripted and seeded suites run multiple clients against real servers. They check +visible bytes, UUID mappings, canonical server state, content heads and event cursors. +Polling is disabled except in tests specifically exercising it. Faults must fire; +missed injections fail cleanup rather than silently passing. Workload traces, +per-worker results and logs are saved by `scripts/e2e.sh` for reproduction. + +Run from the repository root: + +```sh +E2E_WORKERS=32 E2E_SEED=12 E2E_TIMEOUT_SECONDS=9000 E2E_ITERATIONS=100 scripts/e2e.sh +``` + +The command rebuilds peers and both production integrations, runs core and adapter +unit tests plus harness/protocol/scripted suites, and then +runs seeds 12 through 43. `E2E_ARTIFACTS` optionally selects the artifact directory. +The runner tests also verify subprocess exit codes, log errors, spawn failures, +timeouts, workspace dependency consistency and CI artifact wiring. diff --git a/frontend/test-support/canonical.ts b/frontend/test-support/canonical.ts new file mode 100644 index 00000000..296413a2 --- /dev/null +++ b/frontend/test-support/canonical.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import type { StoredDatabase } from "sync-client"; +import { assertQuiescent, assertSameFiles } from "./oracles"; + +export interface CanonicalSnapshot { + headEventId: number; + fileManifest: { fileManifestId: number; entries: Record }; + documents: { documentId: string; vaultUpdateId: number }[]; +} +export interface InspectableClient { + files(): Map; + database(): Partial | undefined; +} + +export async function getJson(url: string, token: string): Promise { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + "Device-Id": "harness-observer" + }, + signal: AbortSignal.timeout(5_000) + }); + assert(response.ok, `Observer GET ${url}: HTTP ${response.status}`); + return (await response.json()) as T; +} + +/** Equality alone can pass when every replica loses the same bytes. Check the + * canonical UUIDs, content heads, receipts and event cursor as well. + * Scenario-specific content/identity oracles still run separately. + */ +export async function assertCanonical( + clients: InspectableClient[], + vaultUrl: string, + token: string +): Promise { + assert.equal( + new Set(clients).size, + clients.length, + "Cannot compare a client to itself" + ); + const files = clients.map((client) => client.files()); + const states = clients.map((client) => client.database()); + assertSameFiles(files); + const snapshot = await getJson( + `${vaultUrl}/vault-snapshot`, + token + ); + const entries = snapshot.fileManifest.entries; + for (const id of Object.keys(entries)) + assert( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + id + ), + `Non-UUID document identity: ${id}` + ); + const heads = Object.fromEntries( + snapshot.documents.map((doc) => [doc.documentId, doc]) + ); + assert.equal( + Object.keys(heads).length, + snapshot.documents.length, + "Duplicate document heads" + ); + assert.deepEqual( + Object.keys(heads).sort(), + Object.keys(entries).sort(), + "Snapshot heads differ from membership" + ); + for (const [i, state] of states.entries()) { + assert(state?.initialized, `Client ${i} never initialized`); + assertQuiescent(state, entries, snapshot.headEventId); + assert.equal( + state.fileManifest?.fileManifestId, + snapshot.fileManifest.fileManifestId, + "Wrong manifest CAS parent" + ); + assert.deepEqual( + [...files[i].keys()].sort(), + Object.values(entries).sort(), + "Untracked or missing disk files" + ); + for (const id of Object.keys(entries)) { + assert( + state.documents?.[id]?.materialized, + `Unmaterialized document: ${id}` + ); + assert.equal( + state.documents?.[id]?.base?.vaultUpdateId, + heads[id]?.vaultUpdateId, + `Wrong content CAS parent: ${id}` + ); + } + } + for (const [id, path] of Object.entries(entries)) { + const doc = await getJson<{ contentBase64: string }>( + `${vaultUrl}/documents/${id}`, + token + ); + const serverBytes = new Uint8Array( + Buffer.from(doc.contentBase64, "base64") + ); + assert.deepEqual( + files[0].get(path), + serverBytes, + `Server bytes differ for ${id} at ${path}` + ); + const hash = createHash("sha256").update(serverBytes).digest("hex"); + for (const [i, state] of states.entries()) { + assert.equal( + state!.documents?.[id]?.base?.hash, + hash, + `Wrong base hash: client ${i}, ${id}` + ); + assert.equal( + state!.documents?.[id]?.observedHash, + hash, + `Wrong observed hash: client ${i}, ${id}` + ); + } + } + const events: { eventId: number; requestId: string }[] = []; + let after = 0; + do { + const batch = await getJson<{ + headEventId: number; + endEventId?: number; + events: { eventId: number; requestId: string }[]; + }>(`${vaultUrl}/events-since?after=${after}`, token); + assert.equal( + batch.headEventId, + snapshot.headEventId, + "Server changed while checking convergence" + ); + const end = batch.endEventId ?? batch.headEventId; + assert.equal( + end, + batch.events.at(-1)?.eventId ?? after, + "Event page cursor differs from its contents" + ); + if (after < snapshot.headEventId) + assert(end > after, "Event page made no progress"); + events.push(...batch.events); + after = end; + } while (after < snapshot.headEventId); + assert.deepEqual( + events.map((e) => e.eventId), + Array.from({ length: snapshot.headEventId }, (_, i) => i + 1), + "Server event log is not contiguous and totally ordered" + ); + assert.equal( + new Set(events.map((e) => e.requestId)).size, + events.length, + "An idempotent request emitted multiple events" + ); + return snapshot; +} diff --git a/frontend/test-support/content-limits.test.ts b/frontend/test-support/content-limits.test.ts new file mode 100644 index 00000000..baf510a4 --- /dev/null +++ b/frontend/test-support/content-limits.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +test("stale manifest reconciliation must not fetch excluded remote content", async () => { + const f = await fixture(); + try { + const id = randomUUID(), + big = randomUUID(); + await f.put("/documents/" + id, f.content("A")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.disk.userRename("a.md", "renamed.md"); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "renamed.md" + }); + let injected = false, + downloaded = 0; + f.setFetchHook(async (input, init) => { + const path = String(input); + if ( + !injected && + init?.method === "PUT" && + path.endsWith("/file-manifest") + ) { + injected = true; + await f.put("/documents/" + big, f.content("B".repeat(4096))); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "a.md", [big]: "big.bin" } + }); + } + const response = await fetch(input, init); + if ( + path.includes("/documents/" + big) && + !path.endsWith("/metadata") && + (!init?.method || init.method === "GET") + ) + downloaded += (await response.clone().arrayBuffer()).byteLength; + return response; + }); + await f.client.setSetting("isSyncEnabled", true); + + assert(injected); + assert(!f.disk.userFiles().has("big.bin")); + assert.equal(downloaded, 0); + } finally { + await f.dispose(); + } +}); + +test("stale content acknowledgements contain metadata without the excluded payload", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + await f.put("/documents/" + id, f.content("B".repeat(4096))); + const stale = await f.put("/documents/" + id, f.content("LOCAL")); + assert.equal(stale.type, "StaleBase"); + assert.equal(stale.contentSize, 4096); + assert.equal(stale.contentBase64, undefined); + const metadata = await f.get("/documents/" + id + "/metadata"); + assert.equal(metadata.contentSize, 4096); + assert.equal(metadata.contentBase64, undefined); + } finally { + await f.dispose(); + } +}); + +test("highly fragmented local edits use a snapshot within the server operation budget", async () => { + const { fixture } = await import("./sync-fixture"); + const { toStoredSnapshot } = await import( + "../sync-client/src/sync-operations/content" + ); + const f = await fixture({}); + f.contentCache.resize(1024 * 1024); + const original = Array.from({ length: 12_000 }, (_, i) => `word${i}`).join( + " " + ); + const edited = Array.from({ length: 12_000 }, (_, i) => + i % 2 ? `changed${i}` : `word${i}` + ).join(" "); + f.contentCache.put(1, bytes(original)); + const syncer = f.syncer as unknown as { + pushContent( + path: string, + version: number, + snapshot: unknown + ): Promise<{ type: string }>; + }; + const request = await syncer.pushContent( + "a.md", + 1, + await toStoredSnapshot({ content: bytes(edited) }) + ); + assert.equal(request.type, "Snapshot"); +}); diff --git a/frontend/test-support/engine-regressions.test.ts b/frontend/test-support/engine-regressions.test.ts new file mode 100644 index 00000000..f75aee62 --- /dev/null +++ b/frontend/test-support/engine-regressions.test.ts @@ -0,0 +1,757 @@ +import assert from "node:assert/strict"; +import { PermanentSyncError } from "../sync-client/src/errors/errors"; +import { test } from "node:test"; +import { Database, emptyState } from "../sync-client/src/persistence/database"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { SyncClient, type StoredClient } from "../sync-client/src/sync-client"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { allocatePortablePath } from "../sync-client/src/utils/portable-path"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +import { bytes, head, fixture } from "./sync-fixture"; + +test("offline content notifications need no saves and restart reads the current bytes", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("original")); + const initial = emptyState(JSON.stringify(["http://offline.test", "test"])); + initial.initialized = true; + initial.local = { a: "a.md" }; + initial.documents.a = { + materialized: true, + observedHash: (await toStoredSnapshot({ content: bytes("original") })) + .hash + }; + const store = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + let client = await SyncClient.create({ fs: disk, persistence: store }); + try { + const saved = store.snapshot(); + store.boundary = () => { + throw new Error("content notifications must not save"); + }; + for (let edit = 0; edit < 20; edit++) { + await disk.userWrite("a.md", bytes(`edit ${edit}`)); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + await client.destroy(); + assert.deepEqual(store.snapshot(), saved); + store.boundary = () => {}; + // A final edit has no notification at all. + await disk.userWrite("a.md", bytes("latest offline bytes")); + client = await SyncClient.create({ fs: disk, persistence: store }); + const internals = client as unknown as { + database: Database; + syncer: { + stopped: boolean; + scan(): Promise; + preparePush(): Promise; + }; + }; + await internals.syncer.scan(); + internals.syncer.stopped = false; + assert.equal(await internals.syncer.preparePush(), true); + const pending = internals.database.state.pending; + assert(pending?.type === "content"); + assert.equal(pending.documentId, "a"); + assert.equal( + Buffer.from(pending.snapshot.contentBase64, "base64").toString(), + "latest offline bytes" + ); + } finally { + store.boundary = () => {}; + await client.destroy(); + } +}); + +test("legacy content notifications preserve the acknowledged namespace prefix on restart", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("d.md", bytes("A")); + await disk.userWrite("b.md", bytes("B")); + const initial = emptyState(JSON.stringify(["http://offline.test", "test"])); + initial.initialized = true; + initial.local = { a: "c.md", b: "b.md" }; + initial.fileManifest.entries = { ...initial.local }; + for (const [id, content] of [ + ["a", "A"], + ["b", "B"] + ]) + initial.documents[id] = { + materialized: true, + observedHash: (await toStoredSnapshot({ content: bytes(content) })) + .hash + }; + initial.lastAppliedLocalChangeId = "consumed-update"; + const saved: StoredClient = { + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" }, + localChangesVaultKey: initial.vaultKey, + localChanges: [ + { + type: "move", + oldPath: "a.md", + relativePath: "b.md", + identities: { a: "a.md" }, + changeId: "consumed-move" + }, + { type: "update", path: "c.md", changeId: "consumed-update" }, + { type: "update", path: "b.md", changeId: "new-update" }, + { + type: "move", + oldPath: "c.md", + relativePath: "d.md", + identities: { a: "c.md" }, + changeId: "new-move" + } + ] + }; + const store = new MemoryPersistence(saved); + const client = await SyncClient.create({ fs: disk, persistence: store }); + try { + const internals = client as unknown as { + database: Database; + syncer: { scan(): Promise }; + }; + await internals.syncer.scan(); + assert.deepEqual(internals.database.state.local, { + a: "d.md", + b: "b.md" + }); + assert.equal( + internals.database.state.lastAppliedLocalChangeId, + "new-move" + ); + } finally { + await client.destroy(); + } +}); + +test("remote binary replacement and deletion do not create backup artifacts", async () => { + const f = await fixture({ "a.bin": "private edit" }); + await f.files.apply(structuredClone(f.database.state), { + a: { replacement: await toStoredSnapshot({ content: bytes("remote") }) } + }); + assert.deepEqual(await f.disk.listFilesRecursively(), ["a.bin"]); + assert.deepEqual(f.disk.userFiles(), new Map([["a.bin", bytes("remote")]])); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + assert.deepEqual(await f.disk.listFilesRecursively(), []); +}); + +test("rejected content recovery remains idempotent across a rename and network failure", async () => { + let offline = true; + const f = await fixture(undefined, { + getDocumentVersionContent: async () => { + if (offline) throw new Error("offline"); + return bytes("base"); + } + }); + const snapshot = (await f.files.snapshot("a.md"))!; + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.pending = { + type: "content", + documentId: "a", + snapshot, + request: { + requestId: "request", + parentVersionId: 1, + content: { type: "Snapshot", value: snapshot.contentBase64 } + }, + response: { + type: "StaleBase", + ...head("a", 2, "remote") + } + }; + await f.database.save(); + await f.disk.userRename("a.md", "renamed.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "renamed.md" + }); + await assert.rejects(f.internals.finishPending(), /offline/); + offline = false; + await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.equal(f.database.state.local.a, "renamed.md"); +}); + +test("conflict names respect component limits without imposing a total path limit", () => { + const wanted = "a".repeat(235) + ".md"; + const allocated = allocatePortablePath( + wanted, + "00000000-0000-4000-8000-000000000001", + { other: wanted } + ); + assert(Buffer.byteLength(allocated) <= 255); + assert(allocated.endsWith(".md")); + assert.notEqual(allocated, wanted); + const longPath = "folder/".repeat(100) + "note.md"; + assert.equal(allocatePortablePath(longPath, "id", {}), longPath); +}); + +test("an unmaterialized file with a directory at its path is deconflicted", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("remote") } + ); + await f.disk.userWrite("folder/note.md", bytes("local")); + f.database.state.local.a = "folder"; + f.database.state.documents.a = { materialized: false }; + f.database.state.fileManifest.entries.a = "folder"; + await f.database.save(); + await f.internals.incorporateContent(head("a", 2, "remote")); + assert( + [...f.disk.userFiles().values()].some( + (value) => Buffer.from(value).toString() === "remote" + ) + ); + assert( + [...f.disk.userFiles().values()].some( + (value) => Buffer.from(value).toString() === "local" + ) + ); +}); + +test("ignored materialized content survives remote deletion", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); + assert.deepEqual(f.database.state.fileManifest.entries, {}); +}); + +test("unfinished bootstrap cannot be rebound to another vault", async () => { + const database = emptyState(JSON.stringify(["http://example.test", "old"])); + database.bootstrap = { + headEventId: 1, + fileManifest: { fileManifestId: 1, entries: {} }, + documents: [] + }; + const persistence = new MemoryPersistence({ + settings: { remoteUri: "http://example.test", vaultName: "old" }, + database + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence + }); + try { + await assert.rejects( + client.setSettings({ vaultName: "new" }), + /separate state store/ + ); + assert.equal(persistence.snapshot().settings!.vaultName, "old"); + } finally { + await client.destroy(); + } +}); + +test("a rejected upload is retained locally and does not block another file or a corrected snapshot", async () => { + let reject = true; + const sent: string[] = []; + const f = await fixture( + { "a.md": "too large", "b.md": "other" }, + { + putFileContent: async (id) => { + sent.push(id); + if (id === "a" && reject) + throw new PermanentSyncError("HTTP 413"); + return { + type: "Accepted", + ...head(id, sent.length + 2, "small") + }; + } + } + ); + f.internals.stopped = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.match(f.database.state.documents.a.rejected!.message, /413/); + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(sent, ["a", "b"]); + assert.equal(await f.internals.preparePush(), false); + await f.disk.userWrite("a.md", bytes("small")); + reject = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(sent, ["a", "b", "a"]); + assert.equal(f.database.state.documents.a.rejected, undefined); +}); + +test("a transient upload failure retries the exact request and recovers its acknowledgement", async () => { + const requests: unknown[] = []; + const f = await fixture(undefined, { + putFileContent: async (_id, request) => { + requests.push(structuredClone(request)); + if (requests.length === 1) throw new Error("lost upload reply"); + return { type: "Accepted", ...head("a", 2, "local") }; + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + await assert.rejects(f.internals.finishPending(), /lost upload reply/); + assert(f.database.state.pending); + await f.internals.finishPending(); + assert.equal(requests.length, 2); + assert.deepEqual(requests[1], requests[0]); + assert.equal(f.database.state.documents.a.base!.vaultUpdateId, 2); + assert.equal(f.database.state.documents.a.rejected, undefined); + assert.equal(f.database.state.pending, undefined); +}); + +test("a persisted upload rejection finishes without filesystem writes", async () => { + let uploads = 0; + const f = await fixture(undefined, { + putFileContent: async () => { + uploads++; + throw new PermanentSyncError("HTTP 413"); + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + f.disk.boundary = (label) => { + if (!label.startsWith("read:")) + throw new Error("Unexpected filesystem mutation"); + }; + await f.internals.finishPending(); + assert.equal(uploads, 1); + assert.equal(f.database.state.pending, undefined); + assert.equal(f.database.state.documents.a.rejected?.message, "HTTP 413"); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); +}); + +test("event replay incorporates an earlier accepted upload after a permanent retry rejection", async () => { + const f = await fixture(undefined, { + putFileContent: async () => { + throw new PermanentSyncError( + "HTTP 413 after proxy reconfiguration" + ); + }, + getDocumentVersionContent: async () => bytes("local") + }); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert(pending); + const { requestId } = pending.request; + await f.internals.finishPending(); + await assert.rejects(f.syncer.waitUntilFinished(), /413/); + f.database.state.lastSeenUpdateId = 1; + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "local") + } + ] + }); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + assert.equal(f.database.state.pending, undefined); + assert.equal(await f.internals.preparePush(), false); + await f.syncer.waitUntilFinished(); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); +}); + +test("rejected manifests can be replaced after editing the local namespace", async () => { + let reject = true; + const f = await fixture( + {}, + { + pushFileManifest: async () => { + if (reject) throw new PermanentSyncError("HTTP 413"); + return { type: "Accepted", fileManifestId: 3 }; + } + } + ); + f.database.state.fileManifest.entries = { a: "old.md" }; + f.database.state.pending = { + type: "fileManifest", + request: { + requestId: "request", + parentFileManifestId: 1, + entries: { a: "new.md" } + } + }; + f.database.state.local = { a: "new.md" }; + f.database.state.documents.a = { + materialized: false, + base: { ...head("a", 1, "x"), hash: "hash" } + }; + await f.database.save(); + await f.internals.finishPending(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + f.database.state.local = {}; + reject = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(f.database.state.fileManifest.entries, {}); +}); + +test("offline delete/recreate identities survive restarting before any network connection", async () => { + const key = JSON.stringify(["http://offline.test", "test"]); + const initial = emptyState(key); + initial.initialized = true; + initial.local.a = "a.md"; + initial.documents.a = { materialized: true }; + initial.fileManifest = { fileManifestId: 1, entries: { a: "a.md" } }; + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("same")); + const persistence = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + const first = await SyncClient.create({ fs: disk, persistence }); + await disk.userDelete("a.md"); + await first.syncLocallyDeletedFile("a.md"); + await disk.userWrite("a.md", bytes("same")); + await first.syncLocallyCreatedFile("a.md"); + await first.destroy(); + const second = await SyncClient.create({ fs: disk, persistence }); + try { + const internals = second as unknown as { + syncer: { scan(): Promise }; + database: Database; + }; + await internals.syncer.scan(); + assert.equal(internals.database.state.local.a, undefined); + assert.equal( + Object.values(internals.database.state.local).filter( + (path) => path === "a.md" + ).length, + 1 + ); + } finally { + await second.destroy(); + } +}); + +test("offline notifications bind an otherwise empty client to its vault", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("local")); + const persistence = new MemoryPersistence({ + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + const client = await SyncClient.create({ fs: disk, persistence }); + await client.syncLocallyCreatedFile("a.md"); + await assert.rejects( + client.setSettings({ vaultName: "other" }), + /separate state store/ + ); + await client.destroy(); + const stored = persistence.snapshot(); + stored.settings!.vaultName = "other"; + await persistence.save(stored); + await assert.rejects( + SyncClient.create({ fs: disk, persistence }), + /another vault/ + ); +}); + +test("unignoring a deleted file resumes the deferred remote deletion without resurrection", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert.equal(f.disk.userFiles().size, 0); + assert.deepEqual(f.database.state.local, {}); +}); + +test("an ignored file does not move when a remote rename targets its path", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "c.md", b: "a.md" } + }); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes("A")], + ["b.md", bytes("B")] + ]) + ); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes("A")], + ["b.md", bytes("B")] + ]) + ); +}); + +for (const kind of ["path", "content"] as const) { + test(`a remote ${kind} change that was reverted before observation preserves the local edit`, async () => { + const f = await fixture( + { "a.md": "local" }, + { + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes(vaultUpdateId === 2 ? "temporary" : "base") + } + ); + f.database.state.lastSeenUpdateId = 1; + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + if (kind === "path") { + await f.disk.userRename("a.md", "c.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "c.md" + }); + } + await f.internals.incorporateEventBatch({ + headEventId: 3, + events: [2, 3].map((id) => + kind === "path" + ? { + eventId: id, + requestId: String(id), + type: "fileManifest", + fileManifest: { + fileManifestId: id, + entries: { a: id === 2 ? "b.md" : "a.md" } + } + } + : { + eventId: id, + requestId: String(id), + type: "content", + document: head( + "a", + id, + id === 2 ? "temporary" : "base" + ) + } + ) + }); + assert.equal( + f.database.state.local.a, + kind === "path" ? "c.md" : "a.md" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))!.content + ).toString(), + "local" + ); + assert.equal(f.database.state.lastSeenUpdateId, 3); + }); +} + +test("concurrent settings updates cannot bind the engine to a different vault than its settings", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence(); + const client = await SyncClient.create({ fs: disk, persistence }); + try { + await Promise.all([ + client.setSettings({ vaultName: "changed" }), + client.setSettings({ maxFileSizeMB: 20 }) + ]); + const stored = persistence.snapshot(); + assert.equal( + stored.database!.vaultKey, + JSON.stringify([ + stored.settings!.remoteUri, + stored.settings!.vaultName + ]) + ); + assert.equal(stored.settings!.maxFileSizeMB, 20); + } finally { + await client.destroy(); + } +}); + +test("destroy waits for an in-progress offline notification to become durable", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence(); + const client = await SyncClient.create({ fs: disk, persistence }); + await disk.userWrite("a.md", bytes("local")); + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + persistence.boundary = async (label) => { + if (label === "before:save") { + reached.resolve(); + await release.promise; + } + }; + const notification = client.syncLocallyCreatedFile("a.md"); + await reached.promise; + let destroyed = false; + const destroying = client.destroy().then(() => { + destroyed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(destroyed, false); + release.resolve(); + await Promise.all([notification, destroying]); +}); + +test("ignoring a locally renamed file defers its move without forgetting the local rename", async () => { + const f = await fixture(); + f.database.state.documents.a.base = { + ...head("a", 1, "local"), + hash: f.database.state.documents.a.observedHash! + }; + await f.disk.userRename("a.md", "ignored.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "ignored.md" + }); + await f.settings.setSettings({ ignorePatterns: ["ignored.md"] }); + await f.internals.scan(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "a.md" } + }); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert.equal(f.database.state.local.a, "ignored.md"); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("ignored.md"))!.content + ).toString(), + "local" + ); +}); + +test("a notified deletion of an ignored file does not delete its server membership", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.disk.userDelete("a.md"); + await f.syncer.syncLocallyDeletedFile("a.md"); + await f.internals.scan(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + assert.equal(f.database.state.local.a, undefined); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert(await f.internals.preparePush()); + assert.equal(f.database.state.pending?.type, "fileManifest"); +}); + +test("an incoming ignored document cannot force an existing ignored file to move", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md", "*conflict*"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "b.md", other: "a.md" } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); + assert.equal( + new Set(Object.values(f.database.state.local)).size, + Object.keys(f.database.state.local).length + ); +}); + +test("rejected content is reported to callers and an explicit retry clears the rejection", async () => { + const f = await fixture(undefined, { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + await f.internals.finishPending(); + await assert.rejects(f.syncer.waitUntilFinished(), /413/); + assert(f.syncer.isBusy); + await f.syncer.retryRejectedRequests(); + assert(await f.internals.preparePush()); +}); + +test("notification saves cannot roll back an uncertain engine commit, even if the first reload fails", async () => { + const key = JSON.stringify(["http://offline.test", "test"]); + const initial = emptyState(key); + initial.initialized = true; + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + let failLoad = false; + const persistence = { + load: async () => { + if (failLoad) { + failLoad = false; + throw new Error("reload failed"); + } + return store.load(); + }, + save: async (state: Parameters[0]) => + store.save(state) + }; + const client = await SyncClient.create({ fs: disk, persistence }); + try { + const internals = client as unknown as { + database: Database; + syncer: Syncer; + }; + const next = structuredClone(initial); + next.lastSeenUpdateId = 2; + store.boundary = (label) => { + if (label === "durable:save") { + store.boundary = () => {}; + throw new Error("durable save failed"); + } + }; + await assert.rejects( + internals.database.commit(next), + /durable save failed/ + ); + await disk.userWrite("a.md", bytes("new")); + failLoad = true; + await assert.rejects( + client.syncLocallyCreatedFile("a.md"), + /reload failed/ + ); + await internals.syncer.flushLocalChanges(); + assert.equal(store.snapshot().database!.lastSeenUpdateId, 2); + await internals.database.recoverPersistence(); + assert.equal(internals.database.state.lastSeenUpdateId, 2); + } finally { + await client.destroy(); + } +}); + +test( + "a settings listener can await client shutdown without deadlocking the lifecycle lock", + { timeout: 1_000 }, + async () => { + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: new MemoryPersistence() + }); + let notified = false; + client.onSettingsChanged.add(async () => { + await client.destroy(); + notified = true; + }); + await client.setSettings({ maxFileSizeMB: 20 }); + assert(notified); + } +); diff --git a/frontend/test-support/event-delivery.test.ts b/frontend/test-support/event-delivery.test.ts new file mode 100644 index 00000000..012e24fd --- /dev/null +++ b/frontend/test-support/event-delivery.test.ts @@ -0,0 +1,175 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import type { EventBatch } from "../sync-client/src/services/types/EventBatch"; +import type { EventRecord } from "../sync-client/src/services/types/EventRecord"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +for (const [count, delivery] of [ + [1, "before the loop"], + [9, "before the loop"], + [3, "during the first batch"] +] as const) { + test( + `${count} WS hints delivered ${delivery} catch up through HTTP without polling`, + { timeout: 5_000 }, + async () => { + const logger = new Logger(); + const initial = emptyState("event-delivery"); + initial.initialized = true; + const persistence = new MemoryPersistence({ database: initial }); + const database = new Database( + initial, + async (next) => persistence.save({ database: next }), + initial.vaultKey, + async () => + (await persistence.load()).database as StoredDatabase + ); + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + assert.equal(settings.getSettings().syncIntervalMs, 0); + const disk = new MemoryDisk(); + const initializing = Promise.withResolvers(); + const initialized = Promise.withResolvers(); + const firstBatch = Promise.withResolvers(); + const continueBatch = Promise.withResolvers(); + let heldBatch = false; + const applied: number[] = []; + persistence.boundary = async (label) => { + if (label !== "durable:save") return; + const cursor = + persistence.snapshot().database!.lastSeenUpdateId; + if (cursor && applied.at(-1) !== cursor) applied.push(cursor); + if ( + delivery === "during the first batch" && + cursor === 1 && + !heldBatch + ) { + heldBatch = true; + firstBatch.resolve(); + await continueBatch.promise; + } + }; + const config = { + initialize: async () => { + initializing.resolve(); + await initialized.promise; + }, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const events: EventRecord[] = Array.from( + { length: count }, + (_, index) => ({ + eventId: index + 1, + requestId: `manifest-${index + 1}`, + type: "fileManifest", + fileManifest: { fileManifestId: index + 1, entries: {} } + }) + ); + let available: number = delivery === "before the loop" ? count : 1; + const fetchedAfter: number[] = []; + const service = { + events: async (after: number): Promise => { + fetchedAfter.push(after); + return { + headEventId: available, + events: events.filter( + (event) => + event.eventId > after && + event.eventId <= available + ) + }; + } + } as unknown as SyncService; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => unknown + >() + }; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service, + websocket as unknown as WebSocketManager, + new FileOperations(disk.session(), database, config), + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(1024) + ); + const deliver = async (event: EventRecord) => { + await websocket.onRemoteVaultUpdateReceived.triggerAsync({ + headEventId: event.eventId, + // Payloads are deliberately inconsistent with HTTP. The socket + // only wakes the loop; it cannot supply reconciliation state. + events: [ + { + ...event, + type: "fileManifest", + fileManifest: { fileManifestId: 999, entries: {} } + } + ] + }); + }; + try { + syncer.start(); + await initializing.promise; + if (delivery === "before the loop") { + for (const event of events) await deliver(event); + initialized.resolve(); + } else { + await deliver(events[0]); + initialized.resolve(); + await firstBatch.promise; + available = count; + for (const event of events.slice(1)) await deliver(event); + continueBatch.resolve(); + } + await syncer.waitUntilFinished(); + assert.equal( + database.state.lastSeenUpdateId, + count, + "Client became idle with already-delivered events unapplied" + ); + assert.deepEqual( + applied, + delivery === "before the loop" ? [count] : [1, count], + "Each HTTP catchup commits its final cursor" + ); + assert.deepEqual( + fetchedAfter, + delivery === "before the loop" ? [0] : [0, 1] + ); + assert.equal(database.state.fileManifest.fileManifestId, count); + assert.equal(database.state.pending, undefined); + assert.equal(syncer.isBusy, false); + } finally { + initialized.resolve(); + continueBatch.resolve(); + await syncer.stop(); + } + } + ); +} diff --git a/frontend/test-support/file-application.test.ts b/frontend/test-support/file-application.test.ts new file mode 100644 index 00000000..957fe62f --- /dev/null +++ b/frontend/test-support/file-application.test.ts @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes } from "./sync-fixture"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { LocalChangesDuringReconciliation } from "../sync-client/src/errors/errors"; + +for (const equal of [false, true]) + test(`swaps preserve identities with ${equal ? "equal" : "distinct"} content and save once`, async () => { + const f = await fixture({ "a.md": "A", "b.md": equal ? "A" : "B" }); + let saves = 0; + f.persistence.boundary = (label) => { + if (label === "before:save") saves++; + }; + const next = structuredClone(f.database.state); + next.local = { a: "b.md", b: "a.md" }; + await f.files.apply(next); + assert.deepEqual(f.database.state.local, next.local); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes(equal ? "A" : "B")], + ["b.md", bytes("A")] + ]) + ); + assert.equal(saves, 1); + assert.deepEqual(await f.disk.listFilesRecursively(), ["a.md", "b.md"]); + }); + +for (const direction of ["file-to-directory", "directory-to-file"]) + test(direction, async () => { + const f = await fixture( + direction === "file-to-directory" + ? { folder: "A", "b.md": "B" } + : { "a.md": "A", "folder/b.md": "B" } + ); + const ids = Object.keys(f.database.state.local); + const next = structuredClone(f.database.state); + next.local = + direction === "file-to-directory" + ? { [ids[0]]: "folder/a.md", [ids[1]]: "b.md" } + : { [ids[0]]: "folder", [ids[1]]: "other.md" }; + const expected = structuredClone(next.local); + await f.files.apply(next); + assert.deepEqual(f.database.state.local, expected); + assert.deepEqual( + new Set(f.disk.userFiles().keys()), + new Set(Object.values(f.database.state.local)) + ); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((value) => Buffer.from(value).toString()) + .sort(), + ["A", "B"] + ); + }); + +test("an unexpected occupant is preserved under a visible conflict name", async () => { + const f = await fixture(); + await f.disk.userWrite("remote.md", bytes("unexpected")); + const next = structuredClone(f.database.state); + next.local.a = "remote.md"; + await f.files.apply(next); + assert.equal(f.database.state.local.a, "remote.md"); + assert.equal(f.disk.userFiles().size, 2); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((value) => Buffer.from(value).toString()) + .sort(), + ["local", "unexpected"] + ); + assert.equal(Object.keys(f.database.state.local).length, 2); +}); + +test("content edited after planning is merged with the incoming snapshot", async () => { + const f = await fixture({ "a.md": "base\n" }); + const expected = (await f.files.snapshot("a.md"))!; + await f.disk.userWrite("a.md", bytes("base\nlocal\n")); + await f.files.apply(structuredClone(f.database.state), { + a: { + expected, + replacement: await toStoredSnapshot({ + content: bytes("remote\nbase\n") + }) + } + }); + assert.deepEqual( + f.disk.userFiles().get("a.md"), + bytes("remote\nbase\nlocal\n") + ); +}); + +test("deleting the source after planning does not resurrect it", async () => { + const f = await fixture(); + const next = structuredClone(f.database.state); + next.local.a = "remote.md"; + await f.disk.userDelete("a.md"); + await f.files.apply(next, { + a: { replacement: await toStoredSnapshot({ content: bytes("remote") }) } + }); + assert.deepEqual(f.disk.userFiles(), new Map()); + assert.deepEqual(f.database.state.local, {}); +}); + +for (const protect of ["ignore", "size"] as const) + test(`protect ${protect} at application time`, async () => { + const f = await fixture(); + const next = structuredClone(f.database.state); + next.local.a = "remote.md"; + if (protect === "ignore") + await f.settings.setSetting("ignorePatterns", ["a.md"]); + else await f.settings.setSetting("maxFileSizeMB", 0); + await f.files.apply(next, { + a: { + replacement: await toStoredSnapshot({ + content: bytes("remote") + }) + } + }); + assert.deepEqual( + f.disk.userFiles(), + new Map([["a.md", bytes("local")]]) + ); + assert.equal(f.database.state.local.a, "a.md"); + assert.equal(f.database.state.excluded?.a, "a.md"); + }); + +for (const when of ["planning", "installed"]) + test(`notified move during ${when} is attributed to the right identity on rescan`, async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let changed = false; + f.disk.boundary = async (label) => { + if ( + changed || + label !== + (when === "planning" + ? "read:stat:a.md" + : "visible:rename:a.md->b.md") + ) + return; + changed = true; + const oldPath = when === "planning" ? "a.md" : "b.md"; + await f.disk.userRename(oldPath, "editor.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath, + relativePath: "editor.md" + }); + }; + const next = structuredClone(f.database.state); + next.local = { a: "b.md", b: "a.md" }; + if (when === "planning") + await assert.rejects( + f.files.apply(next), + LocalChangesDuringReconciliation + ); + else await f.files.apply(next); + assert(changed); + await f.internals.scan(); + assert.equal(f.database.state.local.a, "editor.md"); + assert.deepEqual( + f.disk.userFiles().get(f.database.state.local.a), + bytes("A") + ); + assert.deepEqual( + f.disk.userFiles().get(f.database.state.local.b), + bytes("B") + ); + }); + +for (const action of ["move", "delete"] as const) + test(`notified ${action}/recreate between file operations does not repurpose the replacement file`, async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + let changed = false; + f.disk.boundary = async (label) => { + if (changed || label !== "visible:rename:a.md->remote-a.md") return; + changed = true; + if (action === "move") { + await f.disk.userRename("b.md", "private.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "private.md" + }); + } else { + await f.disk.userDelete("b.md"); + await f.syncer.syncLocallyDeletedFile("b.md"); + } + await f.disk.userWrite("b.md", bytes("replacement")); + await f.syncer.syncLocallyCreatedFile("b.md"); + }; + const next = structuredClone(f.database.state); + next.local = { a: "remote-a.md", b: "remote-b.md" }; + await f.files.apply(next); + assert(changed); + await f.internals.scan(); + assert.deepEqual(f.disk.userFiles().get("b.md"), bytes("replacement")); + assert.equal(f.disk.userFiles().has("remote-b.md"), false); + assert.equal( + f.database.state.local.b, + action === "move" ? "private.md" : undefined + ); + if (action === "move") + assert.deepEqual(f.disk.userFiles().get("private.md"), bytes("B")); + }); + +test("a file and nested directory exchange their exact paths", async () => { + const f = await fixture({ a: "A", "b/deep/child": "B" }); + const [a, b] = Object.keys(f.database.state.local); + const next = structuredClone(f.database.state); + next.local = { [a]: "b", [b]: "a/deep/child" }; + await f.files.apply(next); + assert.deepEqual(f.database.state.local, { [a]: "b", [b]: "a/deep/child" }); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["b", bytes("A")], + ["a/deep/child", bytes("B")] + ]) + ); +}); diff --git a/frontend/test-support/harness.test.ts b/frontend/test-support/harness.test.ts new file mode 100644 index 00000000..e4604b4b --- /dev/null +++ b/frontend/test-support/harness.test.ts @@ -0,0 +1,316 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { NetworkFaults, classifyRequest } from "./network"; +import { + ContentLedger, + assertManifest, + assertSameFiles, + pathDecision +} from "./oracles"; +import { Random } from "./random"; +import { TestRunner } from "../deterministic-tests/src/test-runner"; +import type { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { ManagedWebSocketFactory } from "../deterministic-tests/src/managed-websocket"; + +const bytes = (text: string) => new TextEncoder().encode(text); + +test("consistency oracle rejects self comparison, divergent paths and invalid UTF-8 bytes", () => { + const first = new Map([["a.bin", new Uint8Array([0xff])]]); + assert.throws(() => assertSameFiles([first, first]), /itself/); + assert.throws(() => + assertSameFiles([first, new Map([["a.bin", new Uint8Array([0xfe])]])]) + ); + assert.throws(() => assertSameFiles([first, new Map()])); + assertSameFiles([first, structuredClone(first)]); +}); + +test("ledger catches shared loss, duplicated bytes, and unrelated loss in delete workloads", () => { + const ledger = new ContentLedger(); + ledger.add("first-marker"); + ledger.add("second-marker"); + ledger.removedBy("first-marker", "explicit delete of doc A"); + assert.throws( + () => ledger.assertPreserved(new Map()), + /second-marker.*lost/ + ); + assert.throws( + () => + ledger.assertPreserved( + new Map([["a", bytes("second-marker second-marker")]]) + ), + /duplicated/ + ); + ledger.assertPreserved(new Map([["a", bytes("second-marker")]])); +}); + +test("memory adapter enforces v4 operations and snapshot ownership", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("folder/a", bytes("A")); + const snap = (await disk.readSnapshot("folder/a"))!; + snap.content[0] = 0; + assert.equal( + Buffer.from((await disk.readSnapshot("folder/a"))!.content).toString(), + "A" + ); + await assert.rejects( + disk.write("folder/a", { content: bytes("B") }), + /exists/ + ); + await disk.userWrite("folder/b", bytes("B")); + await assert.rejects(disk.rename("folder/a", "folder/b"), /exists/); + await assert.rejects(disk.delete("folder/a"), /regular file/); + await assert.rejects(disk.delete("folder"), /regular file/); + await assert.rejects(disk.readSnapshot("../outside"), /Unsafe/); + await assert.rejects(disk.stat("folder/a/child"), /ancestor/); + await assert.rejects(disk.readSnapshot("folder"), /directory/); +}); + +for (const phase of ["before", "visible", "durable"] as const) { + for (const powerLoss of [false, true]) { + test(`${powerLoss ? "power loss" : "process death"} at ${phase} rename boundary`, async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a", bytes("A")); + const session = disk.session(); + let fired = false; + disk.boundary = (label) => { + if (label === `${phase}:rename:a->b`) { + fired = true; + disk.crash(powerLoss); + throw new Error("crash"); + } + }; + await assert.rejects(session.rename("a", "b"), /crash/); + assert(fired); + disk.boundary = () => {}; + const moved = + phase === "durable" || (phase === "visible" && !powerLoss); + assert.deepEqual([...disk.userFiles().keys()], [moved ? "b" : "a"]); + assert.throws( + () => session.write("stale", { content: bytes("bad") }), + /crashed/ + ); + }); + } +} + +test("persistence has deep ownership and both old/new uncertain save outcomes", async () => { + for (const phase of ["before", "durable"]) { + const store = new MemoryPersistence({ settings: { vaultName: "old" } }); + const loaded = await store.load(); + loaded.settings!.vaultName = "mutated"; + assert.equal(store.snapshot().settings!.vaultName, "old"); + store.boundary = (label) => { + if (label === `${phase}:save`) throw new Error("save failed"); + }; + await assert.rejects( + store.save({ settings: { vaultName: "new" } }), + /save failed/ + ); + assert.equal( + store.snapshot().settings!.vaultName, + phase === "before" ? "old" : "new" + ); + } +}); + +test("CAS hooks match v4 create, update and manifest and preserve retry payload", async () => { + for (const kind of ["create", "content", "manifest"] as const) { + const net = new NetworkFaults(); + const url = `http://test/vaults/test/${kind === "manifest" ? "file-manifest" : "documents/uuid"}`; + const body = { + requestId: "same", + parentVersionId: kind === "create" ? null : 3 + }; + const init = { method: "PUT", body: JSON.stringify(body) }; + assert.equal(classifyRequest(url, init)?.kind, kind); + let committed = 0; + const wrapped = net.wrap(async () => { + committed++; + return new Response('{"type":"Accepted"}'); + }); + net.arm(kind); + await assert.rejects(wrapped(url, init), /Injected after-commit/); + await net.wait(); + assert.throws(() => net.assertConsumed(), /never retried/); + assert.equal(committed, 1); + await wrapped(url, init); + net.assertConsumed(); + await assert.rejects( + wrapped(url, { + ...init, + body: JSON.stringify({ ...body, changed: true }) + }), + /Retry changed/ + ); + } + assert.equal( + classifyRequest("http://test/documents", { method: "POST" }), + undefined + ); +}); + +test("unconsumed and before-send faults cannot silently pass", async () => { + const net = new NetworkFaults(); + net.arm("manifest", "before"); + assert.throws(() => net.assertConsumed(), /never fired/); + let called = false; + await assert.rejects( + net.wrap(async () => { + called = true; + return new Response(); + })("http://test/file-manifest", { + method: "PUT", + body: '{"requestId":"r"}' + }) + ); + assert(!called); + assert.throws(() => net.assertConsumed(), /never retried/); + await net.wrap(async () => new Response())("http://test/file-manifest", { + method: "PUT", + body: '{"requestId":"r"}' + }); + net.assertConsumed(); +}); + +test("HTTP observation checkpoint blocks catchup, resumes, and respects abort", async () => { + const network = new NetworkFaults(); + const wrapped = network.wrap(async () => new Response("{}")); + network.pauseObservation(); + let delivered = false; + const request = wrapped("http://test/events-since?after=0").then(() => { + delivered = true; + }); + await network.waitForObservation(); + assert(!delivered, "Catchup bypassed the observation checkpoint"); + network.resumeObservation(); + await request; + network.assertConsumed(); + network.pauseObservation(); + const controller = new AbortController(); + const interrupted = wrapped("http://test/vault-snapshot", { + signal: controller.signal + }); + const rejected = assert.rejects(interrupted, /abort/i); + await network.waitForObservation(); + controller.abort(); + await rejected; + network.resumeObservation(); + network.assertConsumed(); +}); + +test("independent path decision truth table covers create, delete and rename", () => { + for (const base of [undefined, "a", "b"]) + for (const local of [undefined, "a", "b"]) + for (const remote of [undefined, "a", "b"]) { + const result = pathDecision(base, local, remote); + assert.equal(result, remote !== base ? remote : local); + } + assertManifest({ a: "folder/é.md", b: "other.bin" }); + for (const entries of [ + { a: "A", b: "a" }, + { a: "é", b: "e\u0301" }, + { a: "x", b: "x/y" } + ]) + assert.throws(() => assertManifest(entries)); +}); + +test("seeded actions are repeatable, including seed zero", () => { + for (const seed of [0, 1, 12345, 0xffffffff]) { + const first = new Random(seed); + const second = new Random(seed); + assert.deepEqual( + Array.from({ length: 100 }, () => first.int(100)), + Array.from({ length: 100 }, () => second.int(100)) + ); + } +}); + +test("runner cleanup errors fail an otherwise successful test", async () => { + const server = { + isRunning: () => true, + resume: () => {} + } as unknown as ServerControl; + const runner = new TestRunner( + server, + new Logger(), + "unused", + "http://unused" + ); + let cleaned = 0; + Object.assign(runner, { + initializeAgents: async () => {}, + agents: [ + { + clientId: 0, + database: () => ({ evidence: "before cleanup" }), + disk: { image: () => ({ bytes: [0, 255] }) }, + network: { requests: [{ body: { requestId: "original" } }] }, + cleanup: async () => { + cleaned++; + throw new Error("late background error"); + } + } + ] + }); + const result = await runner.runTest("cleanup-probe", { + clients: 2, + steps: [] + }); + assert.equal(result.success, false); + assert.match(result.error!, /late background error/); + assert.equal(cleaned, 1, "Failed cleanup was run twice"); + assert.deepEqual(result.diagnostics, [ + { + client: 0, + database: { evidence: "before cleanup" }, + disk: { bytes: [0, 255] }, + requests: [{ body: { requestId: "original" } }] + } + ]); +}); + +test("disposed harness rejects late websocket reconnects", async () => { + const factory = new ManagedWebSocketFactory(); + const Socket = factory.constructorFn; + await factory.finish(); + assert.throws(() => new Socket("ws://unused"), /after harness cleanup/); +}); + +test("runner preserves primary failure and cleanup diagnostics", async () => { + const server = { + isRunning: () => true, + resume: () => {} + } as unknown as ServerControl; + const runner = new TestRunner( + server, + new Logger(), + "unused", + "http://unused" + ); + Object.assign(runner, { + initializeAgents: async () => {}, + executeStep: async () => { + throw new Error("primary assertion failed"); + }, + agents: [ + { + clientId: 0, + database: () => ({}), + disk: { image: () => ({}) }, + network: { requests: [] }, + cleanup: async () => { + throw new Error("cleanup failed too"); + } + } + ] + }); + const result = await runner.runTest("cleanup-probe", { + clients: 2, + steps: [{ type: "barrier" }] + }); + assert.equal(result.success, false); + assert.match(result.error!, /primary assertion failed/); + assert.match(result.error!, /cleanup failed too/); +}); diff --git a/frontend/test-support/listener-crash-child.ts b/frontend/test-support/listener-crash-child.ts new file mode 100644 index 00000000..bd703090 --- /dev/null +++ b/frontend/test-support/listener-crash-child.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { fixture } from "./sync-fixture"; +(async () => { + const f = await fixture( + {}, + { + events: async () => ({ headEventId: 0, events: [] }) + } + ); + let delivered = 0; + f.syncer.onRemainingOperationsCountChanged.add(() => { + throw new Error("observer failure"); + }); + f.syncer.onRemainingOperationsCountChanged.add(async () => { + throw new Error("async observer failure"); + }); + f.syncer.onRemainingOperationsCountChanged.add(() => { + delivered++; + }); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + const before = delivered; + f.syncer.wake(); + await f.syncer.waitUntilFinished(); + assert(delivered > before, "Background sync must remain usable"); + } finally { + await f.syncer.stop(); + } +})(); diff --git a/frontend/test-support/network.ts b/frontend/test-support/network.ts new file mode 100644 index 00000000..7df61083 --- /dev/null +++ b/frontend/test-support/network.ts @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; + +export type RequestKind = "create" | "content" | "manifest"; +export interface RequestRecord { + kind: RequestKind; + method: string; + url: string; + body: Record; + status?: number; +} + +interface InjectedFault { + kind: RequestKind; + point: "before" | "after"; + fired: boolean; + interrupted?: RequestRecord; + retried: boolean; + reached: Promise; + resolve: () => void; +} + +export function classifyRequest( + input: RequestInfo | URL, + init?: RequestInit +): RequestRecord | undefined { + const url = new URL(input instanceof Request ? input.url : String(input)); + const method = ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(); + if (method !== "PUT") return undefined; + // The engine passes JSON strings. Fail loudly if that contract changes. + if (!/\/(documents\/[^/]+|file-manifest)\/?$/.test(url.pathname)) + return undefined; + assert.equal( + typeof init?.body, + "string", + "CAS request body must be observable JSON" + ); + const body = JSON.parse(init!.body as string) as Record; + const kind = /\/file-manifest\/?$/.test(url.pathname) + ? "manifest" + : body.parentVersionId === null + ? "create" + : "content"; + assert.equal( + typeof body.requestId, + "string", + "CAS request missing requestId" + ); + return { kind, method, url: url.href, body }; +} + +export class NetworkFaults { + public readonly requests: RequestRecord[] = []; + private observation?: { + released: boolean; + fired: boolean; + held: Promise; + release: () => void; + reached: Promise; + reach: () => void; + }; + public pauseObservation(): void { + assert( + !this.observation || this.observation.released, + "Observation already paused" + ); + let release!: () => void, reach!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const reached = new Promise((resolve) => { + reach = resolve; + }); + this.observation = { + released: false, + fired: false, + held, + release, + reached, + reach + }; + } + public async waitForObservation(): Promise { + assert(this.observation, "No observation checkpoint armed"); + await this.observation.reached; + } + public resumeObservation(): void { + if (this.observation) { + this.observation.released = true; + this.observation.release(); + } + } + private async observationCheckpoint( + signal?: AbortSignal | null + ): Promise { + const gate = this.observation; + if (!gate || gate.released) return; + gate.fired = true; + gate.reach(); + signal?.throwIfAborted(); + let abort = () => {}; + try { + await Promise.race([ + gate.held, + new Promise((_, reject) => { + abort = () => + reject( + signal?.reason ?? new Error("Observation aborted") + ); + signal?.addEventListener("abort", abort, { once: true }); + }) + ]); + } finally { + signal?.removeEventListener("abort", abort); + } + } + private armed?: InjectedFault; + private readonly faults: InjectedFault[] = []; + public isExpectedFailure(message: string): boolean { + return this.faults.some( + (fault) => + fault.fired && + message.includes( + `Injected ${fault.point}-commit ${fault.kind} network failure` + ) + ); + } + public arm(kind: RequestKind, point: "before" | "after" = "after"): void { + assert( + !this.armed || (this.armed.fired && this.armed.retried), + "Previous network fault was not exercised and retried" + ); + let resolve!: () => void; + const reached = new Promise((r) => { + resolve = r; + }); + this.armed = { + kind, + point, + fired: false, + retried: false, + reached, + resolve + }; + this.faults.push(this.armed); + } + public async wait(): Promise { + assert(this.armed, "No network fault armed"); + await this.armed.reached; + } + public assertConsumed(): void { + assert( + !this.observation || + (this.observation.fired && this.observation.released), + "Observation checkpoint was not exercised and released" + ); + for (const fault of this.faults) { + assert(fault.fired, "Armed network fault never fired"); + assert(fault.retried, "Interrupted CAS request was never retried"); + } + } + public wrap(fetch: typeof globalThis.fetch): typeof globalThis.fetch { + return async (input, init) => { + const url = new URL( + input instanceof Request ? input.url : String(input) + ); + const observing = + ( + init?.method ?? + (input instanceof Request ? input.method : "GET") + ).toUpperCase() === "GET" && !url.pathname.endsWith("/ping"); + const signal = + init?.signal ?? + (input instanceof Request ? input.signal : undefined); + if (observing) await this.observationCheckpoint(signal); + const record = classifyRequest(input, init); + if (record) { + // A client serializes its persisted CAS requests. Its next CAS + // after a lost response must replay the interrupted operation, + // even if it has since observed newer remote state. + const interrupted = this.armed?.interrupted; + if (interrupted && !this.armed!.retried) { + assert.equal( + record.url, + interrupted.url, + "Retry changed target" + ); + assert.deepEqual( + record.body, + interrupted.body, + "Retry changed payload/request identity" + ); + this.armed!.retried = true; + } + const previous = this.requests.find( + (r) => r.body.requestId === record.body.requestId + ); + if (previous) { + assert.equal( + record.url, + previous.url, + "Retry changed target" + ); + assert.deepEqual( + record.body, + previous.body, + "Retry changed payload/request identity" + ); + } + this.requests.push(record); + } + const fault = this.armed; + const matches = + record && fault && !fault.fired && fault.kind === record.kind; + const fail = () => { + fault!.interrupted = structuredClone(record!); + fault!.fired = true; + fault!.resolve(); + throw new TypeError( + `Injected ${fault!.point}-commit ${fault!.kind} network failure` + ); + }; + if (matches && fault.point === "before") fail(); + const response = await fetch(input, init); + if (observing) await this.observationCheckpoint(signal); + if (record) record.status = response.status; + // Only drop accepted replies, never confuse a rejected CAS with commit. + if ( + matches && + !fault.fired && + fault.point === "after" && + response.ok && + ((await response.clone().json()) as { type?: string }).type === + "Accepted" + ) { + await response.body?.cancel(); + fail(); + } + return response; + }; + } +} diff --git a/frontend/test-support/observer.test.ts b/frontend/test-support/observer.test.ts new file mode 100644 index 00000000..8fb2d5e6 --- /dev/null +++ b/frontend/test-support/observer.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +test("a status observer exception must not terminate a background sync process", () => { + const result = spawnSync( + process.execPath, + [ + resolve(__dirname, "../node_modules/tsx/dist/cli.mjs"), + resolve(__dirname, "listener-crash-child.ts") + ], + { encoding: "utf8", timeout: 10000 } + ); + assert.equal(result.status, 0, result.stderr); +}); diff --git a/frontend/test-support/oracle-regressions.test.ts b/frontend/test-support/oracle-regressions.test.ts new file mode 100644 index 00000000..fc2916a4 --- /dev/null +++ b/frontend/test-support/oracle-regressions.test.ts @@ -0,0 +1,351 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { test } from "node:test"; +import type { StoredDatabase } from "sync-client"; +import { assertCanonical } from "./canonical"; +import { assertManifest } from "./oracles"; +import { NetworkFaults } from "./network"; +import { MemoryDisk } from "./storage"; +import { DeterministicAgent } from "../deterministic-tests/src/deterministic-agent"; +import { AssertableState } from "../deterministic-tests/src/utils/assertable-state"; +import type { ExpectedDocument } from "../deterministic-tests/src/test-definition"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const id = "00000000-0000-4000-8000-000000000001"; + +test("canonical checks reject repeated clients before reading snapshots", async () => { + const client = { files: () => new Map(), database: () => ({}) }; + await assert.rejects( + assertCanonical([client, client], "http://unused", "unused"), + /itself/ + ); +}); + +test("canonical checks independently verify both persisted content hashes", async (context) => { + const content = bytes("\ufeffexact bytes\r\n"); + const hash = createHash("sha256").update(content).digest("hex"); + const document = { + documentId: id, + vaultUpdateId: 1, + contentSize: content.length, + userId: "user", + deviceId: "device", + updatedDate: "2026-09-20T00:00:00Z" + }; + const manifest = { fileManifestId: 2, entries: { [id]: "a.md" } }; + const state: Partial = { + initialized: true, + local: manifest.entries, + fileManifest: manifest, + lastSeenUpdateId: 2, + documents: { + [id]: { + materialized: true, + observedHash: hash, + base: { ...document, hash } + } + } + }; + context.mock.method( + globalThis, + "fetch", + async (url: string) => + new Response( + JSON.stringify( + url.endsWith("/vault-snapshot") + ? { + headEventId: 2, + fileManifest: manifest, + documents: [document] + } + : url.includes("/events-since") + ? { + headEventId: 2, + events: [ + { eventId: 1, requestId: "create" }, + { eventId: 2, requestId: "manifest" } + ] + } + : { + contentBase64: + Buffer.from(content).toString("base64") + } + ) + ) + ); + const client = (database = state) => ({ + files: () => new Map([["a.md", content.slice()]]), + database: () => structuredClone(database) + }); + await assertCanonical([client(), client()], "http://test", "token"); + for (const field of ["base", "observed"] as const) { + const corrupt = structuredClone(state); + if (field === "base") corrupt.documents![id].base!.hash = "wrong"; + else corrupt.documents![id].observedHash = "wrong"; + await assert.rejects( + assertCanonical( + [client(), client(corrupt)], + "http://test", + "token" + ), + new RegExp(`Wrong ${field} hash`) + ); + } +}); + +test("lost responses require an identical retry, including request identity and target", async () => { + for (const point of ["before", "after"] as const) { + const faults = new NetworkFaults(); + let sent = 0; + const send = faults.wrap(async () => { + sent++; + return new Response('{"type":"Accepted"}'); + }); + const url = "http://test/documents/a"; + const body = { + requestId: "original", + parentVersionId: 3, + content: { type: "Snapshot", value: "YQ==" } + }; + const init = { method: "PUT", body: JSON.stringify(body) }; + faults.arm("content", point); + await assert.rejects(send(url, init), /Injected/); + assert.throws(() => faults.assertConsumed(), /never retried/); + assert.throws( + () => faults.arm("manifest"), + /not exercised and retried/ + ); + const before = sent; + await assert.rejects( + send(url, { + ...init, + body: JSON.stringify({ ...body, requestId: "regenerated" }) + }), + /Retry changed/ + ); + await assert.rejects( + send(`${url}-different`, init), + /Retry changed target/ + ); + await assert.rejects( + send(url, { + ...init, + body: JSON.stringify({ ...body, parentVersionId: 4 }) + }), + /Retry changed/ + ); + assert.equal(sent, before, "Invalid retries must not reach the server"); + assert.throws(() => faults.assertConsumed(), /never retried/); + await send(url, init); + faults.assertConsumed(); + } +}); + +test("an update fault cannot be consumed by creating a new UUID", async () => { + const faults = new NetworkFaults(); + const send = faults.wrap(async () => new Response('{"type":"Accepted"}')); + faults.arm("content"); + await send("http://test/documents/new", { + method: "PUT", + body: '{"requestId":"new","parentVersionId":null}' + }); + assert.throws(() => faults.assertConsumed(), /never fired/); + const update = { + method: "PUT", + body: '{"requestId":"update","parentVersionId":1}' + }; + await assert.rejects( + send("http://test/documents/existing", update), + /Injected/ + ); + await send("http://test/documents/existing", update); + faults.assertConsumed(); +}); + +test("unlink is idempotent even after an uncertain visible deletion", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a", bytes("A")); + disk.boundary = (label) => { + if (label === "visible:unlink:a") throw new Error("uncertain unlink"); + }; + await assert.rejects(disk.deleteFile("a"), /uncertain/); + disk.boundary = () => {}; + await disk.deleteFile("a"); + disk.crash(true); + assert.equal(await disk.exists("a"), false); + await disk.deleteFile("a"); + await disk.createDirectory("directory"); + await assert.rejects(disk.deleteFile("directory"), /directory/); +}); + +test("text assertions preserve BOM, CRLF, empty content and missing final newline", async () => { + const agent = new DeterministicAgent(0, {}, () => {}); + for (const content of [ + "", + "\ufeffhello", + "\ufeffa\r\nb\r\n", + "last line" + ]) { + await agent.disk.userWrite("a.md", bytes(content)); + assert.equal(await agent.getFileContent("a.md"), content); + } +}); + +test("durably pruning a directory also removes its already-unlinked durable children", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("transaction/source", bytes("staged bytes")); + disk.boundary = (label) => { + if (label === "visible:unlink:transaction/source") { + disk.crash(false); + throw new Error("process died after unlink"); + } + }; + await assert.rejects(disk.deleteFile("transaction/source"), /process died/); + disk.boundary = () => {}; + await disk.delete("transaction"); + disk.crash(true); + assert.deepEqual(disk.userFiles(), new Map()); + assert.equal(await disk.exists("transaction"), false); +}); + +test("portable components are limited by UTF-8 bytes, not characters or total path length", () => { + for (const component of ["a".repeat(255), "é".repeat(127) + "a"]) + assertManifest({ [id]: `${component}/${component}/${component}` }); + for (const component of ["a".repeat(256), "é".repeat(128)]) + assert.throws( + () => assertManifest({ [id]: `nested/${component}` }), + /Oversized component/ + ); +}); + +function state(documents: { path: string; content: string; id: string }[]) { + return new AssertableState({ + files: new Map(documents.map((doc) => [doc.path, doc.content])), + clientFiles: [], + manifests: [], + canonical: Object.fromEntries( + documents.map((doc) => [doc.id, doc.path]) + ), + bytes: new Map(documents.map((doc) => [doc.path, bytes(doc.content)])) + }); +} + +test("complete scenario assertions reject collapsed creates, forgotten edits, reverted moves and recycled UUIDs", () => { + const identities = new Map(); + const expected: ExpectedDocument[] = [ + { key: "a", path: "a.md", content: "A\nimportant edit" }, + { key: "b", path: "b.md", content: "B" } + ]; + const original = [ + { id: "id-a", path: "a.md", content: "A\nimportant edit" }, + { id: "id-b", path: "b.md", content: "B" } + ]; + state(original).assertDocuments(expected, identities); + assert.throws( + () => + state([ + { ...original[0], content: "A\nimportant edit\nB" } + ]).assertDocuments(expected, identities), + /Expected 2 file/ + ); + assert.throws( + () => + state([ + { ...original[0], content: "A" }, + original[1] + ]).assertDocuments(expected, identities), + /Byte mismatch/ + ); + const moved = [ + { ...expected[0], path: "b.md" }, + { ...expected[1], path: "a.md" } + ]; + assert.throws( + () => state(original).assertDocuments(moved, identities), + /Byte mismatch/ + ); + assert.throws( + () => + state([ + { ...original[0], id: "id-b" }, + { ...original[1], id: "id-a" } + ]).assertDocuments(expected, identities), + /identity changed/ + ); + state([original[1]]).assertDocuments([expected[1]], identities); + assert.throws( + () => + state(original).assertDocuments( + [{ ...expected[0], key: "recreated" }, expected[1]], + identities + ), + /reused.*deleted UUID/ + ); + assert.equal(identities.has("recreated"), false); + const conflict = [ + { key: "one", path: "same.md", content: "one" }, + { key: "two", path: "same.md", conflict: true, content: "two" } + ]; + state([ + { id: "one", path: "same.md", content: "one" }, + { id: "two", path: "same (conflict two).md", content: "two" } + ]).assertDocuments(conflict, new Map()); +}); + +test("canonical checks follow pages without weakening gap or duplicate-request detection", async (context) => { + const manifest = { fileManifestId: 3, entries: {} }; + const client = () => ({ + files: () => new Map(), + database: () => ({ + initialized: true, + local: {}, + documents: {}, + fileManifest: manifest, + lastSeenUpdateId: 3 + }) + }); + let fault = "none"; + context.mock.method(globalThis, "fetch", async (input: string) => { + if (input.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 3, + fileManifest: manifest, + documents: [] + }); + const after = Number(new URL(input).searchParams.get("after")); + if (after === 0) + return Response.json({ + headEventId: 3, + endEventId: 1, + events: [{ eventId: 1, requestId: "first" }] + }); + assert.equal(after, 1); + return Response.json({ + headEventId: 3, + endEventId: fault === "empty" ? 1 : 3, + events: + fault === "empty" + ? [] + : [ + ...(fault === "gap" + ? [] + : [ + { + eventId: 2, + requestId: + fault === "duplicate" + ? "first" + : "second" + } + ]), + { eventId: 3, requestId: "third" } + ] + }); + }); + await assertCanonical([client(), client()], "http://test", "token"); + for (fault of ["gap", "duplicate", "empty"]) + await assert.rejects( + assertCanonical([client(), client()], "http://test", "token"), + /contiguous|multiple events|progress/ + ); +}); diff --git a/frontend/test-support/oracles.ts b/frontend/test-support/oracles.ts new file mode 100644 index 00000000..bc59b971 --- /dev/null +++ b/frontend/test-support/oracles.ts @@ -0,0 +1,163 @@ +import assert from "node:assert/strict"; +import type { StoredDatabase } from "sync-client"; + +export function assertSameFiles( + clients: readonly Map[] +): void { + assert( + clients.length >= 2, + "Consistency requires at least two distinct clients" + ); + assert.equal( + new Set(clients).size, + clients.length, + "Cannot compare a client to itself" + ); + const sorted = (files: Map) => + [...files].sort(([a], [b]) => a.localeCompare(b)); + for (let i = 1; i < clients.length; i++) + assert.deepEqual( + sorted(clients[i]), + sorted(clients[0]), + `Client ${i} differs from client 0 (paths or bytes)` + ); +} + +export function assertManifest(entries: Record): void { + const paths = Object.values(entries); + const folded = paths.map((p) => + p.normalize("NFC").toUpperCase().normalize("NFC") + ); + const directorySpellings = new Map(); + for (const path of paths) { + assert.equal( + path, + path.normalize("NFC"), + `Noncanonical Unicode path: ${path}` + ); + const parts = path.split("/"); + parts.forEach((part, i) => { + assert( + Buffer.byteLength(part, "utf8") <= 255, + `Oversized component: ${part}` + ); + assert( + !/[<>:"\\|?*\p{Cc}]|[. ]$/u.test(part), + `Nonportable component: ${part}` + ); + assert( + !/^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$/u.test( + part.split(".")[0].trimEnd().toUpperCase() + ), + `Reserved component: ${part}` + ); + if (i < parts.length - 1) { + const prefix = parts.slice(0, i + 1).join("/"); + const key = prefix.toUpperCase().normalize("NFC"); + const previous = directorySpellings.get(key); + assert( + previous === undefined || previous === prefix, + `Aliased directory spellings: ${prefix} and ${previous}` + ); + directorySpellings.set(key, prefix); + } + }); + } + assert.equal( + new Set(folded).size, + paths.length, + "Manifest contains aliased/duplicate paths" + ); + for (const path of folded) { + assert( + path && !path.startsWith("/") && !path.includes("\\"), + `Invalid path ${path}` + ); + assert( + path.split("/").every((p) => p && p !== "." && p !== ".."), + `Invalid component ${path}` + ); + assert( + path.split("/")[0] !== ".VAULT-LINK-SYNC", + "Internal path in manifest" + ); + assert( + !folded.some( + (other) => other !== path && other.startsWith(`${path}/`) + ), + `File/ancestor conflict: ${path}` + ); + } +} + +/** Independent per-ID policy oracle, intentionally not importing merge code. */ +export function pathDecision( + base: string | undefined, + local: string | undefined, + remote: string | undefined +): string | undefined { + if (local === base) return remote; + if (remote === base) return local; + return remote; +} + +export function assertQuiescent( + state: Partial, + entries: Record, + head: number +): void { + assert.equal( + state.pending, + undefined, + "Pending CAS request at convergence" + ); + assert.deepEqual( + state.local, + entries, + "Local UUID/path map differs from server" + ); + assert.deepEqual( + state.fileManifest?.entries, + entries, + "Client manifest differs from server" + ); + assert.equal( + state.lastSeenUpdateId, + head, + "Client event watermark differs from server" + ); + assertManifest(entries); +} + +/** Removal exemptions name exact markers and the explicit destructive action. + * Deletion mode never disables preservation for unrelated documents. */ +export class ContentLedger { + private readonly markers = new Map(); + public add(marker: string): void { + assert(!this.markers.has(marker), "Duplicate marker"); + this.markers.set(marker, undefined); + } + public removedBy(content: string, action: string): void { + assert(action.length > 0); + for (const marker of this.markers.keys()) + if (content.includes(marker)) this.markers.set(marker, action); + } + public assertPreserved(files: Map): void { + const texts = [...files.values()].map((bytes) => + Buffer.from(bytes).toString("utf8") + ); + for (const [marker, deletion] of this.markers) { + const count = texts.reduce( + (n, text) => n + text.split(marker).length - 1, + 0 + ); + assert(count <= 1, `Marker ${marker} duplicated ${count} times`); + if (!deletion) + assert.equal( + count, + 1, + `Marker ${marker} lost without an explicit delete/overwrite` + ); + } + } +} diff --git a/frontend/test-support/polling.test.ts b/frontend/test-support/polling.test.ts new file mode 100644 index 00000000..18187dae --- /dev/null +++ b/frontend/test-support/polling.test.ts @@ -0,0 +1,316 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import type { EventBatch } from "../sync-client/src/services/types/EventBatch"; +import type { EventRecord } from "../sync-client/src/services/types/EventRecord"; +import type { PutFileContent } from "../sync-client/src/services/types/PutFileContent"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +const documentId = "00000000-0000-4000-8000-000000000001"; +const interval = 1_000; +const retryInterval = 75; +const bytes = (text: string) => new TextEncoder().encode(text); + +/** Real sync loop over in-memory disk/persistence and an observable version + * store. The transport never wakes the engine itself. */ +async function fixture(syncIntervalMs: number | undefined) { + const disk = new MemoryDisk(); + await disk.userWrite("note.bin", bytes("base")); + let headEventId = 2; // Initial content (1), then manifest (2). + const versions = new Map([[1, bytes("base")]]); + const head = (version: number) => ({ + documentId, + vaultUpdateId: version, + contentSize: versions.get(version)!.length, + updatedDate: "2026-09-20T00:00:00Z", + userId: "user", + deviceId: "device" + }); + let latest = head(1); + const events: EventRecord[] = []; + const writes: PutFileContent[] = []; + let eventReads = 0; + let failNextRead = false; + const publishContent = (content: Uint8Array, requestId: string) => { + const eventId = ++headEventId; + versions.set(eventId, new Uint8Array(content)); + latest = head(eventId); + events.push({ eventId, requestId, type: "content", document: latest }); + return latest; + }; + const batch = (after = 2): EventBatch => + structuredClone({ + headEventId, + events: events.filter((event) => event.eventId > after) + }); + const service: Partial = { + events: async (after) => { + eventReads++; + if (failNextRead) { + failNextRead = false; + throw new Error("offline"); + } + return batch(after); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => { + const content = versions.get(vaultUpdateId); + assert(content, `Missing immutable version ${vaultUpdateId}`); + return content.slice(); + }, + putFileContent: async (id, request) => { + assert.equal(id, documentId); + assert.equal(request.parentVersionId, latest.vaultUpdateId); + assert(request.content.type === "Snapshot"); + writes.push(structuredClone(request)); + return { + type: "Accepted", + ...publishContent( + Buffer.from(request.content.value, "base64"), + request.requestId + ) + }; + } + }; + const initial = emptyState("polling-test"); + initial.initialized = true; + initial.lastSeenUpdateId = 2; + initial.local = { [documentId]: "note.bin" }; + initial.fileManifest = { fileManifestId: 2, entries: { ...initial.local } }; + initial.remoteHeads = { [documentId]: latest }; + const hash = (await toStoredSnapshot({ content: bytes("base") })).hash; + initial.documents[documentId] = { + materialized: true, + observedHash: hash, + base: { ...latest, hash } + }; + const persistence = new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + initial, + async (next) => persistence.save({ database: next }), + initial.vaultKey, + async () => (await persistence.load()).database as StoredDatabase + ); + const settings = new Settings( + logger, + { syncIntervalMs, networkRetryIntervalMs: retryInterval }, + async () => {} + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: [] }), + reset: () => {} + } as unknown as ServerConfig; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => Promise + >() + }; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket as unknown as WebSocketManager, + new FileOperations(disk.session(), database, config), + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(0) + ); + return { + disk, + database, + syncer, + writes, + batch, + websocket, + reads: () => eventReads, + serverContent: () => versions.get(latest.vaultUpdateId), + failNextRead: () => { + failNextRead = true; + }, + changeRemote: () => { + publishContent(bytes("remote edit"), "remote-content"); + const eventId = ++headEventId; + events.push({ + eventId, + requestId: "remote-rename", + type: "fileManifest", + fileManifest: { + fileManifestId: eventId, + entries: { [documentId]: "renamed.bin" } + } + }); + } + }; +} + +for (const syncIntervalMs of [undefined, 0, interval]) { + for (const side of ["local", "remote"] as const) { + test(`${side} changes without notifications ${syncIntervalMs ? "are recovered by explicit polling" : `remain unobserved when polling is ${syncIntervalMs === 0 ? "zero" : "unset"}`}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + const f = await fixture(syncIntervalMs); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + const reads = f.reads(); + if (side === "local") + await f.disk.userWrite("note.bin", bytes("local edit")); + else f.changeRemote(); + const assertUnobserved = () => { + assert.equal( + f.reads(), + reads, + "A hidden wakeup fetched events" + ); + assert.equal(f.database.state.lastSeenUpdateId, 2); + assert.equal(f.writes.length, 0); + assert.deepEqual( + f.serverContent(), + bytes(side === "local" ? "base" : "remote edit") + ); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + [ + "note.bin", + bytes(side === "local" ? "local edit" : "base") + ] + ]) + ); + }; + context.mock.timers.tick( + syncIntervalMs ? interval - 1 : 60_000 + ); + await f.syncer.waitUntilFinished(); + assertUnobserved(); + if (syncIntervalMs) context.mock.timers.tick(1); + else if (side === "local") + await f.syncer.syncLocallyUpdatedFile({ + relativePath: "note.bin" + }); + else + await f.websocket.onRemoteVaultUpdateReceived.triggerAsync( + f.batch() + ); + await f.syncer.waitUntilFinished(); + const content = bytes( + side === "local" ? "local edit" : "remote edit" + ); + const path = side === "local" ? "note.bin" : "renamed.bin"; + assert.deepEqual(f.serverContent(), content); + assert.deepEqual( + f.disk.userFiles(), + new Map([[path, content]]) + ); + assert.deepEqual(f.database.state.local, { + [documentId]: path + }); + assert.equal( + f.database.state.lastSeenUpdateId, + side === "local" ? 3 : 4 + ); + assert.equal(f.syncer.isBusy, false); + assert.equal(f.writes.length, side === "local" ? 1 : 0); + if (syncIntervalMs) { + const beforeNextPoll = f.reads(); + context.mock.timers.tick(interval); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + beforeNextPoll + 1, + "Polling stopped after its first successful run" + ); + assert.equal( + f.writes.length, + side === "local" ? 1 : 0, + "An idle poll resubmitted unchanged content" + ); + } + await f.syncer.stop(); + const stoppedReads = f.reads(); + context.mock.timers.tick(60_000); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + stoppedReads, + "Polling continued after stop" + ); + } finally { + await f.syncer.stop(); + context.mock.timers.reset(); + } + }); + } +} + +test("transient failures retry with polling disabled, then leave the client idle", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + const f = await fixture(0); + try { + f.changeRemote(); + f.failNextRead(); + f.syncer.start(); + await assert.rejects(f.syncer.waitUntilFinished(), /offline/); + const reads = f.reads(); + context.mock.timers.tick(retryInterval - 1); + await assert.rejects(f.syncer.waitUntilFinished(), /offline/); + assert.equal(f.reads(), reads); + context.mock.timers.tick(1); + await f.syncer.waitUntilFinished(); + assert.equal(f.reads(), reads + 1); + assert.deepEqual( + f.disk.userFiles(), + new Map([["renamed.bin", bytes("remote edit")]]) + ); + assert.equal(f.database.state.lastSeenUpdateId, 4); + assert.equal(f.syncer.isBusy, false); + context.mock.timers.tick(60_000); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + reads + 1, + "Network retries became clean-state polling" + ); + } finally { + await f.syncer.stop(); + context.mock.timers.reset(); + } +}); + +test("a failed WebSocket connection catches up over HTTP even when polling is disabled", async () => { + const f = await fixture(0); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + f.changeRemote(); + f.websocket.onWebSocketStatusChanged.trigger(false); + await f.syncer.waitUntilFinished(); + assert.equal(f.database.state.lastSeenUpdateId, 4); + assert.deepEqual( + f.disk.userFiles(), + new Map([["renamed.bin", bytes("remote edit")]]) + ); + } finally { + await f.syncer.stop(); + } +}); diff --git a/frontend/test-support/production-adapters.test.ts b/frontend/test-support/production-adapters.test.ts new file mode 100644 index 00000000..64832b8f --- /dev/null +++ b/frontend/test-support/production-adapters.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import { Logger, SyncClient } from "sync-client"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { ClientPersistence } from "../local-client-cli/src/client-persistence"; +import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesystem"; +import { ObsidianFileSystemOperations } from "../obsidian-plugin/src/obsidian-file-system"; +import { getJson, type CanonicalSnapshot } from "./canonical"; + +const root = path.resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const pause = () => new Promise((resolve) => setTimeout(resolve, 50)); +async function until(check: () => Promise, message: string) { + const deadline = Date.now() + 15_000; + while (!(await check())) { + assert(Date.now() < deadline, message); + await pause(); + } +} +async function contents(file: string) { + try { + return await fs.readFile(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return undefined; + throw error; + } +} +async function stop(child: ChildProcess, signal: NodeJS.Signals) { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill(signal); + await exited; +} + +test( + "production CLI watchers, Obsidian adapter, and full metadata survive process replacement", + { timeout: 90_000 }, + async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-production-") + ); + const server = new ServerControl( + path.join(root, "sync-server/target/release/sync_server"), + path.join(root, "sync-server/config-e2e.yml"), + new Logger() + ); + const children: ChildProcess[] = []; + let obsidian: SyncClient | undefined; + t.after(async () => { + await obsidian?.destroy(); + await Promise.all(children.map((child) => stop(child, "SIGKILL"))); + await server.stop(); + await fs.rm(directory, { recursive: true, force: true }); + }); + await server.start(); + const a = path.join(directory, "a"), + b = path.join(directory, "b"), + c = path.join(directory, "obsidian"); + await Promise.all([a, b, c].map((p) => fs.mkdir(p))); + await fs.writeFile(path.join(a, "note.md"), "initial\n"); + async function launch(vault: string) { + const child = spawn( + process.execPath, + [ + path.join(root, "frontend/local-client-cli/dist/cli.js"), + "-l", + vault, + "-r", + server.remoteUri, + "-t", + token, + "-v", + "production", + "--ignore-pattern", + "**/*.tmp" + ], + { stdio: ["ignore", "pipe", "pipe"] } + ); + children.push(child); + let log = ""; + child.stdout!.on("data", (data: Buffer) => { + log += data.toString(); + }); + child.stderr!.on("data", (data: Buffer) => { + log += data.toString(); + }); + await until(async () => { + assert( + child.exitCode === null && child.signalCode === null, + `CLI exited: ${log}` + ); + return log.includes("File watcher started"); + }, `CLI did not start: ${log}`); + return child; + } + let first = await launch(a); + await launch(b); + await until( + async () => + (await contents(path.join(b, "note.md"))) === "initial\n", + "CLI initial download failed" + ); + const persistence = (vault: string) => + new ClientPersistence( + path.join(vault, ".vaultlink/sync-data.json"), + {} + ); + const original = (await persistence(a).load()).database!; + const [documentId] = Object.keys(original.local!); + assert(documentId); + + const settings = { + remoteUri: server.remoteUri, + token, + vaultName: "production", + isSyncEnabled: true, + enableTelemetry: false, + ignorePatterns: [".vaultlink/**"], + syncIntervalMs: 0 + }; + obsidian = await SyncClient.create({ + fs: new ObsidianFileSystemOperations( + new NodeFileSystemOperations(c), + () => undefined + ), + persistence: new ClientPersistence( + path.join(c, ".vaultlink/sync-data.json"), + settings + ) + }); + await obsidian.start(); + await obsidian.waitUntilFinished(); + assert.equal(await contents(path.join(c, "note.md")), "initial\n"); + await fs.writeFile( + path.join(a, "replacement.tmp"), + "atomic editor save\n" + ); + await fs.rename( + path.join(a, "replacement.tmp"), + path.join(a, "note.md") + ); + await until( + async () => + (await contents(path.join(b, "note.md"))) === + "atomic editor save\n" && + (await contents(path.join(c, "note.md"))) === + "atomic editor save\n", + "Editor replacement did not converge" + ); + assert.equal( + (await persistence(a).load()).database!.local![documentId], + "note.md" + ); + await fs.rename(path.join(b, "note.md"), path.join(b, "renamed.md")); + await until( + async () => + (await contents(path.join(a, "renamed.md"))) === + "atomic editor save\n" && + (await contents(path.join(a, "note.md"))) === undefined, + "CLI rename did not converge" + ); + await until( + async () => + (await persistence(a).load()).database!.local![documentId] === + "renamed.md", + "Rename changed document identity" + ); + const saved = await persistence(a).load(); + assert( + saved.historyCheckpoint?.checkpoint, + "CLI dropped the history checkpoint" + ); + await stop(first, "SIGKILL"); + await fs.writeFile(path.join(a, "renamed.md"), "edited after kill\n"); + first = await launch(a); + await until( + async () => + (await contents(path.join(b, "renamed.md"))) === + "edited after kill\n" && + (await contents(path.join(c, "renamed.md"))) === + "edited after kill\n", + "Restart failed to reconcile current files" + ); + const snapshot = await getJson( + `${server.remoteUri}/vaults/production/vault-snapshot`, + token + ); + assert.deepEqual(snapshot.fileManifest.entries, { + [documentId]: "renamed.md" + }); + await fs.unlink(path.join(b, "renamed.md")); + await until( + async () => + (await contents(path.join(a, "renamed.md"))) === undefined && + (await contents(path.join(c, "renamed.md"))) === undefined, + "Delete did not converge" + ); + await stop(first, "SIGTERM"); + assert.equal(first.exitCode, 0, "CLI shutdown failed"); + } +); diff --git a/frontend/test-support/protocol-audit.test.ts b/frontend/test-support/protocol-audit.test.ts new file mode 100644 index 00000000..bf888cad --- /dev/null +++ b/frontend/test-support/protocol-audit.test.ts @@ -0,0 +1,390 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { resolve, join } from "node:path"; +import { tmpdir } from "node:os"; +import { mkdtemp, rm } from "node:fs/promises"; +const root = resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const bytes = (s: string) => new TextEncoder().encode(s); +const headers = { + Authorization: `Bearer ${token}`, + "Device-Id": "audit", + "Content-Type": "application/json" +}; +async function put(url: string, body: unknown) { + const r = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(body) + }); + assert(r.ok, await r.clone().text()); + return r.json(); +} +const content = (s: string) => ({ + requestId: randomUUID(), + parentVersionId: null, + content: { type: "Snapshot", value: Buffer.from(s).toString("base64") } +}); + +test( + "connection admission rejects excess clients and releases disconnected slots", + { timeout: 15000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "vault-link-admission-")); + const config = join(dir, "config.yml"); + await writeFile( + config, + ( + await readFile(root + "/sync-server/config-e2e.yml", "utf8") + ).replace("max_clients_per_vault: 256", "max_clients_per_vault: 1") + ); + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + config, + new Logger() + ); + const sockets: WebSocket[] = []; + try { + await server.start(); + const url = server.remoteUri + "/vaults/" + randomUUID(); + await put(url + "/documents/" + randomUUID(), content("event")); + const connect = async () => { + const ws = new WebSocket(url.replace("http:", "ws:") + "/ws"); + sockets.push(ws); + const accepted = await new Promise((resolve) => { + ws.onopen = () => + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId: randomUUID() + }) + ); + ws.onmessage = () => resolve(true); + ws.onerror = () => resolve(false); + ws.onclose = () => resolve(false); + }); + return { ws, accepted }; + }; + const first = await connect(); + assert(first.accepted); + assert.equal((await connect()).accepted, false); + await new Promise((resolve) => { + first.ws.onclose = () => resolve(); + first.ws.close(); + }); + // A completed close releases the server's owned connection permit. + assert.equal((await connect()).accepted, true); + } finally { + for (const ws of sockets) ws.close(); + await server.stop(); + await rm(dir, { recursive: true, force: true }); + } + } +); + +test( + "an unauthenticated upgraded socket expires without a handshake", + { timeout: 10000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let ws: WebSocket | undefined; + try { + await server.start(); + ws = new WebSocket( + server.remoteUri.replace("http:", "ws:") + + "/vaults/" + + randomUUID() + + "/ws" + ); + const socket = ws; + const closed = await new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), 6500); + socket.onclose = () => { + clearTimeout(timer); + resolve(true); + }; + socket.onerror = () => {}; + }); + assert(closed, "handshake deadline must release idle sockets"); + } finally { + ws?.close(); + await server.stop(); + } + } +); + +test( + "bootstrap notifications adopt the existing remote document identity", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let client: SyncClient | undefined; + try { + await server.start(); + const vault = randomUUID(), + id = randomUUID(); + const url = server.remoteUri + "/vaults/" + vault; + await put(url + "/documents/" + id, content("REMOTE")); + await put(url + "/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0 + } + }); + client = await SyncClient.create({ fs: disk, persistence: store }); + await disk.userWrite("a.md", bytes("LOCAL")); + await client.syncLocallyCreatedFile("a.md"); + await disk.userWrite("b.md", bytes("B")); + await client.syncLocallyCreatedFile("b.md"); + await client.start(); + const files = [...disk.userFiles().keys()]; + assert.equal(files.length, 2); + } finally { + await client?.destroy(); + await server.stop(); + } + } +); + +test( + "local undo survives a lost accepted reply followed by a rejected retry", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let client: SyncClient | undefined; + class SilentWebSocket { + static OPEN = 1; + readyState = 1; + onopen: ((event: object) => void) | null = null; + onclose: + | ((event: { code: number; reason: string }) => void) + | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: ((event: object) => void) | null = null; + constructor(_url: string | URL) { + queueMicrotask(() => this.onopen?.({})); + } + send(_data: string) {} + close() { + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + try { + await server.start(); + const vault = randomUUID(), + id = randomUUID(); + const url = server.remoteUri + "/vaults/" + vault; + await put(url + "/documents/" + id, content("base")); + await put(url + "/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0, + networkRetryIntervalMs: 0 + } + }); + let mode: "normal" | "lose-reply" | "reject-retry" = "normal"; + let lostId: string | undefined; + client = await SyncClient.create({ + fs: disk, + persistence: store, + webSocket: SilentWebSocket as unknown as typeof WebSocket, + fetch: async (input, init) => { + if (init?.method === "PUT" && mode === "reject-retry") { + assert.equal( + JSON.parse(String(init.body)).requestId, + lostId + ); + mode = "normal"; + return new Response("proxy body limit changed", { + status: 413 + }); + } + const response = await fetch(input, init); + if (init?.method === "PUT" && mode === "lose-reply") { + assert.equal( + (await response.clone().json()).type, + "Accepted" + ); + lostId = JSON.parse(String(init.body)).requestId; + mode = "normal"; + throw new Error("injected lost accepted response"); + } + return response; + } + }); + await client.start(); + mode = "lose-reply"; + await disk.userWrite("a.md", bytes("base A")); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + await assert.rejects(client.waitUntilFinished(), /lost accepted/); + const original = await ( + await fetch(url + "/documents/" + id, { headers }) + ).json(); + await put(url + "/documents/" + id, { + ...content("base A REMOTE"), + parentVersionId: original.vaultUpdateId + }); + mode = "reject-retry"; + await disk.userWrite("a.md", bytes("base")); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + await client.waitUntilFinished(); + const result = Buffer.from( + (await disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(result, "base REMOTE"); + } finally { + await client?.destroy(); + await server.stop(); + } + } +); + +test( + "closing an old connection cannot erase its replacement's cursors", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + const sockets: WebSocket[] = []; + try { + await server.start(); + const url = server.remoteUri + "/vaults/" + randomUUID(); + const id = randomUUID(); + await put(url + "/documents/" + id, content("example")); + const connect = async (deviceId: string) => { + const ws = new WebSocket(url.replace("http:", "ws:") + "/ws"); + sockets.push(ws); + await new Promise((resolve, reject) => { + ws.onopen = () => + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId + }) + ); + ws.onmessage = () => resolve(); + ws.onerror = () => reject(new Error("connection failed")); + }); + return ws; + }; + const observer = await connect("observer"); + const first = await connect("same-device"); + const sendCursor = (ws: WebSocket, start: number) => + ws.send( + JSON.stringify({ + type: "cursorPositions", + documentsWithCursors: [ + { + document_id: id, + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start, end: start }] + } + ] + }) + ); + const seen = (start: number) => + new Promise((resolve) => { + observer.onmessage = (event) => { + const message = JSON.parse(String(event.data)); + if ( + message.type === "cursorPositions" && + message.clients.some( + (client: { + deviceId: string; + documentsWithCursors: { + cursors: { start: number }[]; + }[]; + }) => + client.deviceId === "same-device" && + client.documentsWithCursors[0]?.cursors[0] + ?.start === start + ) + ) + resolve(); + }; + }); + let observed = seen(1); + sendCursor(first, 1); + await observed; + const replacement = await connect("same-device"); + observed = seen(2); + sendCursor(replacement, 2); + await observed; + await new Promise((resolve) => { + first.onclose = () => resolve(); + first.close(); + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const current = new Promise< + { + deviceId: string; + documentsWithCursors: { cursors: { start: number }[] }[]; + }[] + >((resolve) => { + observer.onmessage = (event) => { + const message = JSON.parse(String(event.data)); + if (message.type === "cursorPositions") + resolve(message.clients); + }; + }); + observer.send( + JSON.stringify({ + type: "cursorPositions", + documentsWithCursors: [] + }) + ); + const clients = await current; + assert.equal( + clients.find((client) => client.deviceId === "same-device") + ?.documentsWithCursors[0]?.cursors[0]?.start, + 2 + ); + } finally { + for (const ws of sockets) ws.close(); + await server.stop(); + } + } +); diff --git a/frontend/test-support/protocol.test.ts b/frontend/test-support/protocol.test.ts new file mode 100644 index 00000000..6e96bda6 --- /dev/null +++ b/frontend/test-support/protocol.test.ts @@ -0,0 +1,431 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { Worker } from "node:worker_threads"; +import * as path from "node:path"; +import { Logger } from "sync-client"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { MemoryDisk, type DiskImage, type StoredClient } from "./storage"; +import { getJson, type CanonicalSnapshot } from "./canonical"; +import type { RequestRecord } from "./network"; +import type { CrashWorkerInput } from "../deterministic-tests/src/crash-worker"; + +const root = path.resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const headers = { + Authorization: `Bearer ${token}`, + "Device-Id": "protocol-test", + "Content-Type": "application/json" +}; +const makeServer = () => + new ServerControl( + path.join(root, "sync-server/target/release/sync_server"), + path.join(root, "sync-server/config-e2e.yml"), + new Logger() + ); +async function put(url: string, body: unknown) { + const response = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(5_000) + }); + assert( + response.ok, + `HTTP ${response.status}: ${await response.clone().text()}` + ); + return (await response.json()) as { + type: string; + vaultUpdateId: number; + fileManifestId: number; + }; +} +const content = (text: string, parentVersionId: number | null = null) => ({ + requestId: randomUUID(), + parentVersionId, + content: { type: "Snapshot", value: Buffer.from(text).toString("base64") } +}); + +interface WorkerSnapshot { + type: string; + disk: DiskImage; + stored: StoredClient; + requests: RequestRecord[]; + editedBase?: { documentId: string; vaultUpdateId: number }; +} +async function runWorker(input: CrashWorkerInput): Promise { + const worker = new Worker( + path.join(root, "frontend/deterministic-tests/dist/crash-worker.js"), + { workerData: input } + ); + let timeout: ReturnType | undefined; + try { + return await new Promise((resolve, reject) => { + timeout = setTimeout( + () => + reject(new Error("Worker checkpoint/completion timed out")), + 15_000 + ); + worker.once("error", reject); + worker.once("exit", (code) => + reject( + new Error( + `Worker exited (${code}) before expected checkpoint` + ) + ) + ); + worker.on( + "message", + (message: WorkerSnapshot & { error?: string }) => { + if (message.type === "error") + reject(new Error(message.error)); + else { + assert.equal( + message.type, + input.stopAt ? "checkpoint" : "finished" + ); + resolve(message); + } + } + ); + }); + } finally { + clearTimeout(timeout); + // This is the fault injection, not graceful shutdown masquerading as it. + await worker.terminate(); + } +} + +for (const [kind, format, originalPath] of [ + ["create", "Snapshot", "original.md"], + ["content", "Snapshot", "original.bin"], + ["content", "Diff", "original.md"], + ["manifest", undefined, "original.md"] +] as const) { + test( + `abrupt client death after accepted ${kind} ${format ?? ""} reply, newer head, server SIGKILL, durable retry`, + { timeout: 30_000 }, + async () => { + const server = makeServer(); + try { + await server.start(); + const vault = `crash-${randomUUID()}`; + const url = `${server.remoteUri}/vaults/${vault}`; + const disk = new MemoryDisk(); + await disk.userWrite( + originalPath, + new TextEncoder().encode("preserve-original\n") + ); + let input: CrashWorkerInput = { + disk: disk.image(), + stored: { + settings: { + isSyncEnabled: true, + enableTelemetry: false, + remoteUri: server.remoteUri, + vaultName: vault, + token, + requestTimeoutMs: 5_000, + syncIntervalMs: 0 + } + } as StoredClient + }; + let originalId: string | undefined; + let originalParent: number | undefined; + let uploaded = "preserve-original\n"; + if (kind === "content" && format === "Diff") { + // Keep the established base in the worker's real content + // cache; a cold restart intentionally falls back to Snapshot. + uploaded += "local-update\n"; + input.editAfterSync = { + path: originalPath, + content: uploaded + }; + } else if (kind === "content") { + const initialized = await runWorker(input); + originalId = Object.keys( + initialized.stored.database!.local! + )[0]; + originalParent = + initialized.stored.database!.documents![originalId] + .base!.vaultUpdateId; + disk.restore(initialized.disk); + uploaded += "local-update\n"; + await disk.userWrite( + originalPath, + new TextEncoder().encode(uploaded) + ); + input = { disk: disk.image(), stored: initialized.stored }; + } + const checkpoint = await runWorker({ ...input, stopAt: kind }); + if (format === "Diff") { + assert( + checkpoint.editedBase, + "Warm update baseline was not recorded" + ); + originalId = checkpoint.editedBase.documentId; + originalParent = checkpoint.editedBase.vaultUpdateId; + } + const pending = checkpoint.stored.database!.pending!; + assert( + pending, + "The uncertain request was not persisted before sending" + ); + const interrupted = checkpoint.requests.at(-1)!; + assert.equal( + interrupted.kind, + kind, + "Wrong CAS was interrupted" + ); + if (kind !== "manifest") { + assert.equal(pending.type, "content"); + assert(pending.type === "content"); + assert.equal(pending.request.content.type, format); + assert.equal( + pending.request.parentVersionId, + kind === "create" ? null : originalParent + ); + if (originalId) + assert.equal( + pending.documentId, + originalId, + "Update created a replacement UUID" + ); + } + const accepted = await getJson( + `${url}/vault-snapshot`, + token + ); + // Unpublished content has no manifest membership yet, so it is + // deliberately absent from /vault-snapshot's document list. + const doc = + pending.type === "content" + ? await getJson<{ + documentId: string; + vaultUpdateId: number; + }>(`${url}/documents/${pending.documentId}`, token) + : accepted.documents[0]; + assert(doc); + const remote = await put( + `${url}/documents/${doc.documentId}`, + content(`${uploaded}remote-update\n`, doc.vaultUpdateId) + ); + assert.equal(remote.type, "Accepted"); + const remotePath = originalPath.replace("original", "remote"); + const renamed = await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: accepted.fileManifest.fileManifestId, + entries: { [doc.documentId]: remotePath } + }); + assert.equal(renamed.type, "Accepted"); + const before = await getJson( + `${url}/vault-snapshot`, + token + ); + await server.crash(); + await server.restart(); + assert.deepEqual( + await getJson(`${url}/vault-snapshot`, token), + before, + "Server restart lost acknowledged state" + ); + const restarted = await runWorker({ + disk: checkpoint.disk, + stored: checkpoint.stored + }); + const replay = restarted.requests.find( + (r) => r.body.requestId === pending.request.requestId + ); + assert(replay, "Restart never retried the uncertain request"); + assert.equal( + replay.url, + interrupted.url, + "Restart changed the request target" + ); + assert.deepEqual( + replay.body, + pending.request, + "Restart changed the persisted request" + ); + assert.equal(restarted.stored.database!.pending, undefined); + const recovered = new MemoryDisk(); + recovered.restore(restarted.disk); + assert.deepEqual( + recovered.userFiles(), + new Map([ + [ + remotePath, + new TextEncoder().encode( + `${uploaded}remote-update\n` + ) + ] + ]) + ); + assert.deepEqual(restarted.stored.database!.local, { + [doc.documentId]: remotePath + }); + const events = await getJson<{ + events: { requestId: string }[]; + }>(`${url}/events-since?after=0`, token); + assert.equal( + events.events.filter( + (e) => e.requestId === pending.request.requestId + ).length, + 1, + "Retry emitted a duplicate event" + ); + } finally { + await server.stop(); + } + } + ); +} + +test( + "WebSocket hints trigger HTTP catchup in commit order, including after reconnect", + { timeout: 20_000 }, + async () => { + const server = makeServer(); + let ws: WebSocket | undefined; + try { + await server.start(); + const url = `${server.remoteUri}/vaults/order-${randomUUID()}`; + const received: number[] = []; + let catchup = Promise.resolve(); + let catchupError: unknown; + const connect = async () => { + ws = new WebSocket(`${url.replace(/^http/, "ws")}/ws`); + ws.addEventListener("message", (message) => { + const hint = JSON.parse(String(message.data)); + if (hint.type !== "vaultChanged") return; + catchup = catchup + .then(async () => { + assert.deepEqual(hint, { type: "vaultChanged" }); + const batch = await getJson<{ + events: { eventId: number }[]; + }>( + `${url}/events-since?after=${received.at(-1) ?? 0}`, + token + ); + received.push( + ...batch.events.map((event) => event.eventId) + ); + }) + .catch((error) => { + catchupError = error; + }); + }); + await new Promise((resolve, reject) => { + ws!.onopen = () => resolve(); + ws!.onerror = () => reject(new Error("WebSocket failed")); + }); + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId: "order-observer" + }) + ); + }; + await connect(); + const ids = Array.from({ length: 12 }, () => randomUUID()); + const requests = ids.map((id, i) => ({ + id, + body: content(`bytes-${i}`) + })); + const replies = await Promise.all( + requests.map(({ id, body }) => + put(`${url}/documents/${id}`, body) + ) + ); + assert.deepEqual( + replies.map((r) => r.vaultUpdateId).sort((a, b) => a - b), + ids.map((_, i) => i + 1) + ); + const emptyManifest = await getJson<{ + entries: Record; + }>(`${url}/file-manifest`, token); + assert.deepEqual( + emptyManifest.entries, + {}, + "Content-only CAS changed membership" + ); + const entries = Object.fromEntries( + ids.map((id, i) => [id, `${i}.md`]) + ); + await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: 0, + entries + }); + // All retries must return original receipts without emitting events. + assert.deepEqual( + await Promise.all( + requests.map(({ id, body }) => + put(`${url}/documents/${id}`, body) + ) + ), + replies + ); + const waitForEvents = async (head: number) => { + const deadline = Date.now() + 5_000; + while (received.length < head && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)); + if (catchupError) throw catchupError; + assert.deepEqual( + received, + Array.from({ length: head }, (_, i) => i + 1), + "HTTP catchup after WebSocket hints lost or duplicated an event" + ); + }; + await waitForEvents(13); + await new Promise((resolve) => { + ws!.onclose = () => resolve(); + ws!.close(); + }); + await put( + `${url}/documents/${ids[0]}`, + content("next", replies[0].vaultUpdateId) + ); + await connect(); + await waitForEvents(14); + const log = await getJson<{ events: { eventId: number }[] }>( + `${url}/events-since?after=0`, + token + ); + assert.deepEqual( + received, + log.events.map((event) => event.eventId) + ); + const stale = await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: {} + }); + assert.equal(stale.type, "StaleBase"); + const invalid = await fetch(`${url}/file-manifest`, { + method: "PUT", + headers, + body: JSON.stringify({ + requestId: randomUUID(), + parentFileManifestId: 13, + entries: { [ids[0]]: "same", [ids[1]]: "same" } + }) + }); + assert.equal(invalid.status, 422); + assert.equal( + ( + await getJson( + `${url}/vault-snapshot`, + token + ) + ).headEventId, + 14, + "Rejected CAS emitted an event" + ); + } finally { + ws?.close(); + await server.stop(); + } + } +); diff --git a/frontend/test-support/random.ts b/frontend/test-support/random.ts new file mode 100644 index 00000000..f451ae47 --- /dev/null +++ b/frontend/test-support/random.ts @@ -0,0 +1,25 @@ +/** A seed controls harness choices; production UUIDs and OS scheduling remain + * nondeterministic. The concrete action trace is the stronger replay artifact. */ +export class Random { + private state: number; + public constructor(seed: number) { + if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) + throw new Error("Seed must be a uint32"); + this.state = seed; + } + public next(): number { + this.state = (this.state + 0x6d2b79f5) >>> 0; + let n = this.state; + n = Math.imul(n ^ (n >>> 15), n | 1); + n ^= n + Math.imul(n ^ (n >>> 7), n | 61); + return ((n ^ (n >>> 14)) >>> 0) / 4294967296; + } + public int(max: number): number { + if (!Number.isSafeInteger(max) || max < 1) + throw new Error("Empty random choice"); + return Math.floor(this.next() * max); + } + public pick(values: readonly T[]): T { + return values[this.int(values.length)]; + } +} diff --git a/frontend/test-support/real-disk.test.ts b/frontend/test-support/real-disk.test.ts new file mode 100644 index 00000000..63e5c0de --- /dev/null +++ b/frontend/test-support/real-disk.test.ts @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesystem"; +import { Database, emptyState } from "../sync-client/src/persistence/database"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; + +test( + "real filesystem: exclusive operations, unsafe entries, and file swaps", + { timeout: 20_000 }, + async () => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vault-link-real-fs-") + ); + try { + const root = path.join(directory, "vault"); + await fs.mkdir(root); + const disk = new NodeFileSystemOperations(root); + const snapshot = (text: string) => ({ + content: new TextEncoder().encode(text) + }); + const writes = await Promise.allSettled([ + disk.write("raced.bin", snapshot("first")), + disk.write("raced.bin", snapshot("second")) + ]); + assert.equal( + writes.filter((result) => result.status === "fulfilled").length, + 1, + "Exclusive create overwrote a racing writer" + ); + assert( + ["first", "second"].includes( + Buffer.from( + (await disk.readSnapshot("raced.bin"))!.content + ).toString() + ) + ); + for (const unsafe of [ + "../outside", + "/absolute", + "folder/../outside" + ]) { + await assert.rejects( + disk.write(unsafe, snapshot("blocked")), + /Unsafe path/ + ); + await assert.rejects(disk.readSnapshot(unsafe), /Unsafe path/); + } + // POSIX filenames can contain colons. The engine, not the adapter, + // allocates their portable names for sync. + if (process.platform !== "win32") { + await fs.writeFile( + path.join(root, "local:name.md"), + "local name" + ); + assert.equal( + Buffer.from( + (await disk.readSnapshot("local:name.md"))!.content + ).toString(), + "local name" + ); + await disk.rename("local:name.md", "portable-name.md"); + } + await disk.write("a.md", snapshot("A")); + await disk.write("b.md", snapshot("B")); + // A file in an ancestor position makes descendants absent. + assert.equal(await disk.stat("a.md/absent.md"), undefined); + assert.equal(await disk.readSnapshot("a.md/absent.md"), undefined); + await assert.rejects(disk.rename("a.md", "b.md")); + await assert.rejects(disk.delete("a.md"), /regular file/); + await fs.symlink("a.md", path.join(root, "symlink")); + await assert.rejects(disk.readSnapshot("symlink"), /Symlink/); + await fs.unlink(path.join(root, "symlink")); + await fs.link(path.join(root, "a.md"), path.join(root, "hardlink")); + await assert.rejects(disk.readSnapshot("a.md"), /hardlink/); + await fs.unlink(path.join(root, "hardlink")); + // The host may be case/normalization insensitive. Whichever model + // it exposes, an alias must never silently overwrite existing bytes. + for (const [original, alias] of [ + ["Case.bin", "case.bin"], + ["é.bin", "e\u0301.bin"] + ]) { + await disk.write(original, snapshot("original bytes")); + if (await disk.exists(alias)) + await assert.rejects( + disk.write(alias, snapshot("alias bytes")) + ); + else { + await disk.write(alias, snapshot("alias bytes")); + assert.equal( + Buffer.from( + (await disk.readSnapshot(alias))!.content + ).toString(), + "alias bytes" + ); + } + assert.equal( + Buffer.from( + (await disk.readSnapshot(original))!.content + ).toString(), + "original bytes" + ); + } + const initial = emptyState("real"); + initial.local = { a: "a.md", b: "b.md" }; + for (const [id, relative] of Object.entries(initial.local)) + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot( + (await disk.readSnapshot(relative))! + ) + ).hash + }; + const database = new Database( + initial, + async () => {}, + "real", + async () => initial + ); + const operations = new FileOperations(disk, database, { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig); + const next = structuredClone(initial); + next.local = { a: "b.md", b: "a.md" }; + await operations.apply(next); + assert.equal( + Buffer.from( + (await disk.readSnapshot("a.md"))!.content + ).toString(), + "B" + ); + assert.equal( + Buffer.from( + (await disk.readSnapshot("b.md"))!.content + ).toString(), + "A" + ); + } finally { + await fs.rm(directory, { recursive: true }); + } + } +); diff --git a/frontend/test-support/recovery.test.ts b/frontend/test-support/recovery.test.ts new file mode 100644 index 00000000..25163213 --- /dev/null +++ b/frontend/test-support/recovery.test.ts @@ -0,0 +1,580 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import { mergeFileManifests } from "../sync-client/src/sync-operations/file-manifest"; +import { assertManifest, pathDecision } from "./oracles"; +import { + scanLocalFiles, + type LocalChange +} from "../sync-client/src/sync-operations/scan"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const config = { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) +} as ServerConfig; + +async function fixture( + contents: Record = { + a: { path: "a.md", text: "A" }, + b: { path: "b.md", text: "B" } + } +) { + const disk = new MemoryDisk(); + const initial = emptyState("recovery-test"); + for (const [id, { path, text }] of Object.entries(contents)) { + await disk.userWrite(path, bytes(text)); + initial.local[id] = path; + } + for (const [id, path] of Object.entries(initial.local)) + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot((await disk.readSnapshot(path))!) + ).hash + }; + const persistence = new MemoryPersistence({ database: initial }); + const restart = (changes: LocalChange[] = []) => { + const database = new Database( + persistence.snapshot().database, + async (next) => persistence.save({ database: next }), + "recovery-test", + async () => (await persistence.load()).database as StoredDatabase + ); + return { + database, + files: new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} } + ) + }; + }; + return { + disk, + persistence, + restart, + initial, + permanentFault: (_label: string): void => {}, + expectedLocal: undefined as Record | undefined + }; +} + +async function swapFixture() { + const current = await fixture(); + const next = structuredClone(current.initial); + next.local = { a: "b.md", b: "a.md", c: "nested/c.md" }; + next.documents.c = { materialized: false }; + const writes = { + c: { replacement: await toStoredSnapshot({ content: bytes("C") }) } + }; + const expected = new Map([ + ["a.md", bytes("B")], + ["b.md", bytes("A")], + ["nested/c.md", bytes("C")] + ]); + return { ...current, next, writes, expected }; +} + +async function directoryCycleFixture() { + const current = await fixture({ + a: { path: "a", text: "A" }, + b: { path: "b/deep/child", text: "B" } + }); + const next = structuredClone(current.initial); + next.local = { a: "b", b: "a/deep/child" }; + return { + ...current, + next, + writes: {}, + expected: new Map([ + ["b", bytes("A")], + ["a/deep/child", bytes("B")] + ]) + }; +} + +async function crossDirectoryFixture() { + const current = await fixture({ + a: { path: "src/a.md", text: "A" }, + // Keep src nonempty: deleting an empty source directory would mask an + // unflushed source unlink in the staging rename. + b: { path: "src/keep.md", text: "untouched sibling" } + }); + const next = structuredClone(current.initial); + next.local.a = "dst/a.md"; + const expected = new Map([ + ["dst/a.md", bytes("A")], + ["src/keep.md", bytes("untouched sibling")] + ]); + return { ...current, next, writes: {}, expected }; +} + +async function mergeFixture() { + const base = "first paragraph\n\nlast paragraph\n"; + const local = "local first paragraph\n\nlast paragraph\n"; + const remote = "first paragraph\n\nremote last paragraph\n"; + const current = await fixture({ a: { path: "note.md", text: local } }); + return { + ...current, + next: structuredClone(current.initial), + writes: { + a: { + expected: await toStoredSnapshot({ content: bytes(base) }), + replacement: await toStoredSnapshot({ content: bytes(remote) }) + } + }, + expected: new Map([ + [ + "note.md", + bytes("local first paragraph\n\nremote last paragraph\n") + ] + ]) + }; +} + +async function deletionFixture() { + const current = await fixture({ + a: { path: "a.bin", text: "private binary edit\0" }, + b: { path: "deleted.md", text: "unsent deleted bytes" } + }); + const next = structuredClone(current.initial); + delete next.local.b; + return { + ...current, + next, + writes: { + a: { + replacement: await toStoredSnapshot({ + content: bytes("remote binary\0") + }) + } + }, + expected: new Map([["a.bin", bytes("remote binary\0")]]) + }; +} + +async function permanentDestinationFixture() { + const current = await fixture(); + const next = structuredClone(current.initial); + next.local = { a: "unsupported/a.md", b: "good.md" }; + return { + ...current, + next, + permanentFault: (label: string) => { + if (label === "before:mkdir:unsupported") + throw Object.assign(new Error("Unsupported destination"), { + code: "ENAMETOOLONG" + }); + }, + writes: { + a: { + expected: await toStoredSnapshot({ content: bytes("A") }), + replacement: await toStoredSnapshot({ + content: bytes("REMOTE") + }) + } + }, + expectedLocal: { a: "Recovered a.md", b: "good.md" }, + expected: new Map([ + ["Recovered a.md", bytes("REMOTE")], + ["good.md", bytes("B")] + ]) + }; +} + +for (const [name, create] of [ + ["swap and create", swapFixture], + ["file/directory cycle", directoryCycleFixture], + ["cross-directory move", crossDirectoryFixture], + ["content merge", mergeFixture], + ["binary replacement and deletion", deletionFixture], + ["permanent destination fallback", permanentDestinationFixture] +] as const) { + test(`${name}: interrupted operations leave metadata usable by ordinary scans`, async (t) => { + const baseline = await create(); + const labels: string[] = []; + const trace = (label: string) => { + if (!label.startsWith("read:")) labels.push(label); + }; + baseline.disk.boundary = (label) => { + baseline.permanentFault(label); + trace(label); + }; + baseline.persistence.boundary = trace; + await baseline.restart().files.apply(baseline.next, baseline.writes); + assert.deepEqual(baseline.disk.userFiles(), baseline.expected); + assert.deepEqual( + baseline.persistence.snapshot().database!.local, + baseline.expectedLocal ?? baseline.next.local + ); + assert(labels.length > 0); + for (const powerLoss of [false, true]) + for (let cut = 0; cut < labels.length; cut++) { + await t.test( + `${powerLoss ? "power" : "process"} interruption at ${labels[cut]}`, + async () => { + const current = await create(); + let seen = 0; + let fired = false; + const interrupt = (label: string) => { + if (label.startsWith("read:")) return; + if (seen++ === cut) { + fired = true; + current.disk.crash(powerLoss); + throw new Error("interrupted"); + } + }; + current.disk.boundary = (label) => { + current.permanentFault(label); + interrupt(label); + }; + current.persistence.boundary = interrupt; + await assert.rejects( + current + .restart() + .files.apply(current.next, current.writes), + /interrupted/ + ); + assert(fired); + current.disk.boundary = () => {}; + current.persistence.boundary = () => {}; + // The user may edit after the interruption, including a partial + // write or a file whose metadata was never saved. + await current.disk.userWrite( + "after-restart.md", + bytes("user edit") + ); + const runtime = current.restart(); + const observed = current.disk.userFiles(); + await scanChanges(runtime, []); + assert.deepEqual( + current.disk.userFiles(), + observed, + "A scan must not replay old filesystem intentions" + ); + assertManifest(runtime.database.state.local); + assert.deepEqual( + new Set( + Object.values(runtime.database.state.local) + ), + new Set(observed.keys()) + ); + for (const [id, path] of Object.entries( + runtime.database.state.local + )) { + assert( + runtime.database.state.documents[id] + .materialized + ); + assert.equal( + runtime.database.state.documents[id] + .observedHash, + (await runtime.files.snapshot(path))!.hash + ); + } + assert( + (await runtime.files.listFilesRecursively()).every( + (path) => !path.startsWith(".vault-link-sync/") + ) + ); + const state = structuredClone(runtime.database.state); + await scanChanges(current.restart(), []); + assert.deepEqual( + current.persistence.snapshot().database, + state, + "Second startup must be stable" + ); + } + ); + } + }); +} + +test("production manifest merge obeys independently specified per-document policy", () => { + const paths = [undefined, "a.md", "b.md", "nested/c.md"]; + const map = (path: string | undefined): Record => + path === undefined + ? {} + : { "00000000-0000-4000-8000-000000000001": path }; + for (const base of paths) + for (const local of paths) + for (const remote of paths) { + assert.deepEqual( + mergeFileManifests(map(base), map(local), map(remote)), + map(pathDecision(base, local, remote)) + ); + } + const conflicts = mergeFileManifests( + {}, + { a: "folder" }, + { b: "folder/note.md" } + ); + assertManifest(conflicts); + assert.equal( + conflicts.b, + "folder/note.md", + "Canonical path must win collision" + ); + assert.equal( + Object.keys(conflicts).length, + 2, + "Collision must not discard an ID" + ); +}); + +test("exhaustive two-document manifest states preserve IDs and canonical collision priority", () => { + const ids = [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002" + ]; + const states: Record[] = []; + for (const a of [undefined, "a.md", "b.md"]) + for (const b of [undefined, "a.md", "b.md"]) { + if (a && a === b) continue; + states.push({ + ...(a ? { [ids[0]]: a } : {}), + ...(b ? { [ids[1]]: b } : {}) + }); + } + for (const base of states) + for (const local of states) + for (const remote of states) { + const expected = Object.fromEntries( + ids + .map((id) => [ + id, + pathDecision(base[id], local[id], remote[id]) + ]) + .filter(([, value]) => value !== undefined) + ); + const merged = mergeFileManifests(base, local, remote); + assertManifest(merged); + assert.deepEqual( + Object.keys(merged).sort(), + Object.keys(expected).sort(), + "Merge discarded or invented an identity" + ); + for (const id of ids) + if ( + expected[id] !== undefined && + remote[id] === expected[id] + ) + assert.equal( + merged[id], + remote[id], + "Canonical claimant lost its path" + ); + assert.deepEqual( + mergeFileManifests(remote, merged, remote), + merged, + "Manifest merge is not idempotent" + ); + } +}); + +test("scan failures cannot masquerade as deletions", async () => { + for (const point of ["read:list:", "read:snapshot:a.md"]) { + const current = await fixture(); + const runtime = current.restart(); + const original = structuredClone(runtime.database.state); + let fired = false; + current.disk.boundary = (label) => { + if (label === point) { + fired = true; + throw new Error("permission denied"); + } + }; + await assert.rejects( + scanLocalFiles({ + next: structuredClone(original), + changes: [], + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }), + /permission denied/ + ); + assert(fired); + assert.deepEqual(runtime.database.state, original); + assert.deepEqual(current.persistence.snapshot().database, original); + } +}); + +for (const phase of ["before", "durable"]) { + test(`delete/recreate identity survives ${phase}-save failure and retry`, async () => { + const current = await fixture(); + await current.disk.userDelete("a.md"); + await current.disk.userWrite("a.md", bytes("new incarnation")); + const changes: LocalChange[] = [ + { type: "delete", path: "a.md" }, + { type: "create", path: "a.md" } + ]; + const scan = async () => { + const runtime = current.restart(); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + return runtime.database.state; + }; + current.persistence.boundary = (label) => { + if (label === `${phase}:save`) throw new Error("save failure"); + }; + await assert.rejects(scan(), /save failure/); + current.persistence.boundary = () => {}; + const state = await scan(); + assert.equal( + state.local.a, + undefined, + "Retry reused the deleted document's UUID" + ); + assert.equal( + Object.values(state.local).filter((path) => path === "a.md").length, + 1 + ); + }); +} + +async function scanChanges( + runtime: { database: Database; files: FileOperations }, + changes: LocalChange[] +): Promise { + await runtime.database.recoverPersistence(); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); +} + +for (const editedContent of ["new bytes", "B"]) { + test(`explicit rename then edit to ${JSON.stringify(editedContent)} preserves both UUIDs`, async () => { + const current = await fixture(); + const runtime = current.restart(); + await current.disk.userRename("a.md", "moved.md"); + await current.disk.userWrite("moved.md", bytes(editedContent)); + const changes: LocalChange[] = [ + { type: "move", oldPath: "a.md", relativePath: "moved.md" } + ]; + await scanChanges(runtime, changes); + assert.deepEqual( + runtime.database.state.local, + { a: "moved.md", b: "b.md" }, + "Matching another live document's bytes must not steal its UUID" + ); + assert.deepEqual( + current.disk.userFiles(), + new Map([ + ["moved.md", bytes(editedContent)], + ["b.md", bytes("B")] + ]) + ); + assert.equal(changes.length, 0); + }); +} + +for (const equalContent of [false, true]) { + for (const phase of ["before", "durable"]) { + test(`${equalContent ? "equal" : "different"}-content swap retains UUIDs after ${phase}-save rejection`, async () => { + const current = await fixture({ + a: { path: "a.md", text: "A" }, + b: { path: "b.md", text: equalContent ? "A" : "B" } + }); + const runtime = current.restart(); + const changes: LocalChange[] = []; + for (const [oldPath, relativePath] of [ + ["a.md", "temporary.md"], + ["b.md", "a.md"], + ["temporary.md", "b.md"] + ]) { + await current.disk.userRename(oldPath, relativePath); + changes.push({ type: "move", oldPath, relativePath }); + } + const expected = { a: "b.md", b: "a.md" }; + const expectedFiles = new Map([ + ["a.md", bytes(equalContent ? "A" : "B")], + ["b.md", bytes("A")] + ]); + let fired = false; + current.persistence.boundary = (label) => { + if (!fired && label === `${phase}:save`) { + fired = true; + throw new Error("uncertain save"); + } + }; + await assert.rejects( + scanChanges(runtime, changes), + /uncertain save/ + ); + assert(fired, "The requested save failure must be exercised"); + assert.deepEqual( + current.persistence.snapshot().database!.local, + phase === "durable" ? expected : current.initial.local, + "The fault must occur on the intended side of the durable save" + ); + current.persistence.boundary = () => {}; + // Model an uncertain save in a live client: durable state reloads, + // but the original editor notifications remain in memory. Do not + // clear, reconstruct or deduplicate that queue in the harness. + await scanChanges(runtime, changes); + assert.deepEqual( + runtime.database.state.local, + expected, + "Retry replayed an already committed swap and reversed its identities" + ); + assert.equal(changes.length, 0); + await scanChanges(runtime, changes); + assert.deepEqual(runtime.database.state.local, expected); + const restarted = current.restart(); + await scanChanges(restarted, []); + assert.deepEqual(restarted.database.state.local, expected); + assert.deepEqual(current.disk.userFiles(), expectedFiles); + }); + } +} + +test("normalization-sensitive disk does not lose a distinct decomposed filename", async () => { + const current = await fixture(); + await current.disk.userWrite("é.md", bytes("composed")); + const runtime = current.restart(); + const next = structuredClone(runtime.database.state); + next.local.composed = "é.md"; + next.documents.composed = { materialized: true }; + await runtime.database.commit(next); + await current.disk.userWrite("e\u0301.md", bytes("decomposed")); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes: [], + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + assert.equal( + Object.keys(runtime.database.state.local).length, + 4, + "Distinct Unicode file vanished from the scan" + ); +}); diff --git a/frontend/test-support/restart-reconciliation.test.ts b/frontend/test-support/restart-reconciliation.test.ts new file mode 100644 index 00000000..897b3627 --- /dev/null +++ b/frontend/test-support/restart-reconciliation.test.ts @@ -0,0 +1,231 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +import { assertCanonical } from "./canonical"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +for (const powerLoss of [false, true]) + for (const operation of ["create", "replace", "move", "metadata"] as const) + test( + `${operation}: a ${powerLoss ? "power" : "process"} interruption and later user edits converge without replay`, + { timeout: 20_000 }, + async () => { + const f = await fixture(); + let peer: SyncClient | undefined; + try { + const id = randomUUID(); + const head = await f.put( + `/documents/${id}`, + f.content("BASE\n") + ); + const manifest = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "note.md" } + }); + if (operation !== "create") { + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + if (operation === "move") + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: manifest.fileManifestId, + entries: { [id]: "moved.md" } + }); + else + await f.put( + `/documents/${id}`, + f.content("REMOTE\nBASE\n", head.vaultUpdateId) + ); + } + let fired = false; + let wrote = false; + const interrupt = () => { + fired = true; + f.disk.crash(powerLoss); + throw new Error("injected interruption"); + }; + f.disk.boundary = (label) => { + if (fired) return; + if (label === "durable:write:note.md") wrote = true; + if ( + operation === "move" && + label === "visible:rename:note.md->moved.md" + ) + interrupt(); + if ( + (operation === "create" || + operation === "replace") && + label === "visible:write:note.md" + ) + interrupt(); + }; + f.store.boundary = (label) => { + if ( + !fired && + operation === "metadata" && + wrote && + label === "before:save" + ) + interrupt(); + }; + await assert.rejects( + operation === "create" + ? f.client.start() + : f.client.setSetting("isSyncEnabled", true), + /injected interruption/ + ); + assert( + fired, + "The intended filesystem/save boundary must be reached" + ); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + // Model torn content and further independent edits. Neither is + // recoverable from an old filesystem intention, nor should it be. + await f.disk.userWrite( + operation === "move" ? "moved.md" : "note.md", + bytes("partial\nUSER EDIT\n") + ); + await f.disk.userWrite( + "after.md", + bytes("AFTER INTERRUPTION\n") + ); + await f.reopenClient(); + await f.client.start(); + const inspectable = { + files: () => f.disk.userFiles(), + database: () => f.store.snapshot().database + }; + const peerDisk = new MemoryDisk(); + const peerStore = new MemoryPersistence({ + settings: f.store.snapshot().settings + }); + peer = await SyncClient.create({ + fs: peerDisk, + persistence: peerStore + }); + await peer.start(); + const clients = [ + inspectable, + { + files: () => peerDisk.userFiles(), + database: () => peerStore.snapshot().database + } + ]; + await assertCanonical( + clients, + f.url, + "test-token-change-me" + ); + assert.deepEqual( + f.disk.userFiles().get("after.md"), + bytes("AFTER INTERRUPTION\n") + ); + assert( + (await f.disk.listFilesRecursively()).every( + (path) => !path.startsWith(".vault-link-sync/") + ) + ); + const files = f.disk.userFiles(); + await f.reopenClient(); + await f.client.start(); + await assertCanonical( + clients, + f.url, + "test-token-change-me" + ); + assert.deepEqual( + f.disk.userFiles(), + files, + "A second restart must not replay old writes" + ); + } finally { + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await peer?.destroy(); + await f.dispose(); + } + } + ); + +for (const operation of ["create", "replace", "move"] as const) + test( + `${operation}: metadata can survive while completed user-file writes are lost`, + { timeout: 20_000 }, + async () => { + const f = await fixture(); + let peer: SyncClient | undefined; + try { + const id = randomUUID(); + const head = await f.put( + `/documents/${id}`, + f.content("BASE\n") + ); + const manifest = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "note.md" } + }); + if (operation !== "create") { + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + if (operation === "replace") + await f.put( + `/documents/${id}`, + f.content("REMOTE\nBASE\n", head.vaultUpdateId) + ); + else + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: manifest.fileManifestId, + entries: { [id]: "moved.md" } + }); + } + const oldDisk = f.disk.image(); + if (operation === "create") await f.client.start(); + else await f.client.setSetting("isSyncEnabled", true); + await f.client.destroy(); + // Metadata is newer than disk: completed filesystem calls have no + // flush contract. This is not covered by failing a call mid-flight. + f.disk.restore(oldDisk); + await f.disk.userWrite( + "after.md", + bytes("USER AFTER POWER LOSS\n") + ); + await f.reopenClient(); + await f.client.start(); + const peerDisk = new MemoryDisk(); + const peerStore = new MemoryPersistence({ + settings: f.store.snapshot().settings + }); + peer = await SyncClient.create({ + fs: peerDisk, + persistence: peerStore + }); + await peer.start(); + await assertCanonical( + [ + { + files: () => f.disk.userFiles(), + database: () => f.store.snapshot().database + }, + { + files: () => peerDisk.userFiles(), + database: () => peerStore.snapshot().database + } + ], + f.url, + "test-token-change-me" + ); + assert.deepEqual( + f.disk.userFiles().get("after.md"), + bytes("USER AFTER POWER LOSS\n") + ); + } finally { + await peer?.destroy(); + await f.dispose(); + } + } + ); diff --git a/frontend/test-support/server-restore.test.ts b/frontend/test-support/server-restore.test.ts new file mode 100644 index 00000000..d8598429 --- /dev/null +++ b/frontend/test-support/server-restore.test.ts @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +import { readdir, copyFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +for (const mode of [ + "same-version", + "behind", + "dirty", + "pending", + "legacy-state", + ...["before:save", "durable:save"].flatMap((boundary) => + [1, 2, 3, 4, 5].map((ordinal) => `${boundary}:${ordinal}`) + ) +] as const) + test(`restored history recovers ${mode} without false convergence or lost local work`, async () => { + const f = await fixture(); + const crashRecovery = mode.includes(":save:"); + const hasLocalEdit = + mode === "dirty" || mode === "pending" || crashRecovery; + try { + const id = randomUUID(); + const h = await f.put("/documents/" + id, f.content("BASE")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + const vaults = join(f.server.databaseDirectory, "vaults"); + const db = join( + vaults, + (await readdir(vaults)).find((p) => p.endsWith(".sqlite"))! + ); + const backup = db + ".backup"; + execFileSync("python3", [ + "-c", + "import sqlite3,sys; s=sqlite3.connect(sys.argv[1]); d=sqlite3.connect(sys.argv[2]); s.backup(d); d.close(); s.close()", + db, + backup + ]); + const old = await f.put( + "/documents/" + id, + f.content("OLD BRANCH", h.vaultUpdateId) + ); + await f.client.setSetting("isSyncEnabled", true); + await f.client.setSetting("isSyncEnabled", false); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(), + "OLD BRANCH" + ); + if (hasLocalEdit) { + await f.disk.userWrite("a.md", bytes("OLD BRANCH LOCAL")); + await f.client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + if (mode === "pending") { + // Lose an acknowledgement after the server committed, leaving a + // durable request that must not be retried against the restored DB. + f.setFetchHook(async (input, init) => { + const response = await fetch(input, init); + if (init?.method === "PUT") + throw new Error("lost acknowledgement"); + return response; + }); + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /lost acknowledgement/ + ); + await f.client.setSetting("isSyncEnabled", false); + f.setFetchHook(fetch); + } + await f.server.crash(); + await copyFile(backup, db); + await rm(db + "-wal", { force: true }); + await rm(db + "-shm", { force: true }); + await f.server.restart(); + const fork = + mode === "behind" + ? h + : await f.put( + "/documents/" + id, + f.content("NEW BRANCH", h.vaultUpdateId) + ); + if (mode !== "behind") + assert.equal(fork.vaultUpdateId, old.vaultUpdateId); + let interrupted = false, + mismatch = false; + if (crashRecovery) { + const boundary = mode.split(":").slice(0, 2).join(":"); + const ordinal = Number(mode.split(":")[2]); + let saves = 0; + f.setFetchHook(async (input, init) => { + const response = await fetch(input, init); + if ( + response.headers.get( + "x-vault-link-history-mismatch" + ) === "1" + ) + mismatch = true; + return response; + }); + f.store.boundary = (label) => { + if ( + mismatch && + !interrupted && + label === boundary && + ++saves === ordinal + ) { + // Sweep the scan, durable reset intent, checkpoint + // replacement and final bootstrap-enabling saves. + interrupted = true; + throw new Error("interrupted recovery save"); + } + }; + await assert.rejects( + f.client.setSetting("isSyncEnabled", true) + ); + assert(interrupted && mismatch); + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } + if (mode === "legacy-state") { + await f.client.destroy(); + const stored = f.store.snapshot() as ReturnType< + typeof f.store.snapshot + > & { historyCheckpoint?: unknown }; + delete stored.historyCheckpoint; + await f.store.save(stored); + await f.reopenClient(); + await f.client.start(); + } + await f.client.setSetting("isSyncEnabled", true); + const actual = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + const remote = Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(); + + assert.equal(actual, remote); + assert.equal(remote, mode === "behind" ? "BASE" : "NEW BRANCH"); + if (hasLocalEdit) { + const manifest = await f.get("/file-manifest"); + const recovered = Object.keys(manifest.entries).filter( + (other) => other !== id + ); + assert.equal(recovered.length, 1); + assert.equal( + Buffer.from( + (await f.get("/documents/" + recovered[0])) + .contentBase64, + "base64" + ).toString(), + "OLD BRANCH LOCAL" + ); + assert.equal( + Buffer.from( + f.disk.userFiles().get(manifest.entries[recovered[0]])! + ).toString(), + "OLD BRANCH LOCAL" + ); + } + await f.client.reset(); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(), + remote + ); + } finally { + await f.dispose(); + } + }); diff --git a/frontend/test-support/storage.ts b/frontend/test-support/storage.ts new file mode 100644 index 00000000..a7db333e --- /dev/null +++ b/frontend/test-support/storage.ts @@ -0,0 +1,310 @@ +import assert from "node:assert/strict"; +import type { + FileSnapshot, + FileSystemOperations, + StoredDatabase +} from "sync-client"; + +import type { StoredClient } from "sync-client"; +export type { StoredClient } from "sync-client"; +export type Boundary = (label: string) => void | Promise; +type Entry = FileSnapshot | null; // null is a directory; absence is not a directory. +export interface DiskImage { + visible: [string, Entry][]; + durable: [string, Entry][]; +} +const parent = (path: string) => path.split("/").slice(0, -1).join("/"); +export const isInternal = (path: string) => + path.normalize("NFC").toUpperCase().split("/")[0] === ".VAULT-LINK-SYNC"; + +/** The adapter and the editor are deliberately separate APIs. Engine writes do + * not generate editor notifications. All returned snapshots are owned copies. + * + * This models atomic namespace operations, not a particular filesystem's + * journal. At a visible boundary a power loss may roll back the operation; + * a process death alone does not roll back the kernel's dirty state. + */ +export class MemoryDisk implements FileSystemOperations { + private visible = new Map([["", null]]); + private durable = new Map([["", null]]); + private generation = 0; + public boundary: Boundary = () => {}; + + public image(): DiskImage { + return structuredClone({ + visible: [...this.visible], + durable: [...this.durable] + }); + } + + public restore(image: DiskImage): void { + this.visible = new Map(structuredClone(image.visible)); + this.durable = new Map(structuredClone(image.durable)); + } + + public crash(powerLoss = false): void { + this.generation++; + if (powerLoss) this.visible = structuredClone(this.durable); + } + + /** Fence an abandoned runtime, including calls it makes after a restart. */ + public session(): FileSystemOperations { + const generation = this.generation; + return new Proxy(this, { + get: (target, key) => { + const value = Reflect.get(target, key); + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + assert.equal( + this.generation, + generation, + "Access from crashed filesystem session" + ); + return value.apply(target, args); + }; + } + }); + } + + private check(path: string): void { + assert( + !path.startsWith("/") && + !path.includes("\\") && + !path.includes("\0"), + `Unsafe path: ${path}` + ); + assert( + path === "" || + path + .split("/") + .every( + (part) => part !== "" && part !== "." && part !== ".." + ), + `Unsafe path: ${path}` + ); + for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor)) + assert( + !this.visible.has(ancestor) || + this.visible.get(ancestor) === null, + `File ancestor: ${ancestor}` + ); + } + + private async change( + label: string, + paths: string[], + mutate: () => void + ): Promise { + const generation = this.generation; + await this.boundary(`before:${label}`); + assert.equal( + this.generation, + generation, + "Mutation resumed after crash" + ); + mutate(); + await this.boundary(`visible:${label}`); + assert.equal(this.generation, generation, "Flush resumed after crash"); + for (const path of paths) { + if (this.visible.has(path)) + this.durable.set( + path, + structuredClone(this.visible.get(path)!) + ); + else this.durable.delete(path); + } + await this.boundary(`durable:${label}`); + assert.equal( + this.generation, + generation, + "Operation returned after crash" + ); + } + + public async stat(path: string) { + this.check(path); + await this.boundary(`read:stat:${path}`); + const entry = this.visible.get(path); + return entry === undefined + ? undefined + : entry === null + ? { kind: "directory" as const, size: 0 } + : { kind: "file" as const, size: entry.content.length }; + } + + public async exists(path: string): Promise { + return (await this.stat(path)) !== undefined; + } + + public async readSnapshot(path: string): Promise { + this.check(path); + await this.boundary(`read:snapshot:${path}`); + const entry = this.visible.get(path); + assert(entry !== null, `Cannot read directory: ${path}`); + return structuredClone(entry); + } + + public async listFilesRecursively(root = ""): Promise { + this.check(root); + await this.boundary(`read:list:${root}`); + assert.equal( + this.visible.get(root), + null, + `Cannot list non-directory: ${root}` + ); + return [...this.visible] + .filter( + ([p, entry]) => + entry !== null && (!root || p.startsWith(`${root}/`)) + ) + .map(([p]) => p) + .sort(); + } + + public userFiles(): Map { + return new Map( + [...this.visible] + .filter(([p, e]) => e !== null && !isInternal(p)) + .map(([p, e]) => [p, e!.content.slice()]) + ); + } + + public async createDirectory(path: string): Promise { + this.check(path); + const paths: string[] = []; + for (let p = path; p; p = parent(p)) paths.unshift(p); + await this.change(`mkdir:${path}`, paths, () => { + for (const p of paths) + assert( + !this.visible.has(p) || this.visible.get(p) === null, + `File blocks directory: ${p}` + ); + for (const p of paths) this.visible.set(p, null); + }); + } + + public async write(path: string, snapshot: FileSnapshot): Promise { + this.check(path); + await this.change(`write:${path}`, [path], () => { + assert(!this.visible.has(path), `Destination exists: ${path}`); + assert.equal( + this.visible.get(parent(path)), + null, + `Parent missing: ${path}` + ); + this.visible.set(path, structuredClone(snapshot)); + }); + } + + public async rename(from: string, to: string): Promise { + this.check(from); + this.check(to); + await this.change(`rename:${from}->${to}`, [from, to], () => { + const source = this.visible.get(from); + assert(source, `Source is not a regular file: ${from}`); + assert(!this.visible.has(to), `Destination exists: ${to}`); + assert.equal( + this.visible.get(parent(to)), + null, + `Parent missing: ${to}` + ); + this.visible.set(to, source); + this.visible.delete(from); + }); + } + + public async delete(path: string): Promise { + this.check(path); + assert(path !== "", "Cannot remove vault root"); + const visiblePaths = [...this.visible.keys()].filter( + (p) => p === path || p.startsWith(`${path}/`) + ); + // A durable rmdir removes access to the whole subtree. A child whose + // unlink was visible but unflushed cannot reappear beneath a durably + // removed directory when this flat namespace is restored. + const paths = [ + ...new Set([ + ...visiblePaths, + ...[...this.durable.keys()].filter( + (p) => p === path || p.startsWith(`${path}/`) + ) + ]) + ]; + await this.change(`rmdir:${path}`, paths, () => { + for (const p of visiblePaths) + assert.equal( + this.visible.get(p), + null, + `Cannot unlink regular file: ${p}` + ); + for (const p of visiblePaths) this.visible.delete(p); + }); + } + + public async deleteFile(path: string): Promise { + this.check(path); + await this.change(`unlink:${path}`, [path], () => { + const entry = this.visible.get(path); + assert(entry !== null, `Cannot unlink directory: ${path}`); + this.visible.delete(path); + }); + } + + /** External editor mutations are durable by default and may replace files. */ + public async userWrite(path: string, content: Uint8Array): Promise { + this.check(path); + await this.createDirectory(parent(path)); + assert( + this.visible.get(path) !== null, + `Cannot overwrite directory: ${path}` + ); + this.visible.set(path, { content: content.slice() }); + this.durable.set(path, { content: content.slice() }); + } + + public async userDelete(path: string): Promise { + this.check(path); + assert(this.visible.get(path), `Cannot delete missing file: ${path}`); + this.visible.delete(path); + this.durable.delete(path); + } + + public async userRename(from: string, to: string): Promise { + this.check(from); + this.check(to); + await this.createDirectory(parent(to)); + const source = this.visible.get(from); + assert(source, `Source is not a regular file: ${from}`); + assert( + this.visible.get(to) !== null, + `Cannot overwrite directory: ${to}` + ); + // Editors may explicitly replace a destination. This is NOT the + // engine's rename API, which must never overwrite it. + this.visible.delete(from); + this.durable.delete(from); + this.visible.set(to, structuredClone(source)); + this.durable.set(to, structuredClone(source)); + } +} + +/** Atomic replacement with both definite failure and uncertain success. */ +export class MemoryPersistence { + private durable: StoredClient; + public boundary: Boundary = () => {}; + public constructor(initial: StoredClient = {}) { + this.durable = structuredClone(initial); + } + public async load(): Promise { + return structuredClone(this.durable); + } + public snapshot(): StoredClient { + return structuredClone(this.durable); + } + public async save(next: StoredClient): Promise { + const owned = structuredClone(next); + await this.boundary("before:save"); + this.durable = owned; + await this.boundary("durable:save"); + } +} diff --git a/frontend/test-support/sync-audit.test.ts b/frontend/test-support/sync-audit.test.ts new file mode 100644 index 00000000..d3c49e0c --- /dev/null +++ b/frontend/test-support/sync-audit.test.ts @@ -0,0 +1,878 @@ +import assert from "node:assert/strict"; +import { SUPPORTED_API_VERSION } from "../sync-client/src/consts"; +import { PermanentSyncError } from "../sync-client/src/errors/errors"; +import { test } from "node:test"; +import type { Database } from "../sync-client/src/persistence/database"; +import { emptyState } from "../sync-client/src/persistence/database"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +import { fixture, bytes, head } from "./sync-fixture"; + +test("ignored rename must not produce a duplicate submitted path", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + for (const [id, path] of Object.entries(f.database.state.local)) { + f.database.state.documents[id].base = { + ...head(id, 1, id.toUpperCase()), + hash: f.database.state.documents[id].observedHash! + }; + } + await f.disk.userRename("a.md", "ignored.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "ignored.md" + }); + await f.disk.userRename("b.md", "a.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "a.md" + }); + await f.settings.setSettings({ ignorePatterns: ["ignored.md"] }); + await f.internals.scan(); + f.internals.stopped = false; + await f.internals.preparePush(); + const p = f.database.state.pending; + assert(p?.type === "fileManifest"); + assert.equal( + new Set(Object.values(p.request.entries)).size, + Object.keys(p.request.entries).length + ); +}); + +test("ignored untracked ancestor must remain in place", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("remote") } + ); + await f.disk.userWrite("private", bytes("LOCAL PRIVATE")); + await f.settings.setSettings({ ignorePatterns: ["private"] }); + f.database.state.remoteHeads.a = head("a", 2, "remote"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "private/note.md" } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + assert.deepEqual(await f.disk.readSnapshot("private"), { + content: bytes("LOCAL PRIVATE") + }); +}); + +test("oversized untracked file must remain in place", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("R") } + ); + await f.disk.userWrite("a.md", bytes("LOCAL OVERSIZED")); + await f.settings.setSettings({ maxFileSizeMB: 5 / (1024 * 1024) }); + f.database.state.remoteHeads.a = head("a", 2, "R"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "a.md" } + }); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("LOCAL OVERSIZED") + ); +}); + +test("rejected unknown success must use submitted content as merge base", async () => { + const f = await fixture( + { "a.md": "base" }, + { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes( + vaultUpdateId === 1 + ? "base" + : vaultUpdateId === 2 + ? "base A" + : "base A REMOTE" + ) + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("base A")); + f.internals.stopped = false; + await f.internals.preparePush(); + const req = f.database.state.pending!.request.requestId; + await f.internals.finishPending(); + await f.disk.userWrite("a.md", bytes("base")); + await f.internals.incorporateEventBatch({ + headEventId: 3, + events: [ + { + eventId: 2, + requestId: req, + type: "content", + document: head("a", 2, "base A") + }, + { + eventId: 3, + requestId: "other", + type: "content", + document: head("a", 3, "base A REMOTE") + } + ] + }); + const result = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(result, "base REMOTE"); +}); + +test("bootstrap adopts same-path identity even after multiple notifications", async () => { + const f = await fixture({}); + f.database.state.initialized = false; + f.database.state.fileManifest = { fileManifestId: 0, entries: {} }; + await f.disk.userWrite("a.md", bytes("local a")); + await f.syncer.syncLocallyCreatedFile("a.md"); + await f.disk.userWrite("b.md", bytes("local b")); + await f.syncer.syncLocallyCreatedFile("b.md"); + const initial = { + headEventId: 2, + fileManifest: { fileManifestId: 2, entries: { remoteA: "a.md" } }, + documents: [head("remoteA", 1, "remote a")] + }; + await f.internals.scan(initial); + assert.equal(f.database.state.local.remoteA, "a.md"); +}); + +test("oversized remote head must not cause identical stale uploads forever", async () => { + let attempts = 0; + const remoteText = "R".repeat(20); + const f = await fixture( + { "a.md": "edit" }, + { + putFileContent: async () => { + attempts++; + return { + type: "StaleBase", + ...head("a", 2, remoteText), + contentBase64: Buffer.from(remoteText).toString("base64") + }; + } + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 2, remoteText); + await f.settings.setSettings({ maxFileSizeMB: 10 / (1024 * 1024) }); + f.internals.stopped = false; + // Even a small local edit must wait until its required remote base is readable. + assert.equal(await f.internals.preparePush(), false); + assert.equal(attempts, 0); + assert.equal(f.database.state.pending, undefined); +}); + +test("local undo of accepted-but-rejected manifest must survive replay", async () => { + const f = await fixture( + { "a.md": "A" }, + { + pushFileManifest: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async () => bytes("A") + } + ); + f.database.state.lastSeenUpdateId = 1; + f.database.state.documents.a.base = { + ...head("a", 1, "A"), + hash: f.database.state.documents.a.observedHash! + }; + await f.disk.userRename("a.md", "b.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "b.md" + }); + await f.internals.scan(); + f.internals.stopped = false; + await f.internals.preparePush(); + const req = f.database.state.pending!.request.requestId; + f.internals.stopped = true; + await f.internals.finishPending(); + await f.disk.userRename("b.md", "a.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "a.md" + }); + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId: req, + type: "fileManifest", + fileManifest: { fileManifestId: 2, entries: { a: "b.md" } } + } + ] + }); + assert.equal(f.database.state.local.a, "a.md"); +}); + +test("a settings update resumes the enabled engine", async () => { + const f = await settingsLifecycleFixture(); + try { + await f.client.setSettings({ maxFileSizeMB: 20 }); + assert.equal(f.client.getSettings().maxFileSizeMB, 20); + assert.equal(f.store.snapshot().settings?.maxFileSizeMB, 20); + const before = f.reads(); + await f.client.syncLocallyUpdatedFile({ relativePath: "absent.md" }); + await f.client.waitUntilFinished(); + assert( + f.reads() > before, + "notifications must still wake the enabled engine" + ); + } finally { + await f.client.destroy(); + } +}); + +test("a dirty cursor update must invalidate old precise positions", async () => { + const { CursorTracker } = await import( + "../sync-client/src/sync-operations/cursor-tracker" + ); + const { FileChangeNotifier } = await import( + "../sync-client/src/sync-operations/file-change-notifier" + ); + const f = await fixture({ "a.md": "base" }); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + const ws = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: () => {} + }; + const tracker = new CursorTracker( + f.database, + ws as unknown as WebSocketManager, + f.files, + new FileChangeNotifier() + ); + let latest: import("../sync-client/src/types/maybe-outdated-client-cursors").MaybeOutdatedClientCursors[] = + []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + latest = cursors; + }); + const clean = { + userName: "other", + deviceId: "other", + documentsWithCursors: [ + { + document_id: "a", + relative_path: "a.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 0 }] + } + ] + }; + await ws.onRemoteCursorsUpdateReceived.triggerAsync([clean]); + await ws.onRemoteCursorsUpdateReceived.triggerAsync([ + { + ...clean, + documentsWithCursors: [ + { + ...clean.documentsWithCursors[0], + vault_update_id: null, + cursors: [{ start: 100, end: 100 }] + } + ] + } + ]); + assert.equal(latest[0]?.isOutdated, true); + tracker.reset(); +}); + +test("silent open WebSocket must eventually trigger catchup or reconnect", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout", "setInterval"] }); + let opens = 0, + eventReads = 0; + class SilentWebSocket { + static OPEN = 1; + readyState = 1; + onopen: ((event: object) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = + null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: ((event: object) => void) | null = null; + constructor(_url: string | URL) { + opens++; + queueMicrotask(() => this.onopen?.({})); + } + send(_message: string) {} + close() { + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + const store = new MemoryPersistence({ + settings: { + isSyncEnabled: true, + remoteUri: "http://silent.test", + vaultName: "test", + token: "test", + syncIntervalMs: 0 + } + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: store, + webSocket: SilentWebSocket as unknown as typeof WebSocket, + fetch: async (input) => { + const path = String(input); + if (path.endsWith("/ping")) + return Response.json({ + supportedApiVersion: SUPPORTED_API_VERSION, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (path.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + if (path.includes("/events-since")) { + eventReads++; + return Response.json({ headEventId: 0, events: [] }); + } + throw new Error("Unexpected request " + path); + } + }); + try { + await client.start(); + const before = eventReads; + context.mock.timers.tick(45_000); + context.mock.timers.tick(3500); + for (let i = 0; i < 30; i++) await Promise.resolve(); + assert(opens > 1 || eventReads > before); + } finally { + await client.destroy(); + } +}); + +test("oversized files should be filtered before reading complete snapshots", async () => { + const f = await fixture({}); + await f.disk.userWrite("huge.bin", bytes("too big")); + await f.settings.setSettings({ maxFileSizeMB: 1 / (1024 * 1024) }); + let reads = 0; + f.disk.boundary = (label) => { + if (label === "read:snapshot:huge.bin") reads++; + }; + await f.internals.scan(); + assert.equal(reads, 0); +}); + +test("permanent destination failure replans the affected file and finishes the batch", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + f.disk.boundary = (label) => { + if (label === "before:mkdir:unsupported") + throw Object.assign(new Error("filesystem path limit"), { + code: "ENAMETOOLONG" + }); + }; + const next = structuredClone(f.database.state); + next.local = { a: "unsupported/a.md", b: "good.md" }; + next.fileManifest = { fileManifestId: 2, entries: { ...next.local } }; + await f.files.apply(next); + await f.settings.setSettings({ ignorePatterns: ["unsupported/**"] }); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((b) => Buffer.from(b).toString()) + .sort(), + ["A", "B"] + ); +}); + +test("conflict allocation terminates when a truncated extension ends in a space", async () => { + const { spawnSync } = await import("node:child_process"); + const modulePath = require.resolve( + "../sync-client/src/utils/portable-path" + ); + const result = spawnSync( + process.execPath, + [ + "--require", + require.resolve("tsx/cjs"), + "-e", + ` + const {allocatePortablePath,validatePortablePaths} = require(${JSON.stringify(modulePath)}); + const name = 'a.' + 'x'.repeat(62) + ' ' + 'x'.repeat(100); + const allocated = allocatePortablePath(name, 'new', {old:name}); + validatePortablePaths([name, allocated]); + if (Buffer.byteLength(allocated) > 255) process.exit(2); + ` + ], + { timeout: 3000, encoding: "utf8" } + ); + assert.equal(result.status, 0, String(result.error ?? result.stderr)); +}); + +for (const interrupted of [false, true]) { + test(`paged catchup folds edits and reversions${interrupted ? " again after an interrupted fetch" : ""}`, async () => { + const first = { + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId: "remote-1", + type: "content" as const, + document: head("a", 2, "base REMOTE") + } + ] + }; + const last = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "remote-2", + type: "content" as const, + document: head("a", 3, "base") + } + ] + }; + let fail = interrupted; + let f: Awaited>; + const fetchedAfter: number[] = []; + const service: Partial = { + getDocumentVersionContent: async () => bytes("base"), + events: async (after) => { + fetchedAfter.push(after); + if (after === 1) return first; + assert.equal(after, 2); + assert.equal(f.database.state.lastSeenUpdateId, 1); + assert.deepEqual( + f.persistence.snapshot(), + saved, + "partial pages must not save metadata" + ); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); + if (fail) { + fail = false; + throw new Error("interrupted page fetch"); + } + return last; + } + }; + f = await fixture({ "a.md": "base LOCAL" }, service); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 1, "base"); + f.database.state.lastSeenUpdateId = 1; + await f.database.save(); + const saved = f.persistence.snapshot(); + const catchUp = async () => + f.internals.incorporateEventBatch( + await service.events!(f.database.state.lastSeenUpdateId) + ); + if (interrupted) { + await assert.rejects(catchUp(), /interrupted page fetch/); + f.disk.crash(true); + f = await fixture({}, service, f); + } + await catchUp(); + assert.deepEqual(fetchedAfter, interrupted ? [1, 2, 1, 2] : [1, 2]); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); + }); +} + +test("an ignored occupant arriving after the scan keeps its path and privacy", async () => { + let f: Awaited>; + f = await fixture( + {}, + { + getDocumentVersionContent: async () => { + await f.disk.userWrite("private", bytes("SECRET")); + return bytes("remote"); + } + } + ); + await f.settings.setSettings({ ignorePatterns: ["private"] }); + f.database.state.remoteHeads.a = head("a", 2, "remote"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "private/note.md" } + }); + assert.deepEqual( + (await f.disk.readSnapshot("private"))?.content, + bytes("SECRET") + ); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert( + pending?.type !== "content" || + Buffer.from(pending.snapshot.contentBase64, "base64").toString() !== + "SECRET" + ); +}); + +test("content notifications during every snapshot do not starve a complete namespace scan", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let notifications = 0; + f.disk.boundary = async (label) => { + if ( + label.startsWith("read:snapshot:") && + !label.includes(".vault-link-sync") + ) { + notifications++; + await f.syncer.syncLocallyUpdatedFile({ + relativePath: label.slice("read:snapshot:".length) + }); + } + }; + await f.internals.scan(); + assert.equal(notifications, 2); + assert.deepEqual(f.database.state.local, { a: "a.md", b: "b.md" }); +}); + +for (const boundary of ["before:save", "durable:save"]) { + test(`receipt incorporation after paged replay survives ${boundary} and process replacement`, async () => { + const service: Partial = { + events: async (after) => { + assert.equal(after, 2); + return page; + }, + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes( + vaultUpdateId === 1 + ? "base" + : vaultUpdateId === 2 + ? "base A" + : "base A REMOTE" + ) + }; + let f = await fixture({ "a.md": "base" }, service); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("base A")); + f.internals.stopped = false; + await f.internals.preparePush(); + const { requestId } = f.database.state.pending!.request; + await f.internals.finishPending(); + await f.disk.userWrite("a.md", bytes("base")); + const first = { + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content" as const, + document: head("a", 2, "base A") + } + ] + }; + const page = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "other", + type: "content" as const, + document: head("a", 3, "base A REMOTE") + } + ] + }; + f.persistence.boundary = (label) => { + if (label === boundary) throw new Error("injected crash"); + }; + await assert.rejects( + f.internals.incorporateEventBatch(first), + /injected crash/ + ); + f.persistence.boundary = () => {}; + f.disk.crash(true); + f = await fixture({}, service, f); + assert.equal(f.database.state.lastSeenUpdateId, 1); + await f.internals.incorporateEventBatch(first); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base REMOTE") + ); + }); +} + +test("initial vault settings bind the engine and survive restart", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence({ + settings: { + remoteUri: "http://test", + vaultName: "old", + isSyncEnabled: false + } + }); + class Socket { + static OPEN = 1; + readyState = 3; + onopen = null; + onclose = null; + onerror = null; + onmessage = null; + send() {} + close() {} + } + const options = { + fs: disk, + persistence, + webSocket: Socket as unknown as typeof WebSocket, + fetch: async (input: RequestInfo | URL) => { + if (String(input).endsWith("/ping")) + return Response.json({ + supportedApiVersion: SUPPORTED_API_VERSION, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (String(input).endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + return Response.json({ headEventId: 0, events: [] }); + } + }; + let client = await SyncClient.create(options); + try { + await client.setSettings({ vaultName: "new", isSyncEnabled: true }); + await client.start(); + await client.destroy(); + client = await SyncClient.create(options); + assert.equal(client.getSettings().vaultName, "new"); + await client.start(); + } finally { + await client.destroy(); + } +}); + +test("disabling sync persists across reset without resuming the transport", async () => { + const f = await settingsLifecycleFixture(); + try { + const before = f.reads(); + await f.client.setSettings({ isSyncEnabled: false }); + assert.equal(f.store.snapshot().settings?.isSyncEnabled, false); + await f.client.reset(); + assert.equal(f.client.getSettings().isSyncEnabled, false); + assert.equal(f.reads(), before); + } finally { + await f.client.destroy(); + } +}); + +for (const receipt of [false, true]) { + test(`${receipt ? "replayed" : "direct"} upload acknowledgement survives a local deletion`, async () => { + const f = await fixture( + { "a.md": "base" }, + { + putFileContent: async () => { + if (receipt) + throw new PermanentSyncError( + "HTTP 413 after lost success" + ); + return { type: "Accepted", ...head("a", 2, "edited") }; + } + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("edited")); + f.internals.stopped = false; + await f.internals.preparePush(); + const { requestId } = f.database.state.pending!.request; + if (receipt) await f.internals.finishPending(); + // Recovery folds queued deletions into the durable namespace before + // an outstanding request or its delayed receipt is handled. + f.internals.stopped = true; + await f.disk.userDelete("a.md"); + await f.syncer.syncLocallyDeletedFile("a.md"); + await f.internals.scan(); + assert.deepEqual(f.database.state.local, {}); + if (receipt) { + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "edited") + } + ] + }); + assert.equal(f.database.state.lastSeenUpdateId, 2); + assert.equal(f.database.state.unconfirmed?.length ?? 0, 0); + } else await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.deepEqual(f.database.state.local, {}); + assert.equal(await f.disk.readSnapshot("a.md"), undefined); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + }); +} + +async function settingsLifecycleFixture() { + let eventReads = 0; + let onClose = () => {}; + class Socket { + static OPEN = 1; + readyState = 1; + onopen: (() => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = + null; + constructor() { + queueMicrotask(() => this.onopen?.()); + } + send() {} + close() { + onClose(); + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + const store = new MemoryPersistence({ + settings: { + remoteUri: "http://test", + vaultName: "test", + isSyncEnabled: true + } + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: store, + webSocket: Socket as unknown as typeof WebSocket, + fetch: async (input) => { + const path = String(input); + if (path.endsWith("/ping")) + return Response.json({ + supportedApiVersion: SUPPORTED_API_VERSION, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (path.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + if (path.includes("/events-since")) { + eventReads++; + return Response.json({ headEventId: 0, events: [] }); + } + throw new Error("Unexpected request " + path); + } + }); + await client.start(); + return { + client, + store, + reads: () => eventReads, + onClose: (fn: () => void) => { + onClose = fn; + } + }; +} + +for (const boundary of ["before:save", "durable:save"]) { + test(`settings recovery resumes after the rejection-reset save fails at ${boundary}`, async () => { + const f = await settingsLifecycleFixture(); + try { + const { database } = f.client as unknown as { database: Database }; + await database.commit({ + ...database.state, + rejectedManifest: { + entries: { retired: "old.md" }, + message: "retired rejection" + } + }); + let saves = 0, + failed = false; + f.store.boundary = (label) => { + if (label === "before:save") saves++; + if (saves === 2 && label === boundary && !failed) { + failed = true; + throw new Error("rejection-reset save failed"); + } + }; + await assert.rejects( + f.client.setSettings({ maxFileSizeMB: 20 }), + /rejection-reset save failed/ + ); + assert(failed); + assert.equal(f.client.getSettings().maxFileSizeMB, 20); + const before = f.reads(); + await f.client.syncLocallyUpdatedFile({ + relativePath: "absent.md" + }); + await f.client.waitUntilFinished(); + assert( + f.reads() > before, + "the enabled engine must resume after any settings-phase save failure" + ); + } finally { + await f.client.destroy(); + } + }); +} + +test("a caller cannot mutate settings while transports are being paused", async () => { + const f = await settingsLifecycleFixture(); + const change = { + maxFileSizeMB: 20, + vaultName: "test", + ignorePatterns: ["private/**"] + }; + try { + f.onClose(() => { + change.vaultName = "wrong-vault"; + change.ignorePatterns.push("**"); + }); + await f.client.setSettings(change); + assert.equal(f.client.getSettings().vaultName, "test"); + assert.deepEqual(f.client.getSettings().ignorePatterns, ["private/**"]); + assert.equal( + f.store.snapshot().database?.vaultKey, + JSON.stringify(["http://test", "test"]) + ); + } finally { + await f.client.destroy(); + } +}); diff --git a/frontend/test-support/sync-fixture.ts b/frontend/test-support/sync-fixture.ts new file mode 100644 index 00000000..79047f5d --- /dev/null +++ b/frontend/test-support/sync-fixture.ts @@ -0,0 +1,130 @@ +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import type { LocalChange } from "../sync-client/src/sync-operations/local-changes"; + +export const bytes = (text: string) => new TextEncoder().encode(text); +export const head = (id: string, version: number, content: string) => ({ + documentId: id, + vaultUpdateId: version, + contentSize: bytes(content).length, + updatedDate: "2026-09-19T00:00:00Z", + userId: "user", + deviceId: "device" +}); + +export async function fixture( + contents: Record = { "a.md": "local" }, + service: Partial = {}, + recovered?: { disk: MemoryDisk; persistence: MemoryPersistence } +) { + const disk = recovered?.disk ?? new MemoryDisk(); + const initial = recovered + ? (recovered.persistence.snapshot().database as StoredDatabase) + : emptyState("test"); + if (!recovered) { + initial.initialized = true; + for (const [path, content] of Object.entries(contents)) { + await disk.userWrite(path, bytes(content)); + const id = path.split(".")[0]; + initial.local[id] = path; + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot({ content: bytes(content) }) + ).hash + }; + } + initial.fileManifest = { + fileManifestId: 1, + entries: { ...initial.local } + }; + } + const persistence = + recovered?.persistence ?? new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + initial, + async (next) => persistence.save({ database: next }), + "test", + async () => (await persistence.load()).database as StoredDatabase + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + const changes: LocalChange[] = []; + const files = new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} }, + (path, size) => settings.isIgnored(path) || settings.isOversized(size) + ); + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteVaultUpdateReceived: new EventListeners() + } as unknown as WebSocketManager; + const contentCache = new FixedSizeDocumentCache(1024); + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket, + files, + config, + new SyncHistory(logger), + contentCache, + { entries: changes, save: async () => {} } + ); + const internals = syncer as unknown as { + finishPending: () => Promise; + preparePush: () => Promise; + scan: ( + initial?: import("../sync-client/src/services/types/VaultSnapshot").VaultSnapshot + ) => Promise; + incorporateEventBatch: ( + batch: import("../sync-client/src/services/types/EventBatch").EventBatch + ) => Promise; + stopped: boolean; + incorporateFileManifest: (remote: { + fileManifestId: number; + entries: Record; + }) => Promise; + incorporateContent: (remote: ReturnType) => Promise; + }; + return { + disk, + database, + files, + settings, + syncer, + internals, + persistence, + changes, + contentCache + }; +} diff --git a/frontend/test-support/sync-server-fixture.ts b/frontend/test-support/sync-server-fixture.ts new file mode 100644 index 00000000..8e0d438f --- /dev/null +++ b/frontend/test-support/sync-server-fixture.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { randomUUID } from "node:crypto"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +const root = resolve(__dirname, "../.."); +const token = "test-token-change-me"; +export const bytes = (s: string) => new TextEncoder().encode(s); +const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "Device-Id": "audit" +}; +export async function fixture() { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + await server.start(); + const vault = randomUUID(), + url = server.remoteUri + "/vaults/" + vault; + const put = async (path: string, data: unknown) => { + const r = await fetch(url + path, { + method: "PUT", + headers, + body: JSON.stringify(data) + }); + assert(r.ok, await r.clone().text()); + return r.json(); + }; + const get = async (path: string) => { + const r = await fetch(url + path, { headers }); + assert(r.ok); + return r.json(); + }; + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0, + networkRetryIntervalMs: 0, + requestTimeoutMs: 5000 + } + }); + let fetchHook: typeof fetch | undefined; + const createClient = () => + SyncClient.create({ + fs: disk, + persistence: store, + fetch: (...args) => + fetchHook ? fetchHook(...args) : fetch(...args) + }); + let client = await createClient(); + const content = (text: string, parentVersionId: number | null = null) => ({ + requestId: randomUUID(), + parentVersionId, + content: { + type: "Snapshot", + value: Buffer.from(text).toString("base64") + } + }); + return { + server, + disk, + store, + get client() { + return client; + }, + reopenClient: async () => { + await client.destroy(); + client = await createClient(); + }, + url, + headers, + put, + get, + content, + setFetchHook: (hook: typeof fetch) => { + fetchHook = hook; + }, + dispose: async () => { + await client.destroy(); + await server.stop(); + } + }; +} + +export { randomUUID }; diff --git a/frontend/test-support/tsconfig.json b/frontend/test-support/tsconfig.json new file mode 100644 index 00000000..42eb5312 --- /dev/null +++ b/frontend/test-support/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "skipLibCheck": true, + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "node", + "esModuleInterop": true, + "types": ["node"], + "lib": ["DOM", "ES2024"] + }, + "include": ["*.ts"] +} diff --git a/logs/harness-review/deterministic-results.json b/logs/harness-review/deterministic-results.json new file mode 100644 index 00000000..7ac2c0b3 --- /dev/null +++ b/logs/harness-review/deterministic-results.json @@ -0,0 +1,1121 @@ +[ + { + "name": "v4-binary-bytes-and-independent-identities", + "result": { + "success": true, + "duration": 100 + } + }, + { + "name": "v4-three-file-cycle-preserves-identities", + "result": { + "success": true, + "duration": 170 + } + }, + { + "name": "v4-case-alias-and-file-directory-conflicts", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "v4-create-before-commit-network-loss", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "v4-create-after-commit-network-loss", + "result": { + "success": true, + "duration": 64 + } + }, + { + "name": "v4-content-before-commit-network-loss", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "v4-content-after-commit-network-loss", + "result": { + "success": true, + "duration": 91 + } + }, + { + "name": "v4-manifest-before-commit-network-loss", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "v4-manifest-after-commit-network-loss", + "result": { + "success": true, + "duration": 72 + } + }, + { + "name": "rename-create-conflict", + "result": { + "success": true, + "duration": 121 + } + }, + { + "name": "rename-chain", + "result": { + "success": true, + "duration": 52 + } + }, + { + "name": "rename-update-conflict", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "delete-rename-conflict", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "multi-file-operations", + "result": { + "success": true, + "duration": 129 + } + }, + { + "name": "delete-recreate-same-path", + "result": { + "success": true, + "duration": 63 + } + }, + { + "name": "offline-rename-and-edit", + "result": { + "success": true, + "duration": 79 + } + }, + { + "name": "simultaneous-create-delete-same-path", + "result": { + "success": true, + "duration": 54 + } + }, + { + "name": "sequential-create-duplicate-content", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "mc-three-client-rename-offline-update", + "result": { + "success": true, + "duration": 102 + } + }, + { + "name": "mc-multi-delete-offline-rename", + "result": { + "success": true, + "duration": 115 + } + }, + { + "name": "mc-cross-create-rename-same-target", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "mc-delete-then-offline-rename", + "result": { + "success": true, + "duration": 79 + } + }, + { + "name": "offline-mixed-operations", + "result": { + "success": true, + "duration": 117 + } + }, + { + "name": "offline-concurrent-renames", + "result": { + "success": true, + "duration": 76 + } + }, + { + "name": "offline-multiple-edits", + "result": { + "success": true, + "duration": 74 + } + }, + { + "name": "rename-swap", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "rename-circular", + "result": { + "success": true, + "duration": 113 + } + }, + { + "name": "rename-roundtrip", + "result": { + "success": true, + "duration": 95 + } + }, + { + "name": "offline-rename-remote-create-old-path", + "result": { + "success": true, + "duration": 83 + } + }, + { + "name": "offline-edit-remote-rename", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "rename-chain-then-delete", + "result": { + "success": true, + "duration": 86 + } + }, + { + "name": "offline-delete-remote-rename", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "overlapping-edits-same-section", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "rapid-updates-after-merge", + "result": { + "success": true, + "duration": 117 + } + }, + { + "name": "delete-recreate-concurrent-update", + "result": { + "success": true, + "duration": 90 + } + }, + { + "name": "move-and-concurrent-remote-update", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "double-offline-cycle", + "result": { + "success": true, + "duration": 104 + } + }, + { + "name": "offline-update-both-then-delete-one", + "result": { + "success": true, + "duration": 100 + } + }, + { + "name": "offline-create-same-path-mergeable", + "result": { + "success": true, + "duration": 68 + } + }, + { + "name": "three-client-rename-create-delete", + "result": { + "success": true, + "duration": 106 + } + }, + { + "name": "rename-to-path-of-unconfirmed-delete", + "result": { + "success": true, + "duration": 94 + } + }, + { + "name": "offline-edit-then-move-same-content", + "result": { + "success": true, + "duration": 92 + } + }, + { + "name": "delete-recreate-different-content", + "result": { + "success": true, + "duration": 102 + } + }, + { + "name": "offline-move-then-remote-delete", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "reset-clears-recently-deleted-resurrection", + "result": { + "success": true, + "duration": 75 + } + }, + { + "name": "move-then-delete-stale-path", + "result": { + "success": true, + "duration": 65 + } + }, + { + "name": "offline-delete-vs-remote-update", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "update-does-not-survive-remote-delete", + "result": { + "success": true, + "duration": 64 + } + }, + { + "name": "move-preserves-remote-update", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "recently-deleted-cleared-on-reconnect", + "result": { + "success": true, + "duration": 107 + } + }, + { + "name": "watermark-advances-on-skip", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "watermark-gap-remote-update-not-recorded", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "queue-reset-loses-coalesced-local-edit", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "rename-to-pending-path-fallback", + "result": { + "success": true, + "duration": 55 + } + }, + { + "name": "move-remote-update-reverts-rename", + "result": { + "success": true, + "duration": 99 + } + }, + { + "name": "local-edit-lost-during-create-merge", + "result": { + "success": true, + "duration": 52 + } + }, + { + "name": "create-rename-response-skips-file", + "result": { + "success": true, + "duration": 73 + } + }, + { + "name": "online-create-rename-concurrent-create-orphan", + "result": { + "success": true, + "duration": 69 + } + }, + { + "name": "concurrent-rename-first-wins", + "result": { + "success": true, + "duration": 90 + } + }, + { + "name": "binary-to-text-transition", + "result": { + "success": true, + "duration": 151 + } + }, + { + "name": "text-pending-create-not-displaced", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "binary-pending-create-not-displaced", + "result": { + "success": true, + "duration": 80 + } + }, + { + "name": "coalesce-update-remote-update-data-loss", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "coalesced-remote-update-watermark-loss", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "concurrent-delete-during-remote-update", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "concurrent-edit-exact-same-position", + "result": { + "success": true, + "duration": 97 + } + }, + { + "name": "concurrent-rename-and-create-at-target-rename-first", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "concurrent-rename-and-create-at-target-create-first", + "result": { + "success": true, + "duration": 109 + } + }, + { + "name": "concurrent-rename-same-target", + "result": { + "success": true, + "duration": 110 + } + }, + { + "name": "concurrent-update-diff-consistency", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "user-parenthesized-file-not-deleted", + "result": { + "success": true, + "duration": 116 + } + }, + { + "name": "create-delete-noop", + "result": { + "success": true, + "duration": 30 + } + }, + { + "name": "create-merge-delete", + "result": { + "success": true, + "duration": 73 + } + }, + { + "name": "move-identical-content-ambiguity", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "create-during-reconciliation", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "create-merge-preserves-renamed-update", + "result": { + "success": true, + "duration": 106 + } + }, + { + "name": "create-rename-create-same-path", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "move-chain-three-files", + "result": { + "success": true, + "duration": 116 + } + }, + { + "name": "delete-by-other-client-then-recreate", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "online-delete-recreate-rapid-cycle", + "result": { + "success": true, + "duration": 169 + } + }, + { + "name": "online-edit-vs-delete-convergence", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "rapid-edit-delete-online-convergence", + "result": { + "success": true, + "duration": 144 + } + }, + { + "name": "online-both-create-same-path-deconflict", + "result": { + "success": true, + "duration": 92 + } + }, + { + "name": "online-create-update-while-other-creates-same-path", + "result": { + "success": true, + "duration": 82 + } + }, + { + "name": "displaced-file-not-marked-deleted", + "result": { + "success": true, + "duration": 112 + } + }, + { + "name": "remote-update-resurrects-deleted-doc", + "result": { + "success": false, + "error": "Convergence timed out after 60000ms: Client 1 differs from client 0 (paths or bytes)\n+ actual - expected\n\n+ [\n+ [\n+ 'P.md',\n+ Uint8Array(17) [\n+ 118,\n+ 50,\n+ 49,\n+ 32,\n+ 99,\n+ 111,\n+ 110,\n+ 116,\n+ 101,\n+ 110,\n+ 116,\n+ 32,\n+ 40,\n+ 68,\n+ 50,\n+ 41,\n+ 10\n+ ]\n+ ]\n+ ]\n- []\n", + "diagnostics": [ + { + "client": 0, + "database": { + "vaultKey": "[\"http://localhost:62272\",\"test-df2b81ab-03e9-432a-878a-d6aa1c6c9147\"]", + "initialized": true, + "fileManifest": { + "fileManifestId": 4, + "entries": {} + }, + "local": {}, + "documents": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "materialized": true, + "base": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40, + "hash": "229b36522c531631df2139e070b94a0e1b8d8833ae49d4662cb0d7a8bd158e9b" + }, + "bootstrap": false, + "observedHash": "229b36522c531631df2139e070b94a0e1b8d8833ae49d4662cb0d7a8bd158e9b" + } + }, + "remoteHeads": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40 + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17 + } + }, + "lastSeenUpdateId": 5 + }, + "disk": { + "visible": [ + [ + "", + null + ], + [ + ".vault-link-sync", + null + ], + [ + ".vault-link-sync/transactions", + null + ] + ], + "durable": [ + [ + "", + null + ], + [ + ".vault-link-sync", + null + ], + [ + ".vault-link-sync/transactions", + null + ] + ] + }, + "requests": [ + { + "kind": "content", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "d970973e-d042-4383-9edd-6be699df0c0f", + "parentVersionId": 3, + "content": { + "type": "Diff", + "value": [ + -11, + "v17local edit by client 0", + 15 + ] + } + }, + "status": 200 + } + ] + }, + { + "client": 1, + "database": { + "vaultKey": "[\"http://localhost:62272\",\"test-df2b81ab-03e9-432a-878a-d6aa1c6c9147\"]", + "initialized": true, + "fileManifest": { + "fileManifestId": 6, + "entries": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + } + }, + "local": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + }, + "documents": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "materialized": true, + "bootstrap": false, + "observedHash": "106554e74ef5b4dc8c8668c00ff9e822ebb2573232e6af3c71849bf200685e26", + "base": { + "vaultUpdateId": 3, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.876666Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 26, + "hash": "106554e74ef5b4dc8c8668c00ff9e822ebb2573232e6af3c71849bf200685e26" + } + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "materialized": true, + "bootstrap": false, + "observedHash": "8dd4b135f5ee76639d62ae49fbfc7404962fc5a67f8f60ced508d5cb8ce5f4df", + "base": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17, + "hash": "8dd4b135f5ee76639d62ae49fbfc7404962fc5a67f8f60ced508d5cb8ce5f4df" + } + } + }, + "remoteHeads": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40 + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17 + } + }, + "lastSeenUpdateId": 7 + }, + "disk": { + "visible": [ + [ + "", + null + ], + [ + "P.md", + { + "content": { + "0": 118, + "1": 50, + "2": 49, + "3": 32, + "4": 99, + "5": 111, + "6": 110, + "7": 116, + "8": 101, + "9": 110, + "10": 116, + "11": 32, + "12": 40, + "13": 68, + "14": 50, + "15": 41, + "16": 10 + } + } + ] + ], + "durable": [ + [ + "", + null + ], + [ + "P.md", + { + "content": { + "0": 118, + "1": 50, + "2": 49, + "3": 32, + "4": 99, + "5": 111, + "6": 110, + "7": 116, + "8": 101, + "9": 110, + "10": 116, + "11": 32, + "12": 40, + "13": 68, + "14": 50, + "15": 41, + "16": 10 + } + } + ] + ] + }, + "requests": [ + { + "kind": "create", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "f2abf8a1-8c5d-49d7-8ca8-4d4df56a2aff", + "parentVersionId": null, + "content": { + "type": "Snapshot", + "value": "djggY29udGVudAo=" + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "41f31b38-1edc-4f97-8519-6973e4a04f9f", + "parentFileManifestId": 0, + "entries": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": "P.md" + } + }, + "status": 200 + }, + { + "kind": "content", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "4bd1dcf9-c5d2-42f4-aa96-f1b921362e55", + "parentVersionId": 1, + "content": { + "type": "Diff", + "value": [ + -2, + "v17", + 8, + " from client 1", + 1 + ] + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "3b0cc3a0-b288-47d0-998b-d608f885d4b0", + "parentFileManifestId": 2, + "entries": {} + }, + "status": 200 + }, + { + "kind": "create", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/c6316ddf-539d-4604-8039-5ff6022efef2", + "body": { + "requestId": "edb11740-8c0c-4407-91d2-bd264e26281f", + "parentVersionId": null, + "content": { + "type": "Snapshot", + "value": "djIxIGNvbnRlbnQgKEQyKQo=" + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "c9615094-e693-4b11-8a3d-5c415d3dcdd0", + "parentFileManifestId": 4, + "entries": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + } + }, + "status": 200 + } + ] + } + ], + "duration": 60283 + } + }, + { + "name": "local-update-survives-remote-rename", + "result": { + "success": true, + "duration": 58 + } + }, + { + "name": "catchup-create-and-update-not-skipped", + "result": { + "success": true, + "duration": 55 + } + }, + { + "name": "remote-rename-collides-with-pending-local-create", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "remote-quick-write-rename-before-record", + "result": { + "success": true, + "duration": 42 + } + }, + { + "name": "v4-server-sigkill-retains-acknowledged-state", + "result": { + "success": true, + "duration": 220 + } + }, + { + "name": "idempotency-after-server-pause", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "server-pause-both-clients-create", + "result": { + "success": true, + "duration": 60 + } + }, + { + "name": "server-pause-update-and-create", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "server-pause-rename-edit-resume", + "result": { + "success": true, + "duration": 70 + } + }, + { + "name": "delete-during-pending-create", + "result": { + "success": true, + "duration": 43 + } + }, + { + "name": "rapid-create-update-delete-cycle", + "result": { + "success": true, + "duration": 51 + } + }, + { + "name": "server-pause-both-edit-same-file", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "update-during-create-processing", + "result": { + "success": true, + "duration": 65 + } + }, + { + "name": "interrupted-delete-retry", + "result": { + "success": true, + "duration": 67 + } + }, + { + "name": "rename-pending-create-before-response", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "create-update-coalesce-server-pause", + "result": { + "success": true, + "duration": 67 + } + }, + { + "name": "server-pause-delete-recreate", + "result": { + "success": true, + "duration": 71 + } + }, + { + "name": "merging-update-response-survives-user-rename", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "local-rename-survives-remote-rename", + "result": { + "success": true, + "duration": 82 + } + }, + { + "name": "rename-chain-during-pending-create", + "result": { + "success": true, + "duration": 57 + } + }, + { + "name": "remote-update-survives-user-rename", + "result": { + "success": true, + "duration": 111 + } + }, + { + "name": "same-doc-id-collapse-on-local-create-after-remote-create", + "result": { + "success": true, + "duration": 54 + } + }, + { + "name": "renamed-pending-create-reused-path-then-delete", + "result": { + "success": true, + "duration": 185 + } + }, + { + "name": "rename-pending-create-onto-pending-delete-path", + "result": { + "success": true, + "duration": 166 + } + }, + { + "name": "rename-overwrites-pending-create-then-delete", + "result": { + "success": true, + "duration": 80 + } + }, + { + "name": "same-doc-id-collapse-after-remote-quick-write-and-pending-rename", + "result": { + "success": true, + "duration": 220 + } + }, + { + "name": "delete-recreated-pending-create-with-stale-deleting-record", + "result": { + "success": true, + "duration": 234 + } + }, + { + "name": "queued-create-delete-does-not-hijack-reused-path", + "result": { + "success": true, + "duration": 173 + } + }, + { + "name": "self-merge-pending-rename-aliases-second-create", + "result": { + "success": true, + "duration": 186 + } + } +] \ No newline at end of file diff --git a/logs/harness-review/result-42.json b/logs/harness-review/result-42.json new file mode 100644 index 00000000..fa5492ce --- /dev/null +++ b/logs/harness-review/result-42.json @@ -0,0 +1,4 @@ +{ + "success": true, + "duration": 262 +} \ No newline at end of file diff --git a/logs/harness-review/result-43.json b/logs/harness-review/result-43.json new file mode 100644 index 00000000..cbf0df26 --- /dev/null +++ b/logs/harness-review/result-43.json @@ -0,0 +1,4 @@ +{ + "success": true, + "duration": 273 +} \ No newline at end of file diff --git a/logs/harness-review/results.json b/logs/harness-review/results.json new file mode 100644 index 00000000..b14cc528 --- /dev/null +++ b/logs/harness-review/results.json @@ -0,0 +1,111 @@ +{ + "options": { + "workers": 2, + "seed": 42, + "iterations": 3, + "timeout": 180000 + }, + "results": [ + { + "name": "build-server", + "command": "cargo", + "args": [ + "build", + "--release" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "build-clients", + "command": "npm", + "args": [ + "run", + "build", + "--workspace", + "sync-client", + "--workspace", + "deterministic-tests", + "--workspace", + "test-client" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "harness-self-tests", + "command": "npm", + "args": [ + "run", + "test:harness" + ], + "code": 1, + "signal": null, + "timedOut": false, + "success": false + }, + { + "name": "crash-and-protocol", + "command": "npm", + "args": [ + "run", + "test:protocol" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "deterministic", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/deterministic-tests/dist/cli.js", + "--concurrency", + "2" + ], + "code": 1, + "signal": null, + "timedOut": false, + "success": false + }, + { + "name": "fuzz-42", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/test-client/dist/cli.js", + "--seed", + "42", + "--iterations", + "3", + "--artifacts", + "/Users/schmelczer/Projects/vault-link/logs/harness-review" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "fuzz-43", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/test-client/dist/cli.js", + "--seed", + "43", + "--iterations", + "3", + "--artifacts", + "/Users/schmelczer/Projects/vault-link/logs/harness-review" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + } + ] +} \ No newline at end of file diff --git a/logs/harness-review/trace-42.json b/logs/harness-review/trace-42.json new file mode 100644 index 00000000..c27de220 --- /dev/null +++ b/logs/harness-review/trace-42.json @@ -0,0 +1,368 @@ +{ + "version": 1, + "seed": 42, + "iterations": 3, + "steps": [ + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-0/café/a.md", + "content": "[seed-42-round-0-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-0/café/b.md", + "content": "[seed-42-round-0-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-0/café/a.md", + "key": "round-0/café/a.md" + }, + { + "type": "remember-identity", + "path": "round-0/café/b.md", + "key": "round-0/café/b.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/a.md", + "newPath": "round-0/café/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/c.md", + "newPath": "round-0/café/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/a.md", + "newPath": "round-0/café/c.md" + }, + { + "type": "update", + "client": 0, + "path": "round-0/café/c.md", + "content": "[seed-42-round-0-A]\nrapid final edit" + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-0/café/c.md", + "key": "round-0/café/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-0/café/c.md": "[seed-42-round-0-A]\nrapid final edit", + "round-0/café/b.md": "[seed-42-round-0-B]" + }, + "absent": [ + "round-0/café/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-1/日本語/a.md", + "content": "[seed-42-round-1-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-1/日本語/b.md", + "content": "[seed-42-round-1-B]" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-1/日本語/a.md", + "key": "round-1/日本語/a.md" + }, + { + "type": "remember-identity", + "path": "round-1/日本語/b.md", + "key": "round-1/日本語/b.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/a.md", + "newPath": "round-1/日本語/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/c.md", + "newPath": "round-1/日本語/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/a.md", + "newPath": "round-1/日本語/c.md" + }, + { + "type": "update", + "client": 0, + "path": "round-1/日本語/c.md", + "content": "[seed-42-round-1-A]\nrapid final edit" + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-1/日本語/c.md", + "key": "round-1/日本語/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-1/日本語/c.md": "[seed-42-round-1-A]\nrapid final edit", + "round-1/日本語/b.md": "[seed-42-round-1-B]" + }, + "absent": [ + "round-1/日本語/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]", + "[seed-42-round-1-A]", + "[seed-42-round-1-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-2/café/a.md", + "content": "[seed-42-round-2-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-2/café/b.md", + "content": "[seed-42-round-2-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-2/café/a.md", + "key": "round-2/café/a.md" + }, + { + "type": "remember-identity", + "path": "round-2/café/b.md", + "key": "round-2/café/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/a.md", + "newPath": "round-2/café/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/b.md", + "newPath": "round-2/café/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/c.md", + "newPath": "round-2/café/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-2/café/b.md", + "content": "[seed-42-round-2-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-2/café/b.md", + "key": "round-2/café/a.md" + }, + { + "type": "assert-identity", + "path": "round-2/café/a.md", + "key": "round-2/café/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-2/café/a.md": "[seed-42-round-2-B]", + "round-2/café/b.md": "[seed-42-round-2-A]\nextra line\n" + }, + "absent": [ + "round-2/café/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]", + "[seed-42-round-1-A]", + "[seed-42-round-1-B]", + "[seed-42-round-2-A]", + "[seed-42-round-2-B]" + ], + "removed": [] + } + ] +} \ No newline at end of file diff --git a/logs/harness-review/trace-43.json b/logs/harness-review/trace-43.json new file mode 100644 index 00000000..4d764671 --- /dev/null +++ b/logs/harness-review/trace-43.json @@ -0,0 +1,404 @@ +{ + "version": 1, + "seed": 43, + "iterations": 3, + "steps": [ + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-0/space folder/a.md", + "content": "[seed-43-round-0-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-0/space folder/b.md", + "content": "[seed-43-round-0-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "drop-response", + "client": 0, + "kind": "create", + "point": "after" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-0/space folder/a.md", + "key": "round-0/space folder/a.md" + }, + { + "type": "remember-identity", + "path": "round-0/space folder/b.md", + "key": "round-0/space folder/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/space folder/a.md", + "newPath": "round-0/space folder/c.md" + }, + { + "type": "update", + "client": 1, + "path": "round-0/space folder/a.md", + "content": "[seed-43-round-0-A]\nremote addition\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-0/space folder/c.md", + "key": "round-0/space folder/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-0/space folder/c.md": "[seed-43-round-0-A]\nremote addition\n", + "round-0/space folder/b.md": "[seed-43-round-0-B]" + }, + "absent": [ + "round-0/space folder/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-1/space folder/a.md", + "content": "[seed-43-round-1-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-1/space folder/b.md", + "content": "[seed-43-round-1-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-1/space folder/a.md", + "key": "round-1/space folder/a.md" + }, + { + "type": "remember-identity", + "path": "round-1/space folder/b.md", + "key": "round-1/space folder/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/a.md", + "newPath": "round-1/space folder/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/b.md", + "newPath": "round-1/space folder/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/c.md", + "newPath": "round-1/space folder/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-1/space folder/b.md", + "content": "[seed-43-round-1-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-1/space folder/b.md", + "key": "round-1/space folder/a.md" + }, + { + "type": "assert-identity", + "path": "round-1/space folder/a.md", + "key": "round-1/space folder/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-1/space folder/a.md": "[seed-43-round-1-B]", + "round-1/space folder/b.md": "[seed-43-round-1-A]\nextra line\n" + }, + "absent": [ + "round-1/space folder/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]", + "[seed-43-round-1-A]", + "[seed-43-round-1-B]" + ], + "removed": [] + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-2/nested/a.md", + "content": "[seed-43-round-2-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-2/nested/b.md", + "content": "[seed-43-round-2-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-2/nested/a.md", + "key": "round-2/nested/a.md" + }, + { + "type": "remember-identity", + "path": "round-2/nested/b.md", + "key": "round-2/nested/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/a.md", + "newPath": "round-2/nested/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/b.md", + "newPath": "round-2/nested/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/c.md", + "newPath": "round-2/nested/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-2/nested/b.md", + "content": "[seed-43-round-2-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-2/nested/b.md", + "key": "round-2/nested/a.md" + }, + { + "type": "assert-identity", + "path": "round-2/nested/a.md", + "key": "round-2/nested/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-2/nested/a.md": "[seed-43-round-2-B]", + "round-2/nested/b.md": "[seed-43-round-2-A]\nextra line\n" + }, + "absent": [ + "round-2/nested/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]", + "[seed-43-round-1-A]", + "[seed-43-round-1-B]", + "[seed-43-round-2-A]", + "[seed-43-round-2-B]" + ], + "removed": [] + } + ] +} \ No newline at end of file diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index c87144a9..756d46ef 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -4,11 +4,11 @@ set -e ./scripts/utils/check-node.sh -cd docs +( + cd docs -npm ci -npm run format:check -npm run spell:check -npm run build - -cd - + npm ci + npm run format:check + npm run spell:check + npm run build +) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index bea3d982..67f9580d 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -25,13 +25,16 @@ else fi echo "Bumping sync-server versions" -cd sync-server -cargo set-version --bump $1 +( + cd sync-server + cargo set-version --bump $1 +) echo "Bumping frontend versions" -cd ../frontend -npm version $1 --workspaces -cd .. +( + cd frontend + npm version $1 --workspaces +) cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update diff --git a/scripts/check.sh b/scripts/check.sh index 2ee0dd62..9f5ee98c 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -12,43 +12,42 @@ fi echo "Running checks in sync-server" -cd sync-server -which sqlx || cargo install sqlx-cli -sqlx database create --database-url sqlite://db.sqlite3 -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 +( + cd sync-server + which sqlx || cargo install sqlx-cli + sqlx database create --database-url sqlite://db.sqlite3 + sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 -cargo test --verbose + cargo test --verbose -if [[ "$FIX_MODE" == true ]]; then - cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged - cargo fmt --all -else - cargo clippy --all-targets --all-features - cargo fmt --all -- --check -fi + if [[ "$FIX_MODE" == true ]]; then + cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged + cargo fmt --all + else + cargo clippy --all-targets --all-features + cargo fmt --all -- --check + fi -which cargo-machete || cargo install cargo-machete -cargo machete --with-metadata + which cargo-machete || cargo install cargo-machete + cargo machete --with-metadata +) -cd .. scripts/update-api-types.sh # this will dirty up the git state if not up-to-date echo "Running checks in frontend" -cd frontend +( + cd frontend -if [[ "$FIX_MODE" == true ]]; then - npm install -else - npm ci -fi + if [[ "$FIX_MODE" == true ]]; then + npm install + else + npm ci + fi -cd .. - -cd frontend -npm run build -npm run test -npm run lint -cd .. + npm run build + npm run test + npm run lint +) # Format all files across the project (frontend and backend) # Prettier respects .gitignore by default diff --git a/scripts/e2e-runner.mjs b/scripts/e2e-runner.mjs new file mode 100644 index 00000000..89b2d342 --- /dev/null +++ b/scripts/e2e-runner.mjs @@ -0,0 +1,271 @@ +import { spawn } from "node:child_process"; +import { + createWriteStream, + mkdirSync, + mkdtempSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function parseOptions(env = process.env) { + const workers = positive(env.E2E_WORKERS ?? "1", "E2E_WORKERS"); + + const seed = Number(env.E2E_SEED ?? "1"); + + if ( + !/^\d+$/.test(env.E2E_SEED ?? "1") || + !Number.isInteger(seed) || + seed < 0 || + seed + workers - 1 > 0xffffffff + ) { + throw new Error("E2E_SEED range must fit uint32"); + } + + return { + workers, + seed, + iterations: positive(env.E2E_ITERATIONS ?? "25", "E2E_ITERATIONS"), + timeout: + positive(env.E2E_TIMEOUT_SECONDS ?? "900", "E2E_TIMEOUT_SECONDS") * + 1000, + }; +} + +export async function runCommand( + command, + args, + { cwd, logPath, timeout, active = new Set(), env = process.env }, +) { + const log = createWriteStream(logPath, { flags: "w" }); + const child = spawn(command, args, { + cwd, + env: { ...env, NO_COLOR: "1", FORCE_COLOR: "0" }, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + }); + + let logError; + log.on("error", (error) => { + logError = error; + killOwned(child, "SIGTERM"); + }); + + active.add(child); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + let timedOut = false; + let hardKill; + + const timer = setTimeout(() => { + timedOut = true; + killOwned(child, "SIGTERM"); + hardKill = setTimeout(() => killOwned(child, "SIGKILL"), 5_000); + }, timeout); + + let spawnError; + child.once("error", (error) => { + spawnError = error; + }); + + try { + const result = await new Promise((resolveResult) => + child.once("close", (code, signal) => + resolveResult({ code, signal }), + ), + ); + + if (!logError) { + await new Promise((resolveLog) => { + log.once("error", resolveLog); + log.end(resolveLog); + }); + } + + return { + command, + args, + ...result, + timedOut, + error: spawnError?.message ?? logError?.message, + success: result.code === 0 && !timedOut && !spawnError && !logError, + }; + } finally { + clearTimeout(timer); + clearTimeout(hardKill); + active.delete(child); + } +} + +function killOwned(child, signal) { + if (!child.pid) { + return; + } + + try { + if (process.platform === "win32") { + child.kill(signal); + } else { + process.kill(-child.pid, signal); // the group created by this spawn + } + } catch (error) { + if (error.code !== "ESRCH") { + throw error; + } + } +} + +function positive(value, label) { + const result = Number(value); + if ( + !/^\d+$/.test(String(value)) || + !Number.isSafeInteger(result) || + result < 1 + ) + throw new Error(`${label} must be a positive integer`); + return result; +}; + +async function main() { + const options = parseOptions(); + const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const artifacts = process.env.E2E_ARTIFACTS + ? resolve(process.env.E2E_ARTIFACTS) + : mkdtempSync(join(tmpdir(), "vault-link-e2e-")); + + mkdirSync(artifacts, { recursive: true }); + console.log(`Artifacts: ${artifacts}`); + console.log( + `Seeds: ${options.seed}..${options.seed + options.workers - 1}; iterations: ${options.iterations}`, + ); + + const active = new Set(); + let interrupted = false; + const stop = () => { + interrupted = true; + for (const child of active) killOwned(child, "SIGTERM"); + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + process.on("exit", () => { + for (const child of active) killOwned(child, "SIGKILL"); + }); + + const results = []; + const run = async (name, command, args, cwd = root) => { + if (interrupted) { + throw new Error("Interrupted"); + } + + const logPath = join(artifacts, `${name}.log`); + const result = await runCommand(command, args, { + cwd, + logPath, + timeout: options.timeout, + active, + env: { ...process.env, E2E_ARTIFACTS: artifacts }, + }); + results.push({ name, ...result }); + writeFileSync( + join(artifacts, "results.json"), + JSON.stringify({ options, results }, null, 2), + ); + console.log(`${result.success ? "PASS" : "FAIL"} ${name}: ${logPath}`); + return result.success; + }; + const frontend = join(root, "frontend"); + + // Build production adapters as well as the protocol peers and test clients. + if ( + !(await run( + "build-server", + "cargo", + ["build", "--release"], + join(root, "sync-server"), + )) + ) { + throw new Error("Server build failed"); + } + + if ( + !(await run( + "build-clients", + "npm", + [ + "run", + "build", + "--workspace", + "sync-client", + "--workspace", + "deterministic-tests", + "--workspace", + "test-client", + "--workspace", + "local-client-cli", + "--workspace", + "vault-link-obsidian-plugin", + ], + frontend, + )) + ) { + throw new Error( + "Client build failed (run npm ci in frontend if dependencies are missing)", + ); + } + + await run( + "server-unit-tests", + "cargo", + ["test"], + join(root, "sync-server"), + ); + await run( + "client-unit-tests", + "npm", + ["run", "test", "--workspace", "sync-client"], + frontend, + ); + await run("adapter-unit-tests", "npm", ["run", "test", "--workspace", "local-client-cli", "--workspace", "vault-link-obsidian-plugin"], frontend); + await run("harness-self-tests", "npm", ["run", "test:harness"], frontend); + await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend); + + await run("deterministic", process.execPath, [ + join(frontend, "deterministic-tests/dist/cli.js"), + "--concurrency", + String(options.workers), + ]); + + // Execute all suites even if one reports a regression; failure is never + // inferred from log text and successful workers cannot mask failed ones. + await Promise.all( + Array.from({ length: options.workers }, (_, i) => + run(`fuzz-${options.seed + i}`, process.execPath, [ + join(frontend, "test-client/dist/cli.js"), + "--seed", + String(options.seed + i), + "--iterations", + String(options.iterations), + "--artifacts", + artifacts, + ]), + ), + ); + if (interrupted || results.some((result) => !result.success)) { + throw new Error( + `Tests failed; inspect ${join(artifacts, "results.json")}`, + ); + } + + console.log("All harness suites passed"); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/e2e-runner.test.mjs b/scripts/e2e-runner.test.mjs new file mode 100644 index 00000000..84ac0f04 --- /dev/null +++ b/scripts/e2e-runner.test.mjs @@ -0,0 +1,151 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseOptions, runCommand } from "./e2e-runner.mjs"; + +test("the clean-install lockfile includes every workspace and its declared dependencies", () => { + const frontend = new URL("../frontend/", import.meta.url); + const readJson = (path) => + JSON.parse(readFileSync(new URL(path, frontend), "utf8")); + const root = readJson("package.json"); + const lock = readJson("package-lock.json"); + assert.deepEqual(lock.packages[""].workspaces, root.workspaces); + for (const workspace of ["", ...root.workspaces]) { + const manifest = workspace + ? readJson(`${workspace}/package.json`) + : root; + const locked = lock.packages[workspace]; + assert(locked, `Missing workspace from lockfile: ${workspace}`); + for (const field of [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ]) + assert.deepEqual( + locked[field] ?? {}, + manifest[field] ?? {}, + `Stale ${workspace || "root"} ${field}; regenerate the lockfile before CI`, + ); + if (workspace) { + assert.equal(locked.version, manifest.version); + assert.deepEqual(lock.packages[`node_modules/${manifest.name}`], { + resolved: workspace, + link: true, + }); + } + } +}); + +test("CI installs dependencies before E2E and uploads the runner's artifacts on failure", () => { + const workflow = readFileSync( + new URL("../.forgejo/workflows/e2e.yml", import.meta.url), + "utf8", + ); + // Guard the workflow/runner contract without requiring a CI service or + // installing dependencies just to run the runner's own tests. + const steps = workflow.split(/^ - /m); + const install = steps.findIndex((step) => /run: npm ci\s*$/m.test(step)); + const e2e = steps.findIndex((step) => + /run: scripts\/e2e\.sh\s*$/m.test(step), + ); + assert(install > 0 && install < e2e, "npm ci must precede the E2E build"); + assert.match(steps[install], /working-directory: frontend\s*$/m); + assert.match(workflow, /^ E2E_ARTIFACTS: logs\/e2e\s*$/m); + assert.match(workflow, /^ E2E_WORKERS: 8\s*$/m); + const upload = steps.find((step) => + step.includes("uses: actions/upload-artifact@"), + ); + assert(upload, "CI must collect diagnostics"); + assert.match(upload, /if: always\(\)/); + assert.match(upload, /path: \$\{\{ env\.E2E_ARTIFACTS \}\}/); + assert.doesNotMatch( + workflow, + /cargo run|SERVER_PID|scripts\/clean-up\.sh/, + "The runner owns server lifecycle and isolated temporary databases", + ); +}); + +test("e2e observes exit codes, spawn errors, logs and bounded timeout", async () => { + const directory = mkdtempSync(join(tmpdir(), "vault-link-runner-test-")); + try { + const options = { + cwd: directory, + logPath: join(directory, "run.log"), + timeout: 5_000, + }; + const failed = await runCommand( + process.execPath, + ["-e", "console.log('last line'); process.exitCode = 7"], + options, + ); + assert.equal(failed.code, 7); + assert.equal(failed.success, false); + assert.equal(readFileSync(options.logPath, "utf8").trim(), "last line"); + const missing = await runCommand( + join(directory, "missing-command"), + [], + options, + ); + assert.equal(missing.success, false); + assert(missing.error); + const badLog = await runCommand( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { ...options, logPath: directory }, + ); + assert(!badLog.success); + assert(badLog.error); + const timedOut = await runCommand( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { ...options, timeout: 40 }, + ); + assert(timedOut.timedOut); + assert(!timedOut.success); + assert( + ( + await runCommand( + process.execPath, + ["-e", "process.exit(0)"], + options, + ) + ).success, + ); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +test("E2E includes both peers' unit suites and the permanent audit regressions", () => { + const runner = readFileSync( + new URL("./e2e-runner.mjs", import.meta.url), + "utf8", + ); + assert.match( + runner.replace(/\s+/g, ""), + /run\("server-unit-tests","cargo",\["test"\]/, + ); + assert.match( + runner.replace(/\s+/g, ""), + /run\("client-unit-tests","npm",\["run","test","--workspace","sync-client"\]/, + ); + const scripts = JSON.parse( + readFileSync( + new URL("../frontend/package.json", import.meta.url), + "utf8", + ), + ).scripts; + for (const file of ["engine-regressions", "sync-audit"]) + assert( + scripts["test:harness"].includes(`'test-support/${file}.test.ts'`), + `${file} must be an individually quoted test argument`, + ); + assert( + scripts["test:protocol"].includes( + "'test-support/protocol-audit.test.ts'", + ), + ); +}); diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 7ab8d90c..d0bac3ae 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -1,139 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail -set -e -set -o pipefail - -NO_COLOR=1 -FORCE_COLOR=0 - -./scripts/utils/check-node.sh - -# Check if the argument is provided -if [ $# -eq 0 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Get the number of processes from the first argument -process_count=$1 - -mkdir -p logs - -# Build and restart the server -echo "Building server..." -cd sync-server -cargo build --release - -# Kill any existing server process -echo "Stopping existing server..." -pkill -f "sync_server" 2>/dev/null || true -sleep 1 - -# Clean databases (uses tmpfs via /dev/shm for zero disk I/O) -echo "Cleaning databases..." -rm -rf /host/tmp/vaultlink-e2e-databases - -# Start the server in the background -echo "Starting server..." -./target/release/sync_server config-e2e.yml & -server_pid=$! -echo "Server started with PID: $server_pid" - -# Ensure server is killed on script exit -cleanup_server() { - if [ -n "$server_pid" ]; then - echo "Stopping server (PID: $server_pid)..." - kill $server_pid 2>/dev/null || true - wait $server_pid 2>/dev/null || true - server_pid="" - fi -} -trap cleanup_server EXIT - -cd .. - -cd frontend -npm ci -npm run build - -../scripts/utils/wait-for-server.sh - -pids=() -for i in $(seq 1 $process_count); do - node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 & - pid=$! - pids+=($pid) - echo "Started process $i with PID: $pid (log: logs/log_${i}.log)" -done - -cd .. - -print_failed_log() { - for i in $(seq 1 $process_count); do - if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then - # Get the exit code of the process - wait ${pids[$i-1]} - exit_code=$? - - # Only consider non-zero exit codes as failures - if [ $exit_code -ne 0 ]; then - echo "----- Log for process ${pids[$i-1]} (log_${i}.log) -----" - cat "$(pwd)/logs/log_${i}.log" - echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log" - return 0 - else - echo "Process ${pids[$i-1]} completed successfully with exit code 0" - # Mark this PID as processed by setting it to empty - pids[$i-1]="" - fi - fi - done - return 1 -} - -E2E_TIMEOUT=${2:-3600} -start_time=$(date +%s) -echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)" - -# Monitor processes -while true; do - # Script-level timeout to prevent indefinite hangs - current_time=$(date +%s) - elapsed=$((current_time - start_time)) - if [ $elapsed -ge $E2E_TIMEOUT ]; then - echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes." - for pid in "${pids[@]}"; do - if [ -n "$pid" ]; then - kill $pid 2>/dev/null || true - fi - done - exit 1 - fi - - if print_failed_log; then - # Kill remaining processes - for pid in "${pids[@]}"; do - if [ -n "$pid" ]; then - kill $pid 2>/dev/null || true - fi - done - exit 1 - fi - - # Check if all processes have completed - all_done=true - for pid in "${pids[@]}"; do - if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then - all_done=false - break - fi - done - - if $all_done; then - cleanup_server - echo "All processes completed successfully" - exit 0 - fi - - sleep 0.2 -done +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec node "$script_dir/e2e-runner.mjs" diff --git a/scripts/update-api-types.sh b/scripts/update-api-types.sh index 3f4a9e2a..055dd643 100755 --- a/scripts/update-api-types.sh +++ b/scripts/update-api-types.sh @@ -2,21 +2,19 @@ set -e -rm -rf sync-server/bindings - -cd sync-server -cargo test export_bindings -cd - - -# Both target directories contain only generated bindings — wipe and copy +# This directory contains only generated bindings. ts-rs writes here directly, +# as configured in .cargo/config.toml. rm -f frontend/sync-client/src/services/types/*.ts -rm -f frontend/history-ui/src/lib/types/*.ts -cp -r sync-server/bindings/* frontend/sync-client/src/services/types/ -cp -r sync-server/bindings/* frontend/history-ui/src/lib/types/ -cd frontend -npm run lint -cd .. +( + cd sync-server + cargo test export_bindings +) + +( + cd frontend + npm run lint +) # Format all files across the project (frontend and backend) npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}" diff --git a/sync-server/Cargo.lock b/sync-server/Cargo.lock index b3da1486..f143afcc 100644 --- a/sync-server/Cargo.lock +++ b/sync-server/Cargo.lock @@ -1679,15 +1679,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "sanitize-filename" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" -dependencies = [ - "regex", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2139,17 +2130,20 @@ dependencies = [ "rand 0.9.0", "reconcile-text", "regex", - "sanitize-filename", "serde", "serde_json", "serde_yaml", + "sha2", "sqlx", + "tempfile", "thiserror 2.0.17", "tokio", + "tower", "tower-http", "tracing", "tracing-subscriber", "ts-rs", + "unicode-normalization", "uuid", ] diff --git a/sync-server/Cargo.toml b/sync-server/Cargo.toml index fac06efa..bc39c34b 100644 --- a/sync-server/Cargo.toml +++ b/sync-server/Cargo.toml @@ -24,17 +24,22 @@ humantime-serde = "1.1.1" sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] } chrono = { version = "0.4.41", features = ["serde"] } rand = "0.9.0" -sanitize-filename = "0.6.0" regex = "1.12.2" clap = { version = "4.5.38", features = ["derive"] } futures = "0.3.31" serde_yaml = "0.9.34" serde_json = "1.0.140" bimap = "0.6.3" -ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] } +sha2 = "0.10" +unicode-normalization = "0.1" +ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl", "no-serde-warnings"] } base64 = "0.22.1" reconcile-text = { version = "0.8.0", features = ["serde"] } +[dev-dependencies] +tempfile = "3" +tower = { version = "0.5", features = ["util"] } + [profile.release] codegen-units = 1 lto = true diff --git a/sync-server/README.md b/sync-server/README.md index 16509fb7..a5c40318 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -1,17 +1,143 @@ # Sync server -## Creating/resetting the Database for development +## API v5 -```sh -rm -rf db.sqlite* -sqlx database create --database-url sqlite://db.sqlite3 -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 -cargo sqlx prepare --workspace +The server is a compare-and-swap store. Content versions do not contain paths or +deletions. A separately versioned file manifest owns the namespace and membership. +All routes below are under `/vaults/:vault_id` and require bearer authentication +(except the WebSocket, which authenticates its first message). + +| Endpoint | Purpose | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `PUT /documents/:document_id` | CAS an immutable content version. | +| `GET /documents/:document_id/metadata` | Current version metadata and size without content bytes. | +| `GET /documents/:document_id` | Current content version, including base64 bytes. | +| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. | +| `GET /file-manifest` | Current file manifest. | +| `PUT /file-manifest` | CAS the complete file manifest. | +| `GET /vault-snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. | +| `GET /events-since?after=N` | Every event after N, in order, with the current watermark. | +| `GET /ws` | Ordered event delivery and ephemeral cursor positions. | + +A content push includes a `Device-Id` header and this JSON body: + +```json +{ + "requestId": "2e3ac3fe-a8d2-49ef-bdba-30ab5b7336f1", + "parentVersionId": null, + "content": { "type": "Snapshot", "value": "aGVsbG8=" } +} ``` -## Updating the DB schema through a migration +`parentVersionId: null` creates previously absent content using the UUID in the +URL. Subsequent pushes name its current version. `Diff` with a `value` array of +numbers/strings is also supported; the server only reconstructs the supplied +snapshot. File modification times are not part of the protocol. Paths and +`Delete` payloads are rejected. -```sh -sqlx migrate add --source src/app_state/database/migrations -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 +A file manifest push is: + +```json +{ + "requestId": "69e6a1ba-1f20-4d81-9ae2-e7812c321e65", + "parentFileManifestId": 0, + "entries": { "7534676e-b51a-42d6-8a62-e70a477f731d": "notes/hello.md" } +} ``` + +The initial empty file manifest has version 0. Upload a new document's content +first, then publish its ID in the file manifest. Removal from the file manifest +deletes membership; content updates never restore membership. Unreferenced +content remains readable. + +File manifest versions are stored in `file_manifests`, with one row per +UUID-to-path mapping in `file_manifest_entries`. Each version has a header row +even when its entries are empty. Entries are unique by document ID and exact path +within a version; portable path validation also rejects aliases and directory +conflicts. Reads reconstruct the complete map, and vault snapshots join these +rows to the latest content versions. Event replay and retry acknowledgements are +also reconstructed from normalized content/manifest rows, without storing a +second complete map in the event table. + +Both pushes return a tagged `Accepted` or `StaleBase` response. An accepted file +manifest response returns its assigned file manifest ID. A stale content response +includes metadata and byte length only; clients fetch immutable bytes after applying +their size and exclusion policy. A stale file manifest response includes the complete map. +Accepted requests store their unique request ID and fingerprint in the event row, +atomically with the version. Retrying the exact request reconstructs the original +`Accepted` response from that event before checking the current head, even after +later changes or a restart. Changing a payload while reusing its request ID is +rejected, including reuse across content and file manifest pushes. Session/device +identity is not part of the fingerprint. + +File manifest validation rejects duplicate UUID keys (including aliases), missing +content, path aliases, inconsistent directory spelling, and file/ancestor +conflicts. Paths use `/`, NFC normalization, and portable Windows filename +restrictions without imposing a universal total-path byte limit. Alias comparison is NFC → Unicode uppercase → +NFC. `.vault-link-sync` and its aliases are reserved. The server never sanitizes +or allocates a replacement name. + +## Ordering and recovery + +Every accepted mutation appends one event in the same SQLite write transaction +as its state. Events also provide durable request tracking; there is no separate +receipt table. Event IDs are contiguous safe JavaScript integers; +content/file-manifest version IDs are the event IDs that created them. Rejected +requests and retries of accepted requests append no events. Timestamps never +determine event ordering. + +WebSocket messages are `{type: "vaultChanged"}` hints. Clients fetch ordered +history through HTTP from their last incorporated event ID. The server compares +its current history checkpoint after broadcasts and periodically, so missed wakes +and requests interrupted after commit still trigger catchup. Reconnecting sends a +fresh hint without replaying events over the socket. Cursor messages are unchanged. +A fresh client reads `/vault-snapshot`, applies it durably, then replays after its +watermark. Successful HTTP responses include `X-Vault-Link-History`, containing +an event ID and a random event-incarnation token. Clients persist and echo this +checkpoint on later HTTP requests. The server returns HTTP 409 with +`X-Vault-Link-History-Mismatch: 1` before executing a request whose checkpoint is +absent from the current database. Restored backups retain their original prefix; +new events get new tokens even when numeric IDs or request UUIDs are reused. +Restore databases while the server is stopped. Live replacement of open SQLite +files is not supported. +Cursor notifications are ephemeral and do not consume event IDs. +Cursor expiry and disconnect broadcast the remaining clients, including an empty +list when the last client leaves. + +## Vault names and database files + +Vault names and configured allowlists are trimmed and lowercased consistently. +Empty names, `.` and `..`, path separators, and control characters are rejected. +Other Unicode spellings remain distinct identities, including composed and +decomposed characters. + +Databases use `vaults/.sqlite` underneath the +configured database directory. This prevents filesystem case and Unicode aliases +from sharing a database. This is the only supported storage layout; top-level +`.sqlite` files are not loaded or migrated. Back up the entire database +directory, including the `vaults` subdirectory. + +Vault snapshots query metadata and byte lengths without loading document contents. + +SQLite uses WAL, `synchronous=FULL`, and `fullfsync` where supported. On Unix, startup also +flushes database directory entries; the filesystem must support those durability +operations. Content, file manifests, and events (including request fingerprints) +are retained indefinitely. + +Deploy API v5 clients and server together. The v4 database schema migrates by +adding event-incarnation tokens without deleting content or receipts. Schemas +from before v4 still require a separate database directory and explicit recovery. +Vault databases open and migrate independently on first access; one corrupt or +incompatible vault does not prevent healthy vaults from serving requests. + +Diffs are limited to 10,000 items and reconstructed in one pass over Unicode +scalars on a blocking worker, before taking the SQLite writer transaction. The +transaction rechecks request identity and the parent version before committing. +Clients send a snapshot when an edit exceeds the diff-operation budget. Decoded +content is bounded by `max_body_size_mb` as well as the encoded request limit. + +A missing configuration file is initialized with durably stored credentials. +Existing configurations must explicitly supply `users.user_configs`; omitted +credentials are rejected instead of generating an unpersisted token at startup. +Base64 increases snapshot body size by about one third; request limits apply to +the encoded JSON body. diff --git a/sync-server/build.rs b/sync-server/build.rs index d5068697..ee539ea3 100644 --- a/sync-server/build.rs +++ b/sync-server/build.rs @@ -1,5 +1,5 @@ // generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added - println!("cargo:rerun-if-changed=migrations"); + println!("cargo:rerun-if-changed=src/app_state/database/migrations"); } diff --git a/sync-server/src/app_state.rs b/sync-server/src/app_state.rs index 2019e08e..910564f9 100644 --- a/sync-server/src/app_state.rs +++ b/sync-server/src/app_state.rs @@ -20,7 +20,7 @@ pub struct AppState { impl AppState { pub async fn try_new(config: Config) -> Result { let broadcasts = Broadcasts::new(&config.server); - let database = Database::try_new(&config.database, &broadcasts).await?; + let database = Database::try_new(&config.database).await?; let cursors: Cursors = Cursors::new(&config.database, &broadcasts); Cursors::start_background_task(cursors.clone()); diff --git a/sync-server/src/app_state/cursors.rs b/sync-server/src/app_state/cursors.rs index d083e1ac..e535d5b2 100644 --- a/sync-server/src/app_state/cursors.rs +++ b/sync-server/src/app_state/cursors.rs @@ -2,15 +2,13 @@ use core::time::Duration; use std::{collections::HashMap, sync::Arc}; use tokio::sync::Mutex; +use uuid::Uuid; use super::{ database::models::{DeviceId, VaultId}, websocket::{ - broadcasts::Broadcasts, - models::{ - ClientCursors, CursorPositionFromServer, WebSocketServerMessage, - WebSocketServerMessageWithOrigin, - }, + broadcasts::{Broadcasts, Notification}, + models::{ClientCursors, CursorPositionFromServer}, }, }; use crate::{ @@ -22,6 +20,7 @@ pub struct Cursors { config: DatabaseConfig, broadcasts: Broadcasts, vault_to_cursors: Arc>>>, + sessions: Arc>>>, } impl Cursors { @@ -30,19 +29,44 @@ impl Cursors { config: config.clone(), broadcasts: broadcasts.clone(), vault_to_cursors: Arc::new(Mutex::new(HashMap::new())), + sessions: Arc::default(), } } + pub async fn register_connection(&self, vault_id: &str, device_id: &str) -> Uuid { + let mut sessions = self.sessions.lock().await; + let session = Uuid::new_v4(); + sessions + .entry(vault_id.to_owned()) + .or_default() + .insert(device_id.to_owned(), session); + self.remove_device_cursors(vault_id, device_id).await; + session + } + pub async fn update_cursors( &self, vault_id: VaultId, user_name: String, device_id: &DeviceId, + session: Uuid, document_to_cursors: Vec, ) { + // A reconnect can overlap an old connection's buffered messages and + // cleanup. Only the most recently authenticated session owns presence. + let sessions = self.sessions.lock().await; + if sessions + .get(&vault_id) + .and_then(|devices| devices.get(device_id)) + != Some(&session) + { + return; + } let mut vault_to_cursors = self.vault_to_cursors.lock().await; - let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new); + let all_device_cursors = vault_to_cursors + .entry(vault_id.clone()) + .or_insert_with(Vec::new); all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id); all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors { @@ -51,8 +75,7 @@ impl Cursors { documents_with_cursors: document_to_cursors, })); - drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock - self.broadcast_cursors().await; + self.broadcast_cursors(&vault_id, all_device_cursors).await; } pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec { @@ -81,33 +104,57 @@ impl Cursors { async fn remove_expired_cursors(&self) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; - for (_vault_id, cursors) in vault_to_cursors.iter_mut() { + for (vault_id, cursors) in vault_to_cursors.iter_mut() { + let previous_len = cursors.len(); cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout)); + if cursors.len() != previous_len { + self.broadcast_cursors(vault_id, cursors).await; + } } + vault_to_cursors.retain(|_, cursors| !cursors.is_empty()); } - async fn broadcast_cursors(&self) { - let vault_to_cursors = self.vault_to_cursors.lock().await; - - for (vault_id, cursors) in vault_to_cursors.iter() { - self.broadcasts - .send_document_update( - vault_id.clone(), - WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions( - CursorPositionFromServer { - clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), - }, - )), - ) - .await; - } + async fn broadcast_cursors(&self, vault_id: &str, cursors: &[ClientCursorsWithTimeToLive]) { + self.broadcasts + .send( + vault_id.to_owned(), + Notification::Cursors(CursorPositionFromServer { + clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), + }), + ) + .await; } - pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { + pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str, session: Uuid) { + let mut sessions = self.sessions.lock().await; + let Some(devices) = sessions.get_mut(vault_id) else { + return; + }; + if devices.get(device_id) != Some(&session) { + return; + } + devices.remove(device_id); + if devices.is_empty() { + sessions.remove(vault_id); + } + self.remove_device_cursors(vault_id, device_id).await; + } + + async fn remove_device_cursors(&self, vault_id: &str, device_id: &str) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { + let previous_len = cursors.len(); + cursors.retain(|c| c.client_cursors.device_id != device_id); + + if cursors.len() != previous_len { + self.broadcast_cursors(vault_id, cursors).await; + } + + if cursors.is_empty() { + vault_to_cursors.remove(vault_id); + } } } } @@ -130,3 +177,143 @@ impl ClientCursorsWithTimeToLive { self.last_updated.elapsed() > ttl } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::server_config::ServerConfig; + + #[tokio::test] + async fn disconnect_broadcasts_remaining_clients_and_the_final_empty_state() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + let mut sessions = HashMap::new(); + for device in ["one", "two"] { + let session = cursors.register_connection("vault", device).await; + sessions.insert(device, session); + cursors + .update_cursors( + "vault".to_owned(), + "user".to_owned(), + &device.to_owned(), + session, + vec![], + ) + .await; + } + let mut receiver = broadcasts.get_receiver("vault".to_owned()).await; + let mut other = broadcasts.get_receiver("other".to_owned()).await; + cursors + .remove_cursors_of_device("vault", "one", sessions["one"]) + .await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert_eq!(update.clients.len(), 1); + assert_eq!(update.clients[0].device_id, "two"); + + cursors + .remove_cursors_of_device("vault", "two", sessions["two"]) + .await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(other.try_recv().is_err()); + cursors + .remove_cursors_of_device("vault", "two", sessions["two"]) + .await; + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn expiration_broadcasts_only_changed_vaults_including_empty_ones() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let config = DatabaseConfig::default(); + let cursors = Cursors::new(&config, &broadcasts); + for (vault, device) in [("vault", "expired"), ("vault", "live"), ("other", "live")] { + let session = cursors.register_connection(vault, device).await; + cursors + .update_cursors( + vault.to_owned(), + "user".to_owned(), + &device.to_owned(), + session, + vec![], + ) + .await; + } + cursors + .vault_to_cursors + .lock() + .await + .get_mut("vault") + .unwrap()[0] + .last_updated = std::time::Instant::now() + .checked_sub(config.cursor_timeout + Duration::from_secs(1)) + .unwrap(); + let mut receiver = broadcasts.get_receiver("vault".to_owned()).await; + let mut other = broadcasts.get_receiver("other".to_owned()).await; + cursors.remove_expired_cursors().await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert_eq!(update.clients.len(), 1); + assert_eq!(update.clients[0].device_id, "live"); + assert!(other.try_recv().is_err()); + + cursors + .vault_to_cursors + .lock() + .await + .get_mut("vault") + .unwrap()[0] + .last_updated = std::time::Instant::now() + .checked_sub(config.cursor_timeout + Duration::from_secs(1)) + .unwrap(); + cursors.remove_expired_cursors().await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(cursors.get_cursors(&"vault".to_owned()).await.is_empty()); + cursors.remove_expired_cursors().await; + assert!(receiver.try_recv().is_err()); + assert!(other.try_recv().is_err()); + } + #[tokio::test] + async fn replacement_session_fences_old_updates_and_disconnects() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + let vault = "vault".to_owned(); + let device = "device".to_owned(); + let old = cursors.register_connection(&vault, &device).await; + cursors + .update_cursors(vault.clone(), "old".to_owned(), &device, old, vec![]) + .await; + let new = cursors.register_connection(&vault, &device).await; + cursors + .update_cursors(vault.clone(), "new".to_owned(), &device, new, vec![]) + .await; + let mut receiver = broadcasts.get_receiver(vault.clone()).await; + cursors + .update_cursors( + vault.clone(), + "late old update".to_owned(), + &device, + old, + vec![], + ) + .await; + cursors.remove_cursors_of_device(&vault, &device, old).await; + assert!(receiver.try_recv().is_err()); + assert_eq!(cursors.get_cursors(&vault).await[0].user_name, "new"); + cursors.remove_cursors_of_device(&vault, &device, new).await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(cursors.get_cursors(&vault).await.is_empty()); + assert!(cursors.sessions.lock().await.is_empty()); + } +} diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 75ce6df4..eca69a44 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -1,24 +1,33 @@ use core::time::Duration; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::{Arc, Weak}, +}; use anyhow::{Context as _, Result}; use log::info; -use models::{ - DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId, -}; -use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions, types::chrono::Utc}; - -pub mod models; +use models::VaultId; +use sha2::{Digest, Sha256}; +use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions}; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; use tokio::sync::Mutex; use tokio::time::Instant; -use uuid::fmt::Hyphenated; -use super::websocket::{ - broadcasts::Broadcasts, - models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate}, +pub mod models; + +mod mutations; +mod queries; + +#[cfg(test)] +mod tests; + +use crate::{ + config::database_config::DatabaseConfig, + utils::normalize_vault_id::{normalize_string, validate_vault_id}, }; -use crate::config::database_config::DatabaseConfig; + +const VAULT_DATABASE_DIRECTORY: &str = "vaults"; #[derive(Clone)] struct PoolWithTimestamp { @@ -38,15 +47,18 @@ impl std::fmt::Debug for PoolWithTimestamp { #[derive(Clone, Debug)] pub struct Database { config: DatabaseConfig, - broadcasts: Broadcasts, connection_pools: Arc>>, + opening_pools: Arc>>>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; impl Database { - pub async fn try_new(config: &DatabaseConfig, broadcasts: &Broadcasts) -> Result { - tokio::fs::create_dir_all(&config.databases_directory_path) + pub async fn try_new(config: &DatabaseConfig) -> Result { + let vault_directory = config + .databases_directory_path + .join(VAULT_DATABASE_DIRECTORY); + tokio::fs::create_dir_all(&vault_directory) .await .with_context(|| { format!( @@ -55,35 +67,16 @@ impl Database { ) })?; - let mut connection_pools = std::collections::HashMap::new(); + // SQLite flushes its own files, not ancestors created by this process. + Self::flush_directory_ancestors(&vault_directory).await?; - info!("Applying pending database migrations"); - let mut entries = tokio::fs::read_dir(&config.databases_directory_path).await?; - while let Some(entry) = entries.next_entry().await? { - if !entry.file_name().to_string_lossy().ends_with(".sqlite") { - continue; - } - - let vault: VaultId = entry - .file_name() - .to_string_lossy() - .trim_end_matches(".sqlite") - .to_owned(); - - let pool = Self::create_vault_database(config, &vault).await?; - connection_pools.insert( - vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, - ); - } + // Open and migrate each vault lazily under its own lock. A corrupt or + // incompatible database must fail that vault, not global startup. let database = Self { config: config.clone(), - connection_pools: Arc::new(Mutex::new(connection_pools)), - broadcasts: broadcasts.clone(), + connection_pools: Arc::default(), + opening_pools: Arc::default(), }; // Start background task to cleanup idle connection pools @@ -92,20 +85,28 @@ impl Database { Ok(database) } - async fn create_vault_database( - config: &DatabaseConfig, - vault: &VaultId, - ) -> Result> { - let file_name = config + fn database_path(config: &DatabaseConfig, vault: &str) -> PathBuf { + // Hash the normalized vault name into a fixed-length ASCII filename so + // filesystem case/Unicode aliasing (e.g. composed vs. decomposed é on + // macOS) cannot make separately authorized vaults share a database. + // This is only a storage filename; the user-facing vault name is unchanged. + let digest = Sha256::digest(vault.as_bytes()); + config .databases_directory_path - .join(format!("{vault}.sqlite")); + .join(VAULT_DATABASE_DIRECTORY) + .join(format!("{digest:x}.sqlite")) + } + async fn open_database(config: &DatabaseConfig, file_name: &Path) -> Result> { let connection_options = SqliteConnectOptions::new() - .filename(file_name.clone()) + .filename(file_name) .create_if_missing(true) .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Full) .busy_timeout(Duration::from_secs(3600)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .synchronous(sqlx::sqlite::SqliteSynchronous::Full) + .pragma("fullfsync", "ON") + .foreign_keys(true) .log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30)); let pool = SqlitePoolOptions::new() @@ -118,9 +119,32 @@ impl Database { Self::run_migrations(&pool).await?; + if let Some(directory) = file_name.parent() { + Self::flush_directory_ancestors(directory).await?; + } + Ok(pool) } + async fn flush_directory_ancestors(directory: &Path) -> Result<()> { + #[cfg(unix)] + { + let directory = tokio::fs::canonicalize(directory).await?; + tokio::task::spawn_blocking(move || -> Result<()> { + for ancestor in directory.ancestors() { + std::fs::File::open(ancestor)?.sync_all().with_context(|| { + format!("Cannot flush directory {}", ancestor.display()) + })?; + } + Ok(()) + }) + .await??; + } + #[cfg(not(unix))] + let _ = directory; + Ok(()) + } + async fn run_migrations(pool: &Pool) -> Result<()> { sqlx::migrate!("src/app_state/database/migrations") .run(pool) @@ -129,27 +153,51 @@ impl Database { } async fn get_connection_pool(&self, vault: &VaultId) -> Result> { - let mut pools = self.connection_pools.lock().await; - - if !pools.contains_key(vault) { - let pool = Self::create_vault_database(&self.config, vault).await?; - pools.insert( - vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, - ); + let vault = normalize_string(vault); + validate_vault_id(&vault)?; + if let Some(pool) = self.existing_pool(&vault).await { + return Ok(pool); } - let pool_with_timestamp = pools + // Serialize initialization per vault. SQLite lock waits, migrations and + // directory flushes must never hold the registry for unrelated vaults. + // Weak entries also allow cancelled/failed opens to be retried safely. + let opening = { + let mut openings = self.opening_pools.lock().await; + openings.retain(|_, opening| opening.strong_count() > 0); + if let Some(opening) = openings.get(&vault).and_then(Weak::upgrade) { + opening + } else { + let opening = Arc::new(Mutex::new(())); + openings.insert(vault.clone(), Arc::downgrade(&opening)); + opening + } + }; + let _opening = opening.lock().await; + if let Some(pool) = self.existing_pool(&vault).await { + return Ok(pool); + } + let file_name = Self::database_path(&self.config, &vault); + let pool = Self::open_database(&self.config, &file_name).await?; + self.connection_pools.lock().await.insert( + vault, + PoolWithTimestamp { + pool: pool.clone(), + last_accessed: Instant::now(), + }, + ); + Ok(pool) + } + + async fn existing_pool(&self, vault: &VaultId) -> Option> { + self.connection_pools + .lock() + .await .get_mut(vault) - .expect("Pool was just inserted or already exists"); - - // Update last accessed time - pool_with_timestamp.last_accessed = Instant::now(); - - Ok(pool_with_timestamp.pool.clone()) + .map(|entry| { + entry.last_accessed = Instant::now(); + entry.pool.clone() + }) } /// Attempting to write from this transaction might result in a @@ -166,313 +214,11 @@ impl Database { } pub async fn create_write_transaction(&self, vault: &VaultId) -> Result> { - let mut transaction = self.create_readonly_transaction(vault).await?; - - // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 - sqlx::query!("END; BEGIN IMMEDIATE;") - .execute(&mut *transaction) - .await?; - - Ok(transaction) - } - - /// Return the latest state of all documents in the vault - pub async fn get_latest_documents( - &self, - vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query!( - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - order by vault_update_id - "#, - ); - - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await - } else { - query - .fetch_all(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest documents") - .map(|rows| { - rows.into_iter() - .map(|row| DocumentVersionWithoutContent { - vault_update_id: row.vault_update_id, - document_id: row.document_id.into(), - relative_path: row.relative_path, - updated_date: row.updated_date, - is_deleted: row.is_deleted, - user_id: row.user_id, - device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), - }) - .collect() - }) - } - - /// Return the latest state of all documents (including deleted) in the - /// vault which have changed since the given update id - pub async fn get_latest_documents_since( - &self, - vault: &VaultId, - vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query!( - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - where vault_update_id > ? - order by vault_update_id - "#, - vault_update_id - ); - - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await - } else { - query - .fetch_all(&self.get_connection_pool(vault).await?) - .await - } - .with_context(|| { - format!("Cannot fetch latest documents since vault_update_id `{vault_update_id}`") - }) - .map(|rows| { - rows.into_iter() - .map(|row| DocumentVersionWithoutContent { - vault_update_id: row.vault_update_id, - document_id: row.document_id.into(), - relative_path: row.relative_path, - updated_date: row.updated_date, - is_deleted: row.is_deleted, - user_id: row.user_id, - device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), - }) - .collect() - }) - } - - pub async fn get_max_update_id_in_vault( - &self, - vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result { - let query = sqlx::query!( - r#" - select coalesce(max(vault_update_id), 0) as max_vault_update_id - from documents - "#, - ); - - if let Some(transaction) = transaction { - query.fetch_one(&mut **transaction).await - } else { - query - .fetch_one(&self.get_connection_pool(vault).await?) - .await - } - .map(|row| row.max_vault_update_id) - .context("Cannot fetch max update id in vault") - } - - pub async fn get_latest_document_by_path( - &self, - vault: &VaultId, - relative_path: &str, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id, - has_been_merged - from latest_document_versions - where relative_path = ? and is_deleted = false - order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however, - -- multiple documents can have the same `relative_path`, if they have been deleted. That's - -- why we only care about the latest version of the document with the given relative path. - limit 1 - "#, - relative_path - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest document version") - } - - pub async fn get_latest_document( - &self, - vault: &VaultId, - document_id: &DocumentId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let document_id = document_id.as_hyphenated(); - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id, - has_been_merged - from latest_document_versions - where document_id = ? - "#, - document_id - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest document version") - } - - pub async fn get_document_version( - &self, - vault: &VaultId, - vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id, - has_been_merged - from documents - where vault_update_id = ?"#, - vault_update_id - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch document version") - } - - // inserting the document must be the last step of the transaction if there's one - pub async fn insert_document_version( - &self, - vault_id: &VaultId, - version: &StoredDocumentVersion, - transaction: Option>, - ) -> Result<()> { - let document_id = version.document_id.as_hyphenated(); - let query = sqlx::query!( - r#" - insert into documents ( - vault_update_id, - document_id, - relative_path, - updated_date, - content, - is_deleted, - user_id, - device_id - ) - values (?, ?, ?, ?, ?, ?, ?, ?) - "#, - version.vault_update_id, - document_id, - version.relative_path, - version.updated_date, - version.content, - version.is_deleted, - version.user_id, - version.device_id - ); - - if let Some(mut transaction) = transaction { - query - .execute(&mut *transaction) - .await - .context("Cannot insert document version")?; - - transaction - .commit() - .await - .context("Failed to commit transaction")?; - } else { - query - .execute(&self.get_connection_pool(vault_id).await?) - .await - .context("Cannot insert document version")?; - } - - self.broadcasts - .send_document_update( - vault_id.clone(), - WebSocketServerMessageWithOrigin::with_origin( - version.device_id.clone(), - WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: vec![version.clone().into()], - is_initial_sync: false, - }), - ), - ) - .await; - - Ok(()) + self.get_connection_pool(vault) + .await? + .begin_with("BEGIN IMMEDIATE") + .await + .context("Cannot create write transaction") } /// Cleanup idle connection pools that haven't been accessed in more than 5 minutes @@ -486,16 +232,22 @@ impl Database { .iter() .filter(|(_, pool_with_timestamp)| { now.duration_since(pool_with_timestamp.last_accessed) > idle_timeout + && pool_with_timestamp.pool.num_idle() + == pool_with_timestamp.pool.size() as usize }) .map(|(vault_id, _)| vault_id.clone()) .collect(); - // Close and remove idle pools - for vault_id in &vaults_to_remove { - if let Some(pool_with_timestamp) = pools.remove(vault_id) { - info!("Closing idle database connection pool for vault `{vault_id}`"); - pool_with_timestamp.pool.close().await; - } + let closing: Vec<_> = vaults_to_remove + .into_iter() + .filter_map(|vault| pools.remove(&vault).map(|entry| (vault, entry.pool))) + .collect(); + drop(pools); + // A checkout racing cleanup can delay close. It must never retain the + // mutex used to access every other vault in the process. + for (vault, pool) in closing { + info!("Closing idle database connection pool for vault `{vault}`"); + pool.close().await; } } diff --git a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql index 4a9f31ba..e5a86edc 100644 --- a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql +++ b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql @@ -1,21 +1,33 @@ -CREATE TABLE IF NOT EXISTS documents ( - vault_update_id INTEGER NOT NULL PRIMARY KEY, - document_id TEXT NOT NULL, - relative_path TEXT NOT NULL, - updated_date TIMESTAMP NOT NULL, - content BLOB NOT NULL, - is_deleted BOOLEAN NOT NULL +CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(event_id <= 9007199254740991), -- JavaScript’s Number.MAX_SAFE_INTEGER + request_id TEXT NOT NULL UNIQUE, + request_fingerprint BLOB NOT NULL ); -CREATE VIEW IF NOT EXISTS latest_document_versions AS -SELECT d.* -FROM documents d -INNER JOIN ( - SELECT MAX(vault_update_id) AS max_version_id - FROM documents - GROUP BY document_id -) max_versions -ON d.vault_update_id = max_versions.max_version_id; +CREATE TABLE documents ( + vault_update_id INTEGER PRIMARY KEY REFERENCES events(event_id), + document_id TEXT NOT NULL, + updated_date TEXT NOT NULL, + content BLOB NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT NOT NULL +); -CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path -ON documents (relative_path); +CREATE INDEX documents_by_id ON documents(document_id); + +CREATE VIEW latest_document_versions AS + SELECT d.* FROM documents d JOIN ( + SELECT document_id, MAX(vault_update_id) AS version FROM documents GROUP BY document_id + ) heads ON d.vault_update_id = heads.version; + +CREATE TABLE file_manifests ( + file_manifest_id INTEGER PRIMARY KEY REFERENCES events(event_id) +); + +CREATE TABLE file_manifest_entries ( + file_manifest_id INTEGER NOT NULL REFERENCES file_manifests(file_manifest_id), + document_id TEXT NOT NULL, + path TEXT NOT NULL, + PRIMARY KEY (file_manifest_id, document_id), + UNIQUE (file_manifest_id, path) +); diff --git a/sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql b/sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql deleted file mode 100644 index 06860174..00000000 --- a/sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE documents ADD COLUMN user_id TEXT NOT NULL DEFAULT ""; -ALTER TABLE documents ADD COLUMN device_id TEXT NOT NULL DEFAULT ""; diff --git a/sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql b/sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql deleted file mode 100644 index 5259c175..00000000 --- a/sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql +++ /dev/null @@ -1,13 +0,0 @@ -ALTER TABLE documents ADD COLUMN has_been_merged BOOLEAN NOT NULL DEFAULT False; - -DROP VIEW latest_document_versions; - -CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field -SELECT d.* -FROM documents d -INNER JOIN ( - SELECT MAX(vault_update_id) AS max_version_id - FROM documents - GROUP BY document_id -) max_versions -ON d.vault_update_id = max_versions.max_version_id; diff --git a/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql b/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql new file mode 100644 index 00000000..dc96b259 --- /dev/null +++ b/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql @@ -0,0 +1,4 @@ +-- Tokens identify an event's incarnation, including when a restored backup +-- reuses its numeric ID or a client resubmits the same request UUID. +ALTER TABLE events ADD COLUMN event_token TEXT; +UPDATE events SET event_token = lower(hex(randomblob(16))); diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index a216125a..edb3f885 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -1,93 +1,131 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use chrono::{DateTime, Utc}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use ts_rs::TS; pub type VaultId = String; pub type VaultUpdateId = i64; - pub type DocumentId = uuid::Uuid; pub type UserId = String; pub type DeviceId = String; +pub type FileManifestEntries = BTreeMap; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, sqlx::FromRow)] pub struct StoredDocumentVersion { pub vault_update_id: VaultUpdateId, + #[sqlx(try_from = "uuid::fmt::Hyphenated")] pub document_id: DocumentId, - pub relative_path: String, pub updated_date: DateTime, pub content: Vec, - pub is_deleted: bool, pub user_id: UserId, pub device_id: DeviceId, - #[allow(dead_code)] // This is for manual analysis - pub has_been_merged: bool, } -impl PartialEq for StoredDocumentVersion { - fn eq(&self, other: &Self) -> bool { - self.vault_update_id == other.vault_update_id - } -} - -#[derive(TS, Debug, Clone, Serialize)] +#[derive(TS, Debug, Clone, Serialize, Deserialize, sqlx::FromRow)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct DocumentVersionWithoutContent { - #[ts(as = "i32")] + #[ts(as = "f64")] pub vault_update_id: VaultUpdateId, - + #[sqlx(try_from = "uuid::fmt::Hyphenated")] pub document_id: DocumentId, - pub relative_path: String, pub updated_date: DateTime, - pub is_deleted: bool, pub user_id: UserId, pub device_id: DeviceId, - - #[ts(as = "i32")] - pub content_size: u64, + #[sqlx(try_from = "i64")] + pub content_size: usize, } -impl From for DocumentVersionWithoutContent { - fn from(value: StoredDocumentVersion) -> Self { +impl From<&StoredDocumentVersion> for DocumentVersionWithoutContent { + fn from(v: &StoredDocumentVersion) -> Self { Self { - vault_update_id: value.vault_update_id, - document_id: value.document_id, - relative_path: value.relative_path, - updated_date: value.updated_date, - is_deleted: value.is_deleted, - user_id: value.user_id, - device_id: value.device_id, - content_size: value.content.len() as u64, + vault_update_id: v.vault_update_id, + document_id: v.document_id, + updated_date: v.updated_date, + user_id: v.user_id.clone(), + device_id: v.device_id.clone(), + content_size: v.content.len(), } } } +impl From for DocumentVersionWithoutContent { + fn from(v: StoredDocumentVersion) -> Self { + Self::from(&v) + } +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct DocumentVersion { + #[serde(flatten)] + pub metadata: DocumentVersionWithoutContent, + pub content_base64: String, +} +impl From for DocumentVersion { + fn from(v: StoredDocumentVersion) -> Self { + Self { + metadata: (&v).into(), + content_base64: STANDARD.encode(&v.content), + } + } +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct FileManifest { + #[ts(as = "f64")] + pub file_manifest_id: VaultUpdateId, + #[ts(type = "Record")] + pub entries: FileManifestEntries, +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(export)] +pub enum VaultEvent { + Content { + document: DocumentVersionWithoutContent, + }, + FileManifest { + #[serde(rename = "fileManifest")] + #[ts(rename = "fileManifest")] + file_manifest: FileManifest, + }, +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct EventRecord { + #[ts(as = "f64")] + pub event_id: VaultUpdateId, + pub request_id: uuid::Uuid, + #[serde(flatten)] + pub event: VaultEvent, +} #[derive(TS, Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct DocumentVersion { - #[ts(as = "i32")] - pub vault_update_id: VaultUpdateId, - - pub document_id: DocumentId, - pub relative_path: String, - pub updated_date: DateTime, - pub content_base64: String, - pub is_deleted: bool, - pub user_id: UserId, - pub device_id: DeviceId, +#[ts(export)] +pub struct VaultSnapshot { + #[ts(as = "f64")] + pub head_event_id: VaultUpdateId, + pub file_manifest: FileManifest, + pub documents: Vec, } -impl From for DocumentVersion { - fn from(value: StoredDocumentVersion) -> Self { - Self { - vault_update_id: value.vault_update_id, - document_id: value.document_id, - relative_path: value.relative_path, - updated_date: value.updated_date, - content_base64: STANDARD.encode(&value.content), - is_deleted: value.is_deleted, - user_id: value.user_id, - device_id: value.device_id, - } - } +#[derive(TS, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct EventBatch { + #[ts(as = "f64")] + pub head_event_id: VaultUpdateId, + /// Last event in this page; `head_event_id` is the vault's current head. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional, as = "Option")] + pub end_event_id: Option, + pub events: Vec, } diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs new file mode 100644 index 00000000..6585d102 --- /dev/null +++ b/sync-server/src/app_state/database/mutations.rs @@ -0,0 +1,63 @@ +use super::{ + Database, Transaction, + models::{FileManifest, StoredDocumentVersion, VaultUpdateId}, +}; +use anyhow::Result; + +impl Database { + pub async fn allocate_event( + tx: &mut Transaction<'_>, + request_id: uuid::Uuid, + fingerprint: &[u8], + ) -> Result { + Ok(sqlx::query( + "INSERT INTO events(request_id, request_fingerprint, event_token) VALUES (?, ?, ?)", + ) + .bind(request_id.to_string()) + .bind(fingerprint) + .bind(uuid::Uuid::new_v4().simple().to_string()) + .execute(&mut **tx) + .await? + .last_insert_rowid()) + } + + pub async fn insert_document_version( + tx: &mut Transaction<'_>, + version: &StoredDocumentVersion, + ) -> Result<()> { + sqlx::query("INSERT INTO documents(vault_update_id, document_id, updated_date, content, user_id, device_id) VALUES (?, ?, ?, ?, ?, ?)") + .bind(version.vault_update_id) + .bind(version.document_id.hyphenated()) + .bind(version.updated_date) + .bind(&version.content) + .bind(&version.user_id) + .bind(&version.device_id) + .execute(&mut **tx) + .await?; + + Ok(()) + } + + pub async fn insert_file_manifest( + tx: &mut Transaction<'_>, + file_manifest: &FileManifest, + ) -> Result<()> { + sqlx::query("INSERT INTO file_manifests(file_manifest_id) VALUES (?)") + .bind(file_manifest.file_manifest_id) + .execute(&mut **tx) + .await?; + + for (document_id, path) in &file_manifest.entries { + sqlx::query( + "INSERT INTO file_manifest_entries(file_manifest_id, document_id, path) VALUES (?, ?, ?)", + ) + .bind(file_manifest.file_manifest_id) + .bind(document_id.hyphenated()) + .bind(path) + .execute(&mut **tx) + .await?; + } + + Ok(()) + } +} diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs new file mode 100644 index 00000000..6afec8d7 --- /dev/null +++ b/sync-server/src/app_state/database/queries.rs @@ -0,0 +1,255 @@ +use super::{ + Database, Transaction, + models::{ + DocumentId, DocumentVersionWithoutContent, EventBatch, EventRecord, FileManifest, + StoredDocumentVersion, VaultEvent, VaultId, VaultSnapshot, VaultUpdateId, + }, +}; +use anyhow::{Context as _, Result, ensure}; + +impl Database { + pub async fn missing_document( + tx: &mut Transaction<'_>, + ids: &[DocumentId], + ) -> Result> { + // A single covering-index lookup per requested identity; never load a + // content blob or decode a version while validating manifest references. + Ok(sqlx::query_scalar( + "SELECT value FROM json_each(?) AS requested + WHERE NOT EXISTS (SELECT 1 FROM documents WHERE document_id = requested.value) + LIMIT 1", + ) + .bind(serde_json::to_string(ids)?) + .fetch_optional(&mut **tx) + .await?) + } + + pub async fn get_request_event( + tx: &mut Transaction<'_>, + request_id: uuid::Uuid, + ) -> Result, VaultEvent)>> { + let row: Option<(Vec, VaultUpdateId)> = + sqlx::query_as("SELECT request_fingerprint, event_id FROM events WHERE request_id = ?") + .bind(request_id.to_string()) + .fetch_optional(&mut **tx) + .await?; + match row { + Some((fingerprint, event_id)) => { + Ok(Some((fingerprint, Self::event_by_id(tx, event_id).await?))) + } + None => Ok(None), + } + } + + pub async fn get_latest_document_version( + &self, + vault: &VaultId, + id: &DocumentId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query_as::<_, StoredDocumentVersion>( + "SELECT * FROM documents WHERE document_id = ? ORDER BY vault_update_id DESC LIMIT 1", + ) + .bind(id.hyphenated()); + + Ok(if let Some(tx) = transaction { + query.fetch_optional(&mut **tx).await? + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await? + }) + } + + pub async fn get_document_version( + &self, + vault: &VaultId, + version: VaultUpdateId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query_as::<_, StoredDocumentVersion>( + "SELECT * FROM documents WHERE vault_update_id = ?", + ) + .bind(version); + + Ok(if let Some(tx) = transaction { + query.fetch_optional(&mut **tx).await? + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await? + }) + } + + pub async fn current_file_manifest(tx: &mut Transaction<'_>) -> Result { + let Some(file_manifest_id) = sqlx::query_scalar::<_, VaultUpdateId>( + "SELECT file_manifest_id FROM file_manifests ORDER BY file_manifest_id DESC LIMIT 1", + ) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(FileManifest::default()); + }; + + Self::file_manifest_by_id(tx, file_manifest_id) + .await? + .context("Latest file manifest is missing") + } + + async fn file_manifest_by_id( + tx: &mut Transaction<'_>, + file_manifest_id: VaultUpdateId, + ) -> Result> { + let exists = sqlx::query_scalar::<_, VaultUpdateId>( + "SELECT file_manifest_id FROM file_manifests WHERE file_manifest_id = ?", + ) + .bind(file_manifest_id) + .fetch_optional(&mut **tx) + .await?; + if exists.is_none() { + return Ok(None); + } + let entries = sqlx::query_as::<_, (uuid::fmt::Hyphenated, String)>( + "SELECT document_id, path FROM file_manifest_entries WHERE file_manifest_id = ?", + ) + .bind(file_manifest_id) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|(document_id, path)| (document_id.into_uuid(), path)) + .collect(); + + Ok(Some(FileManifest { + file_manifest_id, + entries, + })) + } + + async fn event_by_id(tx: &mut Transaction<'_>, event_id: VaultUpdateId) -> Result { + let document = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT vault_update_id, document_id, updated_date, user_id, device_id, + length(content) AS content_size + FROM documents WHERE vault_update_id = ?", + ) + .bind(event_id) + .fetch_optional(&mut **tx) + .await?; + if let Some(document) = document { + return Ok(VaultEvent::Content { document }); + } + let file_manifest = Self::file_manifest_by_id(tx, event_id) + .await? + .context("Event has no normalized content or file manifest")?; + Ok(VaultEvent::FileManifest { file_manifest }) + } + + pub async fn latest_event_id(tx: &mut Transaction<'_>) -> Result { + Ok( + sqlx::query_scalar("SELECT COALESCE(MAX(event_id), 0) FROM events") + .fetch_one(&mut **tx) + .await?, + ) + } + + pub async fn vault_snapshot(&self, vault: &VaultId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let head_event_id = Self::latest_event_id(&mut tx).await?; + let file_manifest = Self::current_file_manifest(&mut tx).await?; + + let documents = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT d.vault_update_id, d.document_id, d.updated_date, d.user_id, d.device_id, + length(d.content) AS content_size + FROM latest_document_versions d + JOIN file_manifest_entries m ON m.document_id = d.document_id + WHERE m.file_manifest_id = ? ORDER BY d.vault_update_id", + ) + .bind(file_manifest.file_manifest_id) + .fetch_all(&mut *tx) + .await?; + + Ok(VaultSnapshot { + head_event_id, + file_manifest, + documents, + }) + } + + pub async fn events_after(&self, vault: &VaultId, after: VaultUpdateId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let head_event_id = Self::latest_event_id(&mut tx).await?; + + ensure!( + (0..=head_event_id).contains(&after), + "Event cursor is outside this vault's history" + ); + + let rows: Vec<(VaultUpdateId, String)> = sqlx::query_as( + "SELECT event_id, request_id FROM events WHERE event_id > ? ORDER BY event_id LIMIT 64", + ) + .bind(after) + .fetch_all(&mut *tx) + .await?; + + let mut events = Vec::with_capacity(rows.len()); + let mut bytes = 0; + let mut end_event_id = after; + for (event_id, request_id) in rows { + let event = EventRecord { + event_id, + request_id: request_id.parse()?, + event: Self::event_by_id(&mut tx, event_id).await?, + }; + let size = serde_json::to_vec(&event)?.len(); + // One manifest is atomic and may exceed this budget. Always allow + // one event so even such a page advances; never accumulate history. + if !events.is_empty() && bytes + size > 1024 * 1024 { + break; + } + bytes += size; + end_event_id = event_id; + events.push(event); + } + + Ok(EventBatch { + head_event_id, + end_event_id: Some(end_event_id), + events, + }) + } +} + +impl Database { + pub async fn history_checkpoint(&self, vault: &VaultId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let row: Option<(i64, String)> = sqlx::query_as( + "SELECT event_id, event_token FROM events ORDER BY event_id DESC LIMIT 1", + ) + .fetch_optional(&mut *tx) + .await?; + Ok(row.map_or_else( + || "0:empty".to_owned(), + |(id, token)| format!("{id}:{token}"), + )) + } + + pub async fn contains_checkpoint(&self, vault: &VaultId, checkpoint: &str) -> Result { + if checkpoint == "0:empty" { + return Ok(true); + } + let Some((id, token)) = checkpoint.split_once(':') else { + return Ok(false); + }; + let Ok(id) = id.parse::() else { + return Ok(false); + }; + let mut tx = self.create_readonly_transaction(vault).await?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM events WHERE event_id = ? AND event_token = ?)", + ) + .bind(id) + .bind(token) + .fetch_one(&mut *tx) + .await?; + Ok(exists) + } +} diff --git a/sync-server/src/app_state/database/tests.rs b/sync-server/src/app_state/database/tests.rs new file mode 100644 index 00000000..b3578f46 --- /dev/null +++ b/sync-server/src/app_state/database/tests.rs @@ -0,0 +1,405 @@ +use super::*; +use models::{DocumentId, FileManifest, StoredDocumentVersion}; + +fn config(directory: &tempfile::TempDir) -> DatabaseConfig { + DatabaseConfig { + databases_directory_path: directory.path().to_owned(), + ..Default::default() + } +} + +async fn insert_content( + database: &Database, + vault: &str, + document_id: DocumentId, + content: &[u8], +) -> StoredDocumentVersion { + let mut tx = database + .create_write_transaction(&vault.to_owned()) + .await + .unwrap(); + let request_id = uuid::Uuid::new_v4(); + let version = StoredDocumentVersion { + vault_update_id: Database::allocate_event(&mut tx, request_id, b"test") + .await + .unwrap(), + document_id, + updated_date: chrono::Utc::now(), + content: content.to_vec(), + user_id: "user".to_owned(), + device_id: "device".to_owned(), + }; + Database::insert_document_version(&mut tx, &version) + .await + .unwrap(); + tx.commit().await.unwrap(); + version +} + +#[tokio::test] +async fn unicode_vaults_remain_isolated_after_reopening() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + let document = uuid::Uuid::new_v4(); + let names = ["café", "cafe\u{301}", "σ", "ς"]; + + for name in names { + assert!( + database + .get_latest_document_version(&name.to_owned(), &document, None) + .await + .unwrap() + .is_none() + ); + insert_content(&database, name, document, name.as_bytes()).await; + } + + let reopened = Database::try_new(&config).await.unwrap(); + for name in names { + let version = reopened + .get_latest_document_version(&name.to_owned(), &document, None) + .await + .unwrap() + .unwrap(); + assert_eq!(version.content, name.as_bytes()); + } +} + +#[tokio::test] +async fn rejects_invalid_vault_names_at_the_database_boundary() { + let directory = tempfile::tempdir().unwrap(); + let config = DatabaseConfig { + databases_directory_path: directory.path().join("databases"), + ..Default::default() + }; + let database = Database::try_new(&config).await.unwrap(); + for name in [ + "", + " ", + ".", + "..", + "../escaped", + "/absolute", + "a/b", + "a\\b", + "a\0b", + ] { + assert!( + database.vault_snapshot(&name.to_owned()).await.is_err(), + "accepted {name:?}" + ); + } + assert!(!directory.path().join("escaped.sqlite").exists()); + assert!(database.connection_pools.lock().await.is_empty()); +} + +#[tokio::test] +async fn migrations_can_be_reapplied_without_losing_requests_or_content() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + let vault = "migrations".to_owned(); + let version = insert_content(&database, &vault, uuid::Uuid::new_v4(), b"only copy").await; + let pool = database.get_connection_pool(&vault).await.unwrap(); + Database::run_migrations(&pool).await.unwrap(); + Database::run_migrations(&pool).await.unwrap(); + let reopened = Database::try_new(&config).await.unwrap(); + assert_eq!( + reopened + .get_document_version(&vault, version.vault_update_id, None) + .await + .unwrap() + .unwrap() + .content, + b"only copy" + ); + assert_eq!( + reopened.events_after(&vault, 0).await.unwrap().events.len(), + 1 + ); +} + +#[tokio::test] +async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "snapshot".to_owned(); + let text_id = uuid::Uuid::new_v4(); + let binary_id = uuid::Uuid::new_v4(); + insert_content(&database, &vault, text_id, b"old").await; + let binary = insert_content(&database, &vault, binary_id, &[0, 255, 0, 128]).await; + let text = insert_content(&database, &vault, text_id, "café 🌍".as_bytes()).await; + insert_content(&database, &vault, uuid::Uuid::new_v4(), b"unreferenced").await; + + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + let request_id = uuid::Uuid::new_v4(); + let manifest = FileManifest { + file_manifest_id: Database::allocate_event(&mut tx, request_id, b"manifest") + .await + .unwrap(), + entries: [ + (text_id, "text.md".to_owned()), + (binary_id, "binary.dat".to_owned()), + ] + .into(), + }; + Database::insert_file_manifest(&mut tx, &manifest) + .await + .unwrap(); + tx.commit().await.unwrap(); + + let snapshot = database.vault_snapshot(&vault).await.unwrap(); + assert_eq!(snapshot.head_event_id, manifest.file_manifest_id); + assert_eq!(snapshot.file_manifest.entries, manifest.entries); + assert_eq!(snapshot.documents.len(), 2); + for (actual, expected) in snapshot.documents.iter().zip([binary, text]) { + assert_eq!( + serde_json::to_value(actual).unwrap(), + serde_json::to_value(models::DocumentVersionWithoutContent::from(expected)).unwrap() + ); + } +} + +#[tokio::test] +async fn replay_pages_are_bounded_and_cover_every_event_in_order() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "pages".to_owned(); + let id = uuid::Uuid::new_v4(); + for _ in 0..150 { + insert_content(&database, &vault, id, b"edit").await; + } + let mut after = 0; + while after < 150 { + let batch = database.events_after(&vault, after).await.unwrap(); + assert!(batch.events.len() <= 64, "catchup must bound each page"); + assert!(!batch.events.is_empty()); + assert_eq!(batch.head_event_id, 150); + for event in batch.events { + assert_eq!(event.event_id, after + 1); + after = event.event_id; + } + } + assert!( + database + .events_after(&vault, after) + .await + .unwrap() + .events + .is_empty() + ); +} + +#[tokio::test] +async fn idle_cleanup_does_not_wait_for_an_active_vault() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + database.vault_snapshot(&"other".to_owned()).await.unwrap(); + let vault = "slow".to_owned(); + let transaction = database.create_write_transaction(&vault).await.unwrap(); + database + .connection_pools + .lock() + .await + .get_mut(&vault) + .unwrap() + .last_accessed = Instant::now() - Duration::from_secs(301); + tokio::time::timeout(Duration::from_millis(200), database.cleanup_idle_pools()) + .await + .expect("cleanup must leave an active pool alone"); + let other = tokio::time::timeout( + Duration::from_millis(200), + database.vault_snapshot(&"other".to_owned()), + ) + .await + .expect("other vaults must remain accessible") + .unwrap(); + assert_eq!(other.head_event_id, 0); + assert!( + !database + .get_connection_pool(&vault) + .await + .unwrap() + .is_closed() + ); + drop(transaction); +} + +#[tokio::test] +async fn manifest_existence_checks_are_independent_of_payload_decoding() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "existence".to_owned(); + let id = uuid::Uuid::new_v4(); + insert_content(&database, &vault, id, b"content").await; + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + // Make full-row deserialization fail: checking a foreign identity must not + // decode unrelated version fields or read its potentially enormous blob. + sqlx::query("UPDATE documents SET updated_date = 'unreadable timestamp'") + .execute(&mut *tx) + .await + .unwrap(); + assert_eq!( + Database::missing_document(&mut tx, &[id]).await.unwrap(), + None + ); + let missing = uuid::Uuid::new_v4(); + assert_eq!( + Database::missing_document(&mut tx, &[id, missing]) + .await + .unwrap(), + Some(missing.to_string()) + ); + assert_eq!( + Database::missing_document(&mut tx, &[]).await.unwrap(), + None + ); +} + +#[tokio::test] +async fn replay_byte_budget_prevents_accumulating_large_historical_manifests() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "large-manifests".to_owned(); + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + // Each event fits independently, but fewer than 64 fit the response budget. + let entries: std::collections::BTreeMap<_, _> = (0..1500) + .map(|index| { + ( + uuid::Uuid::new_v4(), + format!("{index}-{}.md", "x".repeat(180)), + ) + }) + .collect(); + for _ in 0..8 { + let file_manifest_id = Database::allocate_event(&mut tx, uuid::Uuid::new_v4(), b"test") + .await + .unwrap(); + Database::insert_file_manifest( + &mut tx, + &FileManifest { + file_manifest_id, + entries: entries.clone(), + }, + ) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + let batch = database.events_after(&vault, 0).await.unwrap(); + assert!(!batch.events.is_empty()); + assert!(batch.events.len() < 8); + assert!(serde_json::to_vec(&batch).unwrap().len() < 1024 * 1024 + 1024); + assert_eq!( + batch.end_event_id, + batch.events.last().map(|event| event.event_id) + ); +} + +#[tokio::test] +async fn opening_a_locked_vault_does_not_block_unrelated_vaults() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + database.vault_snapshot(&"other".to_owned()).await.unwrap(); + let vault = "locked-opening".to_owned(); + // A second process holds the writer lock while this process lazily checks + // migrations. Even already-open, unrelated vaults must remain available. + let external = Database::open_database(&config, &Database::database_path(&config, &vault)) + .await + .unwrap(); + let transaction = external.begin_with("BEGIN IMMEDIATE").await.unwrap(); + let opening_database = database.clone(); + let opening = tokio::spawn(async move { opening_database.get_connection_pool(&vault).await }); + tokio::time::sleep(Duration::from_millis(100)).await; + let opening_is_blocked = !opening.is_finished(); + let other = tokio::time::timeout( + Duration::from_millis(300), + database.vault_snapshot(&"other".to_owned()), + ) + .await; + transaction.rollback().await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), opening) + .await + .unwrap() + .unwrap() + .unwrap(); + external.close().await; + assert!( + opening_is_blocked, + "the test must exercise an in-progress database open" + ); + assert_eq!( + other + .expect("a blocked migration must not lock every vault") + .unwrap() + .head_event_id, + 0 + ); +} + +#[tokio::test] +async fn concurrent_vault_opens_share_a_pool_and_cancelled_opens_can_retry() { + let directory = tempfile::tempdir().unwrap(); + let mut config = config(&directory); + config.max_connections_per_vault = 1; + let database = Database::try_new(&config).await.unwrap(); + let vault = "cancelled-opening".to_owned(); + let external = Database::open_database(&config, &Database::database_path(&config, &vault)) + .await + .unwrap(); + let transaction = external.begin_with("BEGIN IMMEDIATE").await.unwrap(); + let opener = database.clone(); + let name = vault.clone(); + let cancelled = tokio::spawn(async move { opener.get_connection_pool(&name).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!cancelled.is_finished()); + cancelled.abort(); + assert!(cancelled.await.unwrap_err().is_cancelled()); + transaction.rollback().await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..8 { + let database = database.clone(); + let vault = vault.clone(); + tasks.push(tokio::spawn(async move { + database.get_connection_pool(&vault).await.unwrap() + })); + } + let pools = tokio::time::timeout(Duration::from_secs(5), futures::future::join_all(tasks)) + .await + .unwrap(); + let pools: Vec<_> = pools.into_iter().map(Result::unwrap).collect(); + let checked_out = pools[0].acquire().await.unwrap(); + for pool in &pools[1..] { + assert!( + tokio::time::timeout(Duration::from_millis(25), pool.acquire()) + .await + .is_err(), + "all callers must share the single configured connection" + ); + } + drop(checked_out); + external.close().await; +} + +#[tokio::test] +async fn one_corrupt_vault_does_not_prevent_startup_or_healthy_vault_access() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let bad = Database::database_path(&config, "broken"); + tokio::fs::create_dir_all(bad.parent().unwrap()) + .await + .unwrap(); + tokio::fs::write(&bad, b"not a SQLite database") + .await + .unwrap(); + let database = Database::try_new(&config).await.unwrap(); + assert!(database.vault_snapshot(&"broken".to_owned()).await.is_err()); + let saved = insert_content(&database, "healthy", uuid::Uuid::new_v4(), b"safe").await; + assert_eq!(saved.content, b"safe"); + assert_eq!( + tokio::fs::read(bad).await.unwrap(), + b"not a SQLite database" + ); +} diff --git a/sync-server/src/app_state/websocket/broadcasts.rs b/sync-server/src/app_state/websocket/broadcasts.rs index 60ae0219..0c5a8f83 100644 --- a/sync-server/src/app_state/websocket/broadcasts.rs +++ b/sync-server/src/app_state/websocket/broadcasts.rs @@ -1,69 +1,63 @@ +use super::models::CursorPositionFromServer; +use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig}; use std::{collections::HashMap, sync::Arc}; - -use anyhow::Context; -use log::{debug, warn}; use tokio::sync::{Mutex, broadcast}; -use super::models::WebSocketServerMessageWithOrigin; -use crate::{ - app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error, -}; +#[derive(Debug, Clone)] +pub enum Notification { + VaultUpdate, + Cursors(CursorPositionFromServer), +} #[derive(Debug, Clone)] pub struct Broadcasts { - max_clients_per_vault: usize, - tx: Arc>>>, + capacity: usize, + admission: Arc>>>, + senders: Arc>>>, } impl Broadcasts { - pub fn new(server_config: &ServerConfig) -> Self { + pub fn new(config: &ServerConfig) -> Self { Self { - max_clients_per_vault: server_config.max_clients_per_vault, - tx: Arc::new(Mutex::new(HashMap::new())), + capacity: config.max_clients_per_vault, + admission: Arc::default(), + senders: Arc::default(), } } - pub async fn get_receiver( - &self, - vault: VaultId, - ) -> broadcast::Receiver { - let tx = self.get_or_create(vault).await; - - tx.subscribe() + pub fn try_admit(&self, vault: &VaultId) -> Option { + let mut limits = self.admission.lock().unwrap(); + limits.retain(|_, limit| limit.strong_count() > 0); + let limit = limits + .get(vault) + .and_then(std::sync::Weak::upgrade) + .unwrap_or_else(|| { + let limit = Arc::new(tokio::sync::Semaphore::new(self.capacity)); + limits.insert(vault.clone(), Arc::downgrade(&limit)); + limit + }); + limit.try_acquire_owned().ok() } - /// Notify all clients (who are subscribed to the vault) about an update. - /// We only log failures and don't propagate them. - pub async fn send_document_update( - &self, - vault: VaultId, - document: WebSocketServerMessageWithOrigin, - ) { - let tx = self.get_or_create(vault.clone()).await; - - if tx.receiver_count() == 0 { - debug!("Skipping broadcast, no clients connected for vault `{vault}`"); - return; - } - - let result = tx - .send(document) - .context("Cannot broadcast server message to websocket listeners") - .map_err(server_error); - - if result.is_err() { - warn!("Failed to send message: {result:?}"); - } - } - - async fn get_or_create( - &self, - vault: VaultId, - ) -> broadcast::Sender { - let mut tx = self.tx.lock().await; - - tx.entry(vault) - .or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone()) + async fn sender(&self, vault: VaultId) -> broadcast::Sender { + self.senders + .lock() + .await + .entry(vault) + .or_insert_with(|| broadcast::channel(self.capacity.max(1)).0) .clone() } + + pub async fn get_receiver(&self, vault: VaultId) -> broadcast::Receiver { + self.sender(vault).await.subscribe() + } + + pub async fn send(&self, vault: VaultId, notification: Notification) { + // No listeners (or lagging listeners) is normal; they replay the log. + let _ = self.sender(vault).await.send(notification); + } + + pub async fn notify_about_vault_update(&self, vault: VaultId) { + self.send(vault, Notification::VaultUpdate).await; + } } diff --git a/sync-server/src/app_state/websocket/models.rs b/sync-server/src/app_state/websocket/models.rs index e037fb7e..4eb1fa10 100644 --- a/sync-server/src/app_state/websocket/models.rs +++ b/sync-server/src/app_state/websocket/models.rs @@ -1,18 +1,13 @@ use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::app_state::database::models::{ - DeviceId, DocumentId, DocumentVersionWithoutContent, VaultUpdateId, -}; +use crate::app_state::database::models::{DeviceId, DocumentId, VaultUpdateId}; #[derive(TS, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct WebSocketHandshake { pub token: String, pub device_id: DeviceId, - - #[ts(as = "Option")] - pub last_seen_vault_update_id: Option, } #[derive(TS, Deserialize, Clone, Debug)] @@ -57,13 +52,6 @@ pub struct CursorPositionFromServer { pub clients: Vec, } -#[derive(TS, Serialize, Clone, Debug)] -#[serde(rename_all = "camelCase")] -pub struct WebSocketVaultUpdate { - pub documents: Vec, - pub is_initial_sync: bool, -} - #[derive(TS, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase", tag = "type")] #[ts(export)] @@ -76,28 +64,6 @@ pub enum WebSocketClientMessage { #[serde(rename_all = "camelCase", tag = "type")] #[ts(export)] pub enum WebSocketServerMessage { - VaultUpdate(WebSocketVaultUpdate), + VaultChanged, CursorPositions(CursorPositionFromServer), } - -#[derive(Clone, Debug)] -pub struct WebSocketServerMessageWithOrigin { - pub origin_device_id: Option, - pub message: WebSocketServerMessage, -} - -impl WebSocketServerMessageWithOrigin { - pub fn new(message: WebSocketServerMessage) -> Self { - Self { - origin_device_id: None, - message, - } - } - - pub fn with_origin(origin_device_id: DeviceId, message: WebSocketServerMessage) -> Self { - Self { - origin_device_id: Some(origin_device_id), - message, - } - } -} diff --git a/sync-server/src/app_state/websocket/utils.rs b/sync-server/src/app_state/websocket/utils.rs index 1e0dd243..a1423e09 100644 --- a/sync-server/src/app_state/websocket/utils.rs +++ b/sync-server/src/app_state/websocket/utils.rs @@ -4,10 +4,7 @@ use futures::{sink::SinkExt, stream::SplitSink}; use super::models::{WebSocketClientMessage, WebSocketHandshake, WebSocketServerMessage}; use crate::{ - app_state::{ - AppState, - database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId}, - }, + app_state::{AppState, database::models::VaultId}, config::user_config::User, errors::{SyncServerError, server_error, unauthenticated_error}, server::auth::auth, @@ -44,26 +41,6 @@ pub fn get_authenticated_handshake( } } -pub async fn get_unseen_documents( - state: &AppState, - vault_id: &VaultId, - last_seen_vault_update_id: Option, -) -> Result, SyncServerError> { - if let Some(update_id) = last_seen_vault_update_id { - state - .database - .get_latest_documents_since(vault_id, update_id, None) - .await - .map_err(server_error) - } else { - state - .database - .get_latest_documents(vault_id, None) - .await - .map_err(server_error) - } -} - pub async fn send_update_over_websocket( update: &WebSocketServerMessage, sender: &mut SplitSink, @@ -72,9 +49,13 @@ pub async fn send_update_over_websocket( .context("Failed to serialize update") .map_err(server_error)?; - sender - .send(Message::Text(serialized_update)) - .await - .context("Failed to send message over websocket") - .map_err(server_error) + tokio::time::timeout( + std::time::Duration::from_secs(10), + sender.send(Message::Text(serialized_update)), + ) + .await + .context("WebSocket send deadline expired") + .map_err(server_error)? + .context("Failed to send message over websocket") + .map_err(server_error) } diff --git a/sync-server/src/config.rs b/sync-server/src/config.rs index 6a003d2e..c4246403 100644 --- a/sync-server/src/config.rs +++ b/sync-server/src/config.rs @@ -20,7 +20,6 @@ pub struct Config { pub database: DatabaseConfig, #[serde(default)] pub server: ServerConfig, - #[serde(default)] pub users: UserConfig, #[serde(default)] pub logging: LoggingConfig, @@ -28,23 +27,52 @@ pub struct Config { impl Config { pub async fn read_or_create(path: &Path) -> Result { - let config = if path.exists() { - info!( - "Loading configuration from `{}`", - path.canonicalize().unwrap().display() - ); - Self::load_from_file(path).await? - } else { - Self::default() - }; - - config.write(path).await?; - info!( - "Updated configuration at `{}`", - path.canonicalize().unwrap().display() - ); - - Ok(config) + match Self::load_from_file(path).await { + Ok(config) => return Ok(config), + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => {} + Err(error) => return Err(error), + } + let contents = serde_yaml::to_string(&Self::default())?; + let destination = path.to_owned(); + tokio::task::spawn_blocking(move || -> Result<()> { + use std::io::Write; + let path = destination; + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let temporary = parent.join(format!(".vault-link-config-{}.tmp", uuid::Uuid::new_v4())); + let result = (|| -> Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + file.write_all(contents.as_bytes())?; + file.sync_all()?; + // Publish a complete, durable file without replacing a config + // another process may have created while we generated defaults. + match std::fs::hard_link(&temporary, &path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })(); + let _ = std::fs::remove_file(&temporary); + result + }) + .await??; + info!("Loading configuration from `{}`", path.display()); + Self::load_from_file(path).await } pub async fn load_from_file(path: &Path) -> Result { @@ -57,12 +85,54 @@ impl Config { serde_yaml::from_str(&contents).context("Failed to parse configuration") } +} - pub async fn write(&self, path: &Path) -> Result<()> { - let contents = serde_yaml::to_string(&self).context("Failed to serialize configuration")?; +#[cfg(test)] +mod tests { + use super::*; - fs::write(path, contents) - .await - .context("Failed to write configuration to disk") + #[tokio::test] + async fn startup_preserves_existing_configuration_bytes_and_permissions() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + let original = b"# administrator's comments and formatting\nserver: {port: 9123}\nusers: {user_configs: []}\n"; + fs::write(&path, original).await.unwrap(); + let mut permissions = fs::metadata(&path).await.unwrap().permissions(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).await.unwrap(); + let config = Config::read_or_create(&path).await.unwrap(); + assert_eq!(config.server.port, 9123); + assert_eq!(fs::read(&path).await.unwrap(), original); + } + + #[tokio::test] + async fn existing_configuration_cannot_generate_unpersisted_credentials() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + for contents in ["server: {port: 9123}\n", "users: {}\n"] { + fs::write(&path, contents).await.unwrap(); + for _ in 0..2 { + assert!( + Config::read_or_create(&path).await.is_err(), + "Existing credentials must be explicit; random defaults are only for initialization" + ); + } + assert_eq!(fs::read_to_string(&path).await.unwrap(), contents); + } + } + + #[tokio::test] + async fn concurrent_initialization_never_overwrites_configuration() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + let results = + futures::future::join_all((0..16).map(|_| Config::read_or_create(&path))).await; + let config = Config::load_from_file(&path).await.unwrap(); + for result in results { + assert_eq!( + serde_yaml::to_string(&result.unwrap()).unwrap(), + serde_yaml::to_string(&config).unwrap() + ); + } } } diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index 4a9da0f4..fac5cf98 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -7,7 +7,7 @@ use crate::consts::{ DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS, }; -#[derive(Debug, Deserialize, Serialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct ServerConfig { #[serde(default = "default_host")] pub host: String, @@ -28,6 +28,19 @@ pub struct ServerConfig { pub mergeable_file_extensions: Vec, } +impl Default for ServerConfig { + fn default() -> Self { + Self { + host: default_host(), + port: default_port(), + max_body_size_mb: default_max_body_size_mb(), + max_clients_per_vault: default_max_clients_per_vault(), + response_timeout: default_response_timeout(), + mergeable_file_extensions: default_mergeable_file_extensions(), + } + } +} + fn default_host() -> String { debug!("Using default server host: {DEFAULT_HOST}"); DEFAULT_HOST.to_owned() @@ -60,3 +73,37 @@ fn default_mergeable_file_extensions() -> Vec { .map(|s| (*s).to_owned()) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + + #[tokio::test] + async fn fresh_and_omitted_server_configs_use_the_documented_defaults() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.yml"); + let fresh = Config::read_or_create(&path).await.unwrap(); + let reread = Config::load_from_file(&path).await.unwrap(); + let omitted: Config = serde_yaml::from_str("users: {user_configs: []}").unwrap(); + let explicit: Config = + serde_yaml::from_str("server: {}\nusers: {user_configs: []}").unwrap(); + for config in [fresh, reread, omitted, explicit] { + assert_eq!(config.server.host, DEFAULT_HOST); + assert_eq!(config.server.port, DEFAULT_PORT); + assert_eq!(config.server.max_body_size_mb, DEFAULT_MAX_BODY_SIZE_MB); + assert_eq!( + config.server.max_clients_per_vault, + DEFAULT_MAX_CLIENTS_PER_VAULT + ); + assert_eq!( + config.server.response_timeout, + DEFAULT_RESPONSE_TIMEOUT_SECONDS + ); + assert_eq!( + config.server.mergeable_file_extensions, + DEFAULT_MERGEABLE_FILE_EXTENSIONS + ); + } + } +} diff --git a/sync-server/src/config/user_config.rs b/sync-server/src/config/user_config.rs index 8b2537f0..985a5c2a 100644 --- a/sync-server/src/config/user_config.rs +++ b/sync-server/src/config/user_config.rs @@ -2,11 +2,14 @@ use bimap::BiHashMap; use rand::{Rng, distr::Alphanumeric, rng}; use serde::{Deserialize, Deserializer, Serialize, de::Error}; -use crate::app_state::database::models::VaultId; +use crate::{ + app_state::database::models::VaultId, + utils::normalize_vault_id::{normalize_string, validate_vault_id}, +}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct UserConfig { - #[serde(default = "default_users", deserialize_with = "validate_users")] + #[serde(deserialize_with = "validate_users")] pub user_configs: Vec, } @@ -71,9 +74,23 @@ pub enum VaultAccess { #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct AllowListedVaults { + #[serde(deserialize_with = "deserialize_allowed_vaults")] pub allowed: Vec, } +fn deserialize_allowed_vaults<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Vec::::deserialize(deserializer)? + .into_iter() + .map(|name| { + let vault = normalize_string(&name); + validate_vault_id(&vault).map_err(D::Error::custom)?; + Ok(vault) + }) + .collect() +} + fn default_users() -> Vec { vec![User { name: "admin".to_owned(), @@ -95,6 +112,24 @@ mod tests { use super::*; + #[test] + fn allowlists_normalize_case_and_whitespace_but_preserve_unicode_identity() { + let list: AllowListedVaults = serde_json::from_value(json!({ + "allowed": [" MyVault ", "CAFÉ", "CAFE\u{301}"] + })) + .unwrap(); + assert_eq!(list.allowed, ["myvault", "café", "cafe\u{301}"]); + } + + #[test] + fn allowlists_reject_invalid_vault_names() { + for name in ["", " ", ".", "..", "../escaped", "a/b", "a\\b", "a\0b"] { + assert!( + serde_json::from_value::(json!({"allowed": [name]})).is_err() + ); + } + } + #[test] fn test_validate_users_unique_names_and_tokens() { let config_json = json!({ diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index 98ed1c1f..e1fee209 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -20,4 +20,4 @@ pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; -pub const SUPPORTED_API_VERSION: u32 = 2; +pub const SUPPORTED_API_VERSION: u32 = 5; diff --git a/sync-server/src/errors.rs b/sync-server/src/errors.rs index 831b0e86..c505b8ae 100644 --- a/sync-server/src/errors.rs +++ b/sync-server/src/errors.rs @@ -5,7 +5,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use log::{debug, error}; +use log::debug; use serde::Serialize; use thiserror::Error; use ts_rs::TS; diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 01b09cf6..4df81e76 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -1,27 +1,20 @@ pub mod auth; -mod create_document; -mod delete_document; mod device_id_header; -mod fetch_document_version; -mod fetch_document_version_content; -mod fetch_latest_document_version; -mod fetch_latest_documents; -mod index; -mod ping; -mod requests; -mod responses; -mod update_document; -mod websocket; +mod endpoints; +pub(crate) mod requests; +pub(crate) mod responses; -use anyhow::{Context as _, Result, anyhow}; +#[cfg(test)] +mod tests; + +use anyhow::{Context as _, Result}; use auth::auth_middleware; use axum::{ Router, extract::{DefaultBodyLimit, Request}, http::{self, HeaderValue, Method}, middleware, - response::IntoResponse, - routing::{IntoMakeService, delete, get, post, put}, + routing::{IntoMakeService, get}, }; use device_id_header::DEVICE_ID_HEADER_NAME; use log::info; @@ -41,7 +34,6 @@ use tracing::{Level, info_span}; use crate::{ app_state::AppState, config::{Config, server_config::ServerConfig}, - errors::{client_error, not_found_error}, }; pub async fn create_server(config: Config) -> Result<()> { @@ -53,9 +45,12 @@ pub async fn create_server(config: Config) -> Result<()> { let app = Router::new() .nest("/", get_authed_routes(app_state.clone())) - .route("/", get(index::index)) - .route("/vaults/:vault_id/ping", get(ping::ping)) - .route("/vaults/:vault_id/ws", get(websocket::websocket_handler)) + .route("/", get(endpoints::index::index)) + .route("/vaults/:vault_id/ping", get(endpoints::ping::ping)) + .route( + "/vaults/:vault_id/ws", + get(endpoints::websocket::websocket_handler), + ) .layer(DefaultBodyLimit::disable()) .layer(RequestBodyLimitLayer::new( app_state.config.server.max_body_size_mb * 1024 * 1024, @@ -68,8 +63,13 @@ pub async fn create_server(config: Config) -> Result<()> { http::header::CONTENT_TYPE, http::header::AUTHORIZATION, DEVICE_ID_HEADER_NAME.clone(), + http::header::HeaderName::from_static("x-vault-link-history"), ]) - .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]), + .expose_headers([ + http::header::HeaderName::from_static("x-vault-link-history"), + http::header::HeaderName::from_static("x-vault-link-history-mismatch"), + ]) + .allow_methods([Method::GET, Method::PUT]), ) .layer( TraceLayer::new_for_http() @@ -91,8 +91,8 @@ pub async fn create_server(config: Config) -> Result<()> { .on_failure(DefaultOnFailure::new().level(Level::ERROR)), ) .with_state(app_state) - .fallback(handle_404) - .fallback(handle_405) + .fallback(endpoints::not_found::not_found) + .method_not_allowed_fallback(endpoints::method_not_allowed::method_not_allowed) .into_make_service(); start_server(app, &server_config).await @@ -101,36 +101,30 @@ pub async fn create_server(config: Config) -> Result<()> { fn get_authed_routes(app_state: AppState) -> Router { Router::new() .route( - "/vaults/:vault_id/documents", - get(fetch_latest_documents::fetch_latest_documents), + "/vaults/:vault_id/file-manifest", + get(endpoints::get_file_manifest::get_file_manifest) + .put(endpoints::put_file_manifest::put_file_manifest), ) .route( - "/vaults/:vault_id/documents", - post(create_document::create_document), + "/vaults/:vault_id/vault-snapshot", + get(endpoints::vault_snapshot::vault_snapshot), + ) + .route( + "/vaults/:vault_id/events-since", + get(endpoints::events::events), ) .route( "/vaults/:vault_id/documents/:document_id", - get(fetch_latest_document_version::fetch_latest_document_version), + get(endpoints::fetch_latest_document_version::fetch_latest_document_version) + .put(endpoints::put_file_content::put_file_content), ) .route( - "/vaults/:vault_id/documents/:document_id/binary", - put(update_document::update_binary), - ) - .route( - "/vaults/:vault_id/documents/:document_id/text", - put(update_document::update_text), - ) - .route( - "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id", - get(fetch_document_version::fetch_document_version), + "/vaults/:vault_id/documents/:document_id/metadata", + get(endpoints::fetch_latest_document_version::fetch_latest_document_metadata), ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id/content", - get(fetch_document_version_content::fetch_document_version_content), - ) - .route( - "/vaults/:vault_id/documents/:document_id", - delete(delete_document::delete_document), + get(endpoints::fetch_document_version_content::fetch_document_version_content), ) .layer(middleware::from_fn_with_state(app_state, auth_middleware)) } @@ -178,11 +172,3 @@ async fn shutdown_signal() { () = terminate => {}, } } - -async fn handle_404() -> impl IntoResponse { - not_found_error(anyhow!("Page not found")) -} - -async fn handle_405() -> impl IntoResponse { - client_error(anyhow!("Method not allowed")) -} diff --git a/sync-server/src/server/auth.rs b/sync-server/src/server/auth.rs index e56f4acc..c7a49378 100644 --- a/sync-server/src/server/auth.rs +++ b/sync-server/src/server/auth.rs @@ -2,8 +2,9 @@ use std::collections::HashMap; use axum::{ extract::{Path, Request, State}, + http::{HeaderValue, StatusCode}, middleware::Next, - response::Response, + response::{IntoResponse, Response}, }; use axum_extra::{ TypedHeader, @@ -14,8 +15,10 @@ use log::info; use crate::{ app_state::{AppState, database::models::VaultId}, config::user_config::{AllowListedVaults, User, VaultAccess}, - errors::{SyncServerError, permission_denied_error, unauthenticated_error}, - utils::normalize::normalize_string, + errors::{ + SyncServerError, client_error, permission_denied_error, server_error, unauthenticated_error, + }, + utils::normalize_vault_id::{normalize_string, validate_vault_id}, }; pub async fn auth_middleware( @@ -34,12 +37,46 @@ pub async fn auth_middleware( let user = auth(&state, token, &vault_id)?; + // A restore happens while the server is stopped. Validate before executing + // any old request, including retries whose numeric parent IDs were reused. + if let Some(checkpoint) = req.headers().get("x-vault-link-history") { + let checkpoint = checkpoint + .to_str() + .map_err(|error| client_error(error.into()))?; + if !state + .database + .contains_checkpoint(&vault_id, checkpoint) + .await + .map_err(server_error)? + { + let mut response = (StatusCode::CONFLICT, "Server history changed").into_response(); + response.headers_mut().insert( + "x-vault-link-history-mismatch", + HeaderValue::from_static("1"), + ); + return Ok(response); + } + } + req.extensions_mut().insert(user); - Ok(next.run(req).await) + let mut response = next.run(req).await; + if response.status().is_success() { + let checkpoint = state + .database + .history_checkpoint(&vault_id) + .await + .map_err(server_error)?; + response.headers_mut().insert( + "x-vault-link-history", + HeaderValue::from_str(&checkpoint).map_err(|error| server_error(error.into()))?, + ); + } + Ok(response) } pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result { + validate_vault_id(vault_id).map_err(client_error)?; let user = state .config .users diff --git a/sync-server/src/server/create_document.rs b/sync-server/src/server/create_document.rs deleted file mode 100644 index 859c0db4..00000000 --- a/sync-server/src/server/create_document.rs +++ /dev/null @@ -1,109 +0,0 @@ -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use axum_typed_multipart::TypedMultipart; -use log::{debug, info}; -use serde::Deserialize; - -use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion}; -use crate::{ - app_state::{ - AppState, - database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, - }, - config::user_config::User, - errors::{SyncServerError, client_error, server_error}, - utils::{ - find_first_available_path::find_first_available_path, normalize::normalize, - sanitize_path::sanitize_path, - }, -}; - -#[derive(Deserialize)] -pub struct CreateDocumentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} - -/// Create a new document in case a document with the same doesn't exist -/// already. If a document with the same path exists, a new version is created -/// with their content merged. -#[axum::debug_handler] -pub async fn create_document( - Path(CreateDocumentPathParams { vault_id }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - TypedMultipart(request): TypedMultipart, -) -> Result, SyncServerError> { - debug!("Creating document in vault `{vault_id}`"); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; - - let document_id = match request.document_id { - Some(document_id) => { - let existing_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - if existing_version.is_some() { - return Err(client_error(anyhow::anyhow!( - "Document with the same ID `{document_id}` already exists" - ))); - } - - document_id - } - None => uuid::Uuid::new_v4(), - }; - - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - let sanitized_relative_path = sanitize_path(&request.relative_path); - let deduped_path = find_first_available_path( - &vault_id, - &sanitized_relative_path, - &state.database, - &mut transaction, - ) - .await - .map_err(server_error)?; - - if deduped_path != sanitized_relative_path { - info!( - "Document already exists at new location: `{sanitized_relative_path}` when trying to create it in vault `{vault_id}`, deconflicting by creating at `{deduped_path}`" - ); - } - - let new_version = StoredDocumentVersion { - vault_update_id: last_update_id + 1, - document_id, - relative_path: deduped_path, - content: request.content.contents.to_vec(), - updated_date: chrono::Utc::now(), - is_deleted: false, - user_id: user.name, - device_id: device_id.0, - has_been_merged: false, - }; - - state - .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) - .await - .map_err(server_error)?; - - Ok(Json(new_version.into())) -} diff --git a/sync-server/src/server/delete_document.rs b/sync-server/src/server/delete_document.rs deleted file mode 100644 index e126d6b5..00000000 --- a/sync-server/src/server/delete_document.rs +++ /dev/null @@ -1,96 +0,0 @@ -use anyhow::Context; -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use log::{debug, info}; -use serde::Deserialize; - -use super::{device_id_header::DeviceIdHeader, requests::DeleteDocumentVersion}; -use crate::{ - app_state::{ - AppState, - database::models::{ - DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, - }, - }, - config::user_config::User, - errors::{SyncServerError, server_error}, - utils::{normalize::normalize, sanitize_path::sanitize_path}, -}; - -#[derive(Deserialize)] -pub struct DeleteDocumentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, -} - -#[axum::debug_handler] -pub async fn delete_document( - Path(DeleteDocumentPathParams { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - Json(request): Json, -) -> Result, SyncServerError> { - debug!("Deleting document `{document_id}` in vault `{vault_id}`"); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; - - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - let latest_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - if let Some(latest_version) = &latest_version - && latest_version.is_deleted - { - transaction - .rollback() - .await - .context("Failed to roll back transaction") - .map_err(server_error)?; - - info!("Document `{document_id}` has already been deleted",); - return Ok(Json(latest_version.clone().into())); - } - - let latest_content = latest_version.map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it - - let new_version = StoredDocumentVersion { - vault_update_id: last_update_id + 1, - document_id, - relative_path: sanitize_path(&request.relative_path), - content: latest_content, // copy the content from the latest version - updated_date: chrono::Utc::now(), - is_deleted: true, - user_id: user.name, - device_id: device_id.0, - has_been_merged: false, - }; - - state - .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) - .await - .map_err(server_error)?; - - Ok(Json(new_version.into())) -} diff --git a/sync-server/src/server/endpoints.rs b/sync-server/src/server/endpoints.rs new file mode 100644 index 00000000..23323a69 --- /dev/null +++ b/sync-server/src/server/endpoints.rs @@ -0,0 +1,29 @@ +pub mod events; +pub mod fetch_document_version_content; +pub mod fetch_latest_document_version; +pub mod get_file_manifest; +pub mod index; +pub mod method_not_allowed; +pub mod not_found; +pub mod ping; +pub mod put_file_content; +pub mod put_file_manifest; +mod utils; +pub mod vault_snapshot; +pub mod websocket; + +use serde::{Deserialize, Deserializer}; + +use crate::{app_state::database::models::VaultId, utils::normalize_vault_id::normalize_vault_id}; + +#[derive(Debug)] +pub struct VaultPath(pub VaultId); + +impl<'de> Deserialize<'de> for VaultPath { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + normalize_vault_id(deserializer).map(Self) + } +} diff --git a/sync-server/src/server/endpoints/events.rs b/sync-server/src/server/endpoints/events.rs new file mode 100644 index 00000000..2505afba --- /dev/null +++ b/sync-server/src/server/endpoints/events.rs @@ -0,0 +1,40 @@ +use axum::{ + Json, + extract::{Path, Query, State}, +}; +use log::debug; +use serde::Deserialize; + +use super::VaultPath; +use crate::{ + app_state::{AppState, database::models::EventBatch}, + errors::{SyncServerError, client_error, server_error}, +}; + +#[derive(Deserialize)] +pub struct EventsQuery { + pub after: i64, +} + +#[axum::debug_handler] +pub async fn events( + Path(VaultPath(vault_id)): Path, + Query(query): Query, + State(state): State, +) -> Result, SyncServerError> { + debug!( + "Fetching events after `{}` for vault `{}`", + query.after, vault_id + ); + + if query.after < 0 { + return Err(client_error(anyhow::anyhow!("Invalid event cursor"))); + } + + state + .database + .events_after(&vault_id, query.after) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/fetch_document_version_content.rs b/sync-server/src/server/endpoints/fetch_document_version_content.rs similarity index 69% rename from sync-server/src/server/fetch_document_version_content.rs rename to sync-server/src/server/endpoints/fetch_document_version_content.rs index 9fdd0ad8..9e981c5f 100644 --- a/sync-server/src/server/fetch_document_version_content.rs +++ b/sync-server/src/server/endpoints/fetch_document_version_content.rs @@ -4,33 +4,23 @@ use axum::{ extract::{Path, State}, }; use log::debug; -use serde::Deserialize; +use super::VaultPath; use crate::{ app_state::{ AppState, - database::models::{DocumentId, VaultId, VaultUpdateId}, + database::models::{DocumentId, VaultUpdateId}, }, errors::{SyncServerError, not_found_error, server_error}, - utils::normalize::normalize, }; -#[derive(Deserialize)] -pub struct FetchDocumentVersionContentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, - vault_update_id: VaultUpdateId, -} - #[axum::debug_handler] pub async fn fetch_document_version_content( - Path(FetchDocumentVersionContentPathParams { - vault_id, - document_id, - vault_update_id, - }): Path, + Path((VaultPath(vault_id), document_id, vault_update_id)): Path<( + VaultPath, + DocumentId, + VaultUpdateId, + )>, State(state): State, ) -> Result { debug!( diff --git a/sync-server/src/server/endpoints/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs new file mode 100644 index 00000000..3a51f182 --- /dev/null +++ b/sync-server/src/server/endpoints/fetch_latest_document_version.rs @@ -0,0 +1,57 @@ +use anyhow::anyhow; +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; + +use super::VaultPath; +use crate::{ + app_state::{ + AppState, + database::models::{DocumentId, DocumentVersion, DocumentVersionWithoutContent}, + }, + errors::{SyncServerError, not_found_error, server_error}, +}; + +#[axum::debug_handler] +pub async fn fetch_latest_document_version( + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, + State(state): State, +) -> Result, SyncServerError> { + debug!("Fetching latest document version for document `{document_id}` in vault `{vault_id}`"); + + let latest_version = state + .database + .get_latest_document_version(&vault_id, &document_id, None) + .await + .map_err(server_error)? + .map_or_else( + || { + Err(not_found_error(anyhow!( + "Document with id `{document_id}` not found", + ))) + }, + Ok, + )?; + + Ok(Json(latest_version.into())) +} + +pub async fn fetch_latest_document_metadata( + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, + State(state): State, +) -> Result, SyncServerError> { + let mut tx = state + .database + .create_readonly_transaction(&vault_id) + .await + .map_err(server_error)?; + let metadata = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT vault_update_id, document_id, updated_date, user_id, device_id, length(content) AS content_size + FROM documents WHERE document_id = ? ORDER BY vault_update_id DESC LIMIT 1" + ).bind(document_id.hyphenated()).fetch_optional(&mut *tx).await + .map_err(|error| server_error(error.into()))? + .ok_or_else(|| not_found_error(anyhow!("Document does not exist")))?; + Ok(Json(metadata)) +} diff --git a/sync-server/src/server/endpoints/get_file_manifest.rs b/sync-server/src/server/endpoints/get_file_manifest.rs new file mode 100644 index 00000000..3f706696 --- /dev/null +++ b/sync-server/src/server/endpoints/get_file_manifest.rs @@ -0,0 +1,33 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; + +use super::VaultPath; +use crate::{ + app_state::{ + AppState, + database::{Database, models::FileManifest}, + }, + errors::{SyncServerError, server_error}, +}; + +#[axum::debug_handler] +pub async fn get_file_manifest( + Path(VaultPath(vault_id)): Path, + State(state): State, +) -> Result, SyncServerError> { + debug!("Fetching file manifest for vault `{vault_id}`"); + + let mut transaction = state + .database + .create_readonly_transaction(&vault_id) + .await + .map_err(server_error)?; + + Database::current_file_manifest(&mut transaction) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/index.rs b/sync-server/src/server/endpoints/index.rs similarity index 61% rename from sync-server/src/server/index.rs rename to sync-server/src/server/endpoints/index.rs index 64b053f7..f6ea022d 100644 --- a/sync-server/src/server/index.rs +++ b/sync-server/src/server/endpoints/index.rs @@ -1,7 +1,8 @@ use axum::response::{Html, IntoResponse}; +#[axum::debug_handler] pub async fn index() -> impl IntoResponse { - const HTML_CONTENT: &str = include_str!("./assets/index.html"); + const HTML_CONTENT: &str = include_str!("../assets/index.html"); let html_content = HTML_CONTENT; Html(html_content) } diff --git a/sync-server/src/server/endpoints/method_not_allowed.rs b/sync-server/src/server/endpoints/method_not_allowed.rs new file mode 100644 index 00000000..65d9c8c9 --- /dev/null +++ b/sync-server/src/server/endpoints/method_not_allowed.rs @@ -0,0 +1,9 @@ +use anyhow::anyhow; +use axum::response::IntoResponse; + +use crate::errors::client_error; + +#[axum::debug_handler] +pub async fn method_not_allowed() -> impl IntoResponse { + client_error(anyhow!("Method not allowed")) +} diff --git a/sync-server/src/server/endpoints/not_found.rs b/sync-server/src/server/endpoints/not_found.rs new file mode 100644 index 00000000..7c760fa9 --- /dev/null +++ b/sync-server/src/server/endpoints/not_found.rs @@ -0,0 +1,9 @@ +use anyhow::anyhow; +use axum::response::IntoResponse; + +use crate::errors::not_found_error; + +#[axum::debug_handler] +pub async fn not_found() -> impl IntoResponse { + not_found_error(anyhow!("Page not found")) +} diff --git a/sync-server/src/server/ping.rs b/sync-server/src/server/endpoints/ping.rs similarity index 65% rename from sync-server/src/server/ping.rs rename to sync-server/src/server/endpoints/ping.rs index 31aa8acd..412ed0d2 100644 --- a/sync-server/src/server/ping.rs +++ b/sync-server/src/server/endpoints/ping.rs @@ -7,26 +7,15 @@ use axum_extra::{ headers::{Authorization, authorization::Bearer}, }; use log::debug; -use serde::Deserialize; -use super::{auth::auth, responses::PingResponse}; -use crate::{ - app_state::{AppState, database::models::VaultId}, - consts::SUPPORTED_API_VERSION, - errors::SyncServerError, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct PingPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} +use super::VaultPath; +use crate::server::{auth::auth, responses::PingResponse}; +use crate::{app_state::AppState, consts::SUPPORTED_API_VERSION, errors::SyncServerError}; #[axum::debug_handler] pub async fn ping( maybe_auth_header: Option>>, - Path(PingPathParams { vault_id }): Path, + Path(VaultPath(vault_id)): Path, State(state): State, ) -> Result, SyncServerError> { debug!("Pinging vault `{vault_id}`"); diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs new file mode 100644 index 00000000..2796e71c --- /dev/null +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -0,0 +1,679 @@ +use super::{VaultPath, utils::find_already_processed_event}; +use crate::{ + app_state::{ + AppState, + database::{ + Database, + models::{DocumentId, StoredDocumentVersion, VaultEvent}, + }, + }, + config::user_config::User, + errors::{SyncServerError, client_error, not_found_error, server_error}, + server::{ + device_id_header::DeviceIdHeader, + requests::{PushContent, PutFileContent}, + responses::DocumentUpdateResponse, + }, +}; +use anyhow::anyhow; +use axum::{ + Extension, Json, + extract::{Path, State}, +}; +use axum_extra::TypedHeader; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use log::debug; +use reconcile_text::NumberOrText; +use sha2::{Digest, Sha256}; + +#[axum::debug_handler] +pub async fn put_file_content( + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, + Extension(user): Extension, + TypedHeader(device): TypedHeader, + State(state): State, + Json(push): Json, +) -> Result, SyncServerError> { + debug!("Pushing document `{document_id}` in vault `{vault_id}`"); + + let fingerprint = Sha256::digest( + serde_json::to_vec(&("content", document_id, &push)) + .map_err(|error| server_error(error.into()))?, + ); + + // Build bytes against an immutable parent off the async executor and before + // taking the writer lock. The transaction below rechecks CAS and receipts. + let initial_parent = state + .database + .get_latest_document_version(&vault_id, &document_id, None) + .await + .map_err(server_error)?; + let prepared = if initial_parent.as_ref().map(|p| p.vault_update_id) == push.parent_version_id { + let limit = state + .config + .server + .max_body_size_mb + .saturating_mul(1024 * 1024); + Some( + tokio::task::spawn_blocking(move || { + decode_content(initial_parent, push.content, limit) + }) + .await + .map_err(|error| server_error(error.into()))?, + ) + } else { + None + }; + + let mut tx = state + .database + .create_write_transaction(&vault_id) + .await + .map_err(server_error)?; + + if let Some(event) = + find_already_processed_event(&mut tx, push.request_id, &fingerprint).await? + { + return match event { + VaultEvent::Content { document } => { + Ok(Json(DocumentUpdateResponse::Accepted(document))) + } + VaultEvent::FileManifest { .. } => Err(server_error(anyhow!( + "Stored event does not match content request" + ))), + }; + } + + let parent = state + .database + .get_latest_document_version(&vault_id, &document_id, Some(&mut tx)) + .await + .map_err(server_error)?; + + if let Some(parent) = &parent { + if Some(parent.vault_update_id) != push.parent_version_id { + return Ok(Json(DocumentUpdateResponse::StaleBase( + parent.clone().into(), + ))); + } + } else if push.parent_version_id.is_some() { + return Err(not_found_error(anyhow!( + "Document {document_id} does not exist" + ))); + } + + let content = prepared + .ok_or_else(|| { + server_error(anyhow!( + "Document changed while preparing content; retry the request" + )) + })? + .map_err(client_error)?; + + let version = StoredDocumentVersion { + vault_update_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) + .await + .map_err(server_error)?, + document_id, + content, + updated_date: chrono::Utc::now(), + user_id: user.name, + device_id: device.0, + }; + + Database::insert_document_version(&mut tx, &version) + .await + .map_err(server_error)?; + + let response = DocumentUpdateResponse::Accepted((&version).into()); + + tx.commit() + .await + .map_err(|error| server_error(error.into()))?; + + state.broadcasts.notify_about_vault_update(vault_id).await; + + Ok(Json(response)) +} + +const MAX_DIFF_ITEMS: usize = 10_000; + +fn decode_content( + parent: Option, + content: PushContent, + limit: usize, +) -> anyhow::Result> { + let bytes = match content { + PushContent::Snapshot(encoded) => STANDARD.decode(encoded)?, + PushContent::Diff(diff) => { + anyhow::ensure!( + diff.len() <= MAX_DIFF_ITEMS, + "Diff exceeds operation budget; send a snapshot" + ); + let parent = parent.ok_or_else(|| anyhow!("Diff requires a parent"))?; + let text = str::from_utf8(&parent.content)?; + apply_diff(text, diff, limit)? + } + }; + anyhow::ensure!(bytes.len() <= limit, "Content exceeds size limit"); + Ok(bytes) +} + +/// Each Unicode scalar in the parent is visited at most once. No tokenization +/// or repeated scans of a prefix are needed to apply the wire diff. +fn apply_diff(text: &str, diff: Vec, limit: usize) -> anyhow::Result> { + let mut source = text.chars(); + let mut result = String::new(); + for item in diff { + match item { + NumberOrText::Text(insert) => { + anyhow::ensure!( + insert.len() <= limit.saturating_sub(result.len()), + "Content exceeds size limit" + ); + result.push_str(&insert); + } + NumberOrText::Number(length) => { + for _ in 0..length.unsigned_abs() { + let ch = source + .next() + .ok_or_else(|| anyhow!("Diff exceeds parent length"))?; + if length > 0 { + anyhow::ensure!( + ch.len_utf8() <= limit.saturating_sub(result.len()), + "Content exceeds size limit" + ); + result.push(ch); + } + } + } + } + } + Ok(result.into_bytes()) +} + +#[cfg(test)] +mod tests { + use reconcile_text::EditedText; + use std::path::PathBuf; + + use super::*; + use crate::{ + app_state::database::{ + Database, + models::{ + DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId, + }, + }, + config::{ + Config, database_config::DatabaseConfig, server_config::ServerConfig, + user_config::VaultAccess, + }, + }; + + async fn send_document( + state: &AppState, + vault: &VaultId, + document_id: DocumentId, + user_id: String, + device_id: String, + push: PutFileContent, + ) -> Result { + put_file_content( + Path((VaultPath(vault.clone()), document_id)), + Extension(User { + name: user_id, + token: String::new(), + vault_access: VaultAccess::default(), + }), + TypedHeader(DeviceIdHeader(device_id)), + State(state.clone()), + Json(push), + ) + .await + .map(|Json(response)| response) + } + + struct Push { + document_id: DocumentId, + request_id: uuid::Uuid, + parent_version_id: Option, + content: PushContent, + user_id: String, + device_id: String, + } + + impl Push { + async fn send( + self, + state: &AppState, + vault: &VaultId, + ) -> Result { + send_document( + state, + vault, + self.document_id, + self.user_id, + self.device_id, + PutFileContent { + request_id: self.request_id, + parent_version_id: self.parent_version_id, + content: self.content, + }, + ) + .await + } + } + + struct TestVault { + directory: PathBuf, + state: AppState, + vault: VaultId, + } + + impl TestVault { + async fn new() -> Self { + let directory = + std::env::temp_dir().join(format!("vaultlink-cas-{}", uuid::Uuid::new_v4())); + let state = Self::open(&directory).await; + Self { + directory, + state, + vault: "test".to_owned(), + } + } + + async fn open(directory: &std::path::Path) -> AppState { + AppState::try_new(Config { + database: DatabaseConfig { + databases_directory_path: directory.to_owned(), + ..Default::default() + }, + server: ServerConfig { + max_clients_per_vault: 16, + ..Default::default() + }, + ..Default::default() + }) + .await + .unwrap() + } + + async fn send(&self, push: Push) -> DocumentUpdateResponse { + push.send(&self.state, &self.vault).await.unwrap() + } + + async fn latest(&self, document: DocumentId) -> StoredDocumentVersion { + self.state + .database + .get_latest_document_version(&self.vault, &document, None) + .await + .unwrap() + .unwrap() + } + } + + impl Drop for TestVault { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.directory); + } + } + + fn snapshot(document_id: DocumentId, parent: Option, text: &str) -> Push { + Push { + document_id, + request_id: uuid::Uuid::new_v4(), + parent_version_id: parent, + content: PushContent::Snapshot(STANDARD.encode(text)), + user_id: "user".to_owned(), + device_id: "device".to_owned(), + } + } + + fn accepted(response: DocumentUpdateResponse) -> DocumentVersionWithoutContent { + match response { + DocumentUpdateResponse::Accepted(version) => version, + response @ DocumentUpdateResponse::StaleBase(_) => { + panic!("Expected Accepted, got {response:?}") + } + } + } + + #[tokio::test] + async fn fragmented_diff_budget_rejects_without_committing_an_event() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted( + vault + .send(snapshot(document, None, &"a".repeat(1024 * 1024))) + .await, + ); + let mut push = snapshot(document, Some(base.vault_update_id), ""); + let mut fragments = vec![NumberOrText::Number(0); 10_001]; + fragments.push(NumberOrText::Number(1024 * 1024)); + push.content = PushContent::Diff(fragments); + assert!(matches!( + push.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(document).await.vault_update_id, + base.vault_update_id + ); + } + + #[tokio::test] + async fn json_snapshot_preserves_bytes_and_rejects_invalid_base64_without_a_version() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let content = [0, 255, 13, 10, 42]; + let request: PutFileContent = serde_json::from_value(serde_json::json!({ + "requestId": uuid::Uuid::new_v4(), + "parentVersionId": null, + "content": { "type": "Snapshot", "value": STANDARD.encode(content) }, + })) + .unwrap(); + let base = accepted( + send_document( + &vault.state, + &vault.vault, + document, + "user".to_owned(), + "device".to_owned(), + request, + ) + .await + .unwrap(), + ); + assert_eq!(vault.latest(document).await.content, content); + let mut tx = vault + .state + .database + .create_readonly_transaction(&vault.vault) + .await + .unwrap(); + let stored_document_id: String = + sqlx::query_scalar("SELECT document_id FROM documents WHERE vault_update_id = ?") + .bind(base.vault_update_id) + .fetch_one(&mut *tx) + .await + .unwrap(); + assert_eq!(stored_document_id, document.to_string()); + tx.commit().await.unwrap(); + assert!(serde_json::to_value(&base).unwrap().get("mtime").is_none()); + + let invalid: PutFileContent = serde_json::from_value(serde_json::json!({ + "requestId": uuid::Uuid::new_v4(), + "parentVersionId": base.vault_update_id, + "content": { "type": "Snapshot", "value": "not base64!" }, + })) + .unwrap(); + assert!(matches!( + send_document( + &vault.state, + &vault.vault, + document, + "user".to_owned(), + "device".to_owned(), + invalid, + ) + .await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(document).await.vault_update_id, + base.vault_update_id + ); + } + + #[tokio::test] + async fn concurrent_writers_only_accept_one_snapshot_and_preserve_the_base() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted(vault.send(snapshot(document, None, "base")).await); + let (left, right) = tokio::join!( + vault.send(snapshot(document, Some(base.vault_update_id), "left")), + vault.send(snapshot(document, Some(base.vault_update_id), "right")), + ); + let (winner, rejected) = match (left, right) { + ( + DocumentUpdateResponse::Accepted(winner), + DocumentUpdateResponse::StaleBase(rejected), + ) + | ( + DocumentUpdateResponse::StaleBase(rejected), + DocumentUpdateResponse::Accepted(winner), + ) => (winner, rejected), + outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"), + }; + let latest = vault.latest(document).await; + assert_eq!(winner.vault_update_id, rejected.vault_update_id); + assert_eq!(rejected.content_size, latest.content.len()); + assert!(latest.content == b"left" || latest.content == b"right"); + let original = vault + .state + .database + .get_document_version(&vault.vault, base.vault_update_id, None) + .await + .unwrap() + .unwrap(); + assert_eq!(original.content, b"base"); + let mut transaction = vault + .state + .database + .create_readonly_transaction(&vault.vault) + .await + .unwrap(); + assert_eq!( + Database::latest_event_id(&mut transaction).await.unwrap(), + 2 + ); + } + + #[tokio::test] + async fn acknowledgements_survive_reopening_and_return_the_original_version() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let create = snapshot(document, None, "created"); + let create_id = create.request_id; + let base = accepted(vault.send(create).await); + let update = snapshot(document, Some(base.vault_update_id), "updated"); + let update_id = update.request_id; + let updated = accepted(vault.send(update).await); + let latest = accepted( + vault + .send(snapshot(document, Some(updated.vault_update_id), "newest")) + .await, + ); + + let reopened = TestVault::open(&vault.directory).await; + for (request_id, parent, text, original) in [ + (create_id, None, "created", base), + (update_id, Some(1), "updated", updated), + ] { + let mut retry = snapshot(document, parent, text); + retry.request_id = request_id; + retry.device_id = "new-session-after-restart".to_owned(); + let acknowledged = accepted(retry.send(&reopened, &vault.vault).await.unwrap()); + assert_eq!( + serde_json::to_value(acknowledged).unwrap(), + serde_json::to_value(original).unwrap() + ); + } + assert_eq!( + vault.latest(document).await.vault_update_id, + latest.vault_update_id + ); + + let mut reused = snapshot(uuid::Uuid::new_v4(), None, "wrong document"); + reused.request_id = create_id; + assert!(matches!( + reused.send(&reopened, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + + let mut changed_operation = snapshot(document, None, "created"); + changed_operation.content = PushContent::Diff(vec![]); + for mut changed_payload in [ + snapshot(document, None, "different content"), + snapshot(document, Some(1), "created"), + changed_operation, + ] { + changed_payload.request_id = create_id; + assert!(matches!( + changed_payload.send(&reopened, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + } + let events = reopened + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.head_event_id, latest.vault_update_id); + assert_eq!(events.events.len(), 3); + assert_eq!(events.events[0].request_id, create_id); + assert_eq!(events.events[1].request_id, update_id); + } + + #[tokio::test] + async fn concurrent_retries_return_the_same_acknowledgement_and_append_one_event() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let original = snapshot(document, None, "created"); + let request_id = original.request_id; + let mut retry = snapshot(document, None, "created"); + retry.request_id = request_id; + retry.device_id = "reconnected-device".to_owned(); + + let (left, right) = tokio::join!(vault.send(original), vault.send(retry)); + assert_eq!( + serde_json::to_value(accepted(left)).unwrap(), + serde_json::to_value(accepted(right)).unwrap() + ); + let events = vault + .state + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.head_event_id, 1); + assert_eq!(events.events.len(), 1); + assert_eq!(events.events[0].request_id, request_id); + } + + #[tokio::test] + async fn failed_normalized_event_write_rolls_back_and_allows_retrying_the_request() { + let vault = TestVault::new().await; + let mut tx = vault + .state + .database + .create_write_transaction(&vault.vault) + .await + .unwrap(); + sqlx::query("CREATE TRIGGER reject_event BEFORE INSERT ON documents BEGIN SELECT RAISE(ABORT, 'simulated event write failure'); END") + .execute(&mut *tx).await.unwrap(); + tx.commit().await.unwrap(); + + let document = uuid::Uuid::new_v4(); + let request = snapshot(document, None, "created"); + let request_id = request.request_id; + assert!(request.send(&vault.state, &vault.vault).await.is_err()); + assert!( + vault + .state + .database + .get_latest_document_version(&vault.vault, &document, None) + .await + .unwrap() + .is_none() + ); + let events = vault + .state + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.head_event_id, 0); + assert!(events.events.is_empty()); + + let mut tx = vault + .state + .database + .create_write_transaction(&vault.vault) + .await + .unwrap(); + sqlx::query("DROP TRIGGER reject_event") + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + let mut retry = snapshot(document, None, "created"); + retry.request_id = request_id; + let version = accepted(vault.send(retry).await); + assert_eq!(version.vault_update_id, 1); + let events = vault + .state + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.events.len(), 1); + assert_eq!(events.events[0].request_id, request_id); + } + + #[tokio::test] + async fn transport_diffs_reconstruct_exact_snapshots_and_reject_invalid_bases() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let original = "Hello world 🌍"; + let desired = "Hi beautiful world 🌍"; + let base = accepted(vault.send(snapshot(document, None, original)).await); + let mut update = snapshot(document, Some(base.vault_update_id), ""); + update.content = + PushContent::Diff(EditedText::from_strings(original, &desired.into()).to_diff()); + let updated = accepted(vault.send(update).await); + assert_eq!(vault.latest(document).await.content, desired.as_bytes()); + + for invalid_length in [i64::MIN, i64::MAX] { + let mut malformed = snapshot(document, Some(updated.vault_update_id), ""); + malformed.content = PushContent::Diff(vec![NumberOrText::Number(invalid_length)]); + assert!(matches!( + malformed.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + } + assert_eq!( + vault.latest(document).await.vault_update_id, + updated.vault_update_id + ); + + let another = accepted( + vault + .send(snapshot(uuid::Uuid::new_v4(), None, "another document")) + .await, + ); + let mut foreign_base = snapshot(document, Some(another.vault_update_id), ""); + foreign_base.content = PushContent::Diff(vec![]); + assert!(matches!( + vault.send(foreign_base).await, + DocumentUpdateResponse::StaleBase(version) + if version.vault_update_id == updated.vault_update_id + )); + + let binary_document = uuid::Uuid::new_v4(); + let mut binary = snapshot(binary_document, None, ""); + binary.content = PushContent::Snapshot(STANDARD.encode([0xff])); + let binary_base = accepted(vault.send(binary).await); + let mut invalid_text = snapshot(binary_document, Some(binary_base.vault_update_id), ""); + invalid_text.content = PushContent::Diff(vec![]); + assert!(matches!( + invalid_text.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(binary_document).await.vault_update_id, + binary_base.vault_update_id + ); + } +} diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs new file mode 100644 index 00000000..05a26e11 --- /dev/null +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -0,0 +1,97 @@ +use anyhow::anyhow; +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use sha2::{Digest, Sha256}; + +use super::{VaultPath, utils::find_already_processed_event}; +use crate::{ + app_state::{ + AppState, + database::{ + Database, + models::{FileManifest, VaultEvent}, + }, + }, + errors::{SyncServerError, client_error, server_error}, + server::{requests::PushFileManifest, responses::FileManifestUpdateResponse}, +}; + +#[axum::debug_handler] +pub async fn put_file_manifest( + Path(VaultPath(vault_id)): Path, + State(state): State, + Json(push): Json, +) -> Result, SyncServerError> { + debug!("Pushing file manifest for vault `{vault_id}`"); + + // BTreeMap canonicalizes file manifest entries for request fingerprinting. + let fingerprint = Sha256::digest( + serde_json::to_vec(&("file_manifest", &push)) + .map_err(|error| server_error(error.into()))?, + ); + + let mut tx = state + .database + .create_write_transaction(&vault_id) + .await + .map_err(server_error)?; + + if let Some(event) = + find_already_processed_event(&mut tx, push.request_id, &fingerprint).await? + { + return match event { + VaultEvent::FileManifest { file_manifest } => { + Ok(Json(FileManifestUpdateResponse::Accepted { + file_manifest_id: file_manifest.file_manifest_id, + })) + } + VaultEvent::Content { .. } => Err(client_error(anyhow!( + "Stored event does not match file manifest request" + ))), + }; + } + + let latest = Database::current_file_manifest(&mut tx) + .await + .map_err(server_error)?; + + if latest.file_manifest_id != push.parent_file_manifest_id { + return Ok(Json(FileManifestUpdateResponse::StaleBase(latest))); + } + + if let Some(id) = + Database::missing_document(&mut tx, &push.entries.keys().copied().collect::>()) + .await + .map_err(server_error)? + { + return Err(client_error(anyhow!( + "File manifest references missing content: {id}" + ))); + } + + let file_manifest = FileManifest { + file_manifest_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) + .await + .map_err(server_error)?, + entries: push.entries, + }; + + Database::insert_file_manifest(&mut tx, &file_manifest) + .await + .map_err(server_error)?; + + let response = FileManifestUpdateResponse::Accepted { + file_manifest_id: file_manifest.file_manifest_id, + }; + + tx.commit() + .await + .map_err(|error| server_error(error.into()))?; + + state.broadcasts.notify_about_vault_update(vault_id).await; + + Ok(Json(response)) +} diff --git a/sync-server/src/server/endpoints/utils.rs b/sync-server/src/server/endpoints/utils.rs new file mode 100644 index 00000000..c0df4216 --- /dev/null +++ b/sync-server/src/server/endpoints/utils.rs @@ -0,0 +1,27 @@ +use anyhow::anyhow; + +use crate::{ + app_state::database::{Database, Transaction, models::VaultEvent}, + errors::{SyncServerError, client_error, server_error}, +}; + +pub(super) async fn find_already_processed_event( + tx: &mut Transaction<'_>, + request_id: uuid::Uuid, + fingerprint: &[u8], +) -> Result, SyncServerError> { + let Some((stored_fingerprint, event)) = Database::get_request_event(tx, request_id) + .await + .map_err(server_error)? + else { + return Ok(None); + }; + + if stored_fingerprint != fingerprint { + return Err(client_error(anyhow!( + "Request ID was reused with a different payload" + ))); + } + + Ok(Some(event)) +} diff --git a/sync-server/src/server/endpoints/vault_snapshot.rs b/sync-server/src/server/endpoints/vault_snapshot.rs new file mode 100644 index 00000000..93e5e4d1 --- /dev/null +++ b/sync-server/src/server/endpoints/vault_snapshot.rs @@ -0,0 +1,26 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; + +use super::VaultPath; +use crate::{ + app_state::{AppState, database::models::VaultSnapshot}, + errors::{SyncServerError, server_error}, +}; + +#[axum::debug_handler] +pub async fn vault_snapshot( + Path(VaultPath(vault_id)): Path, + State(state): State, +) -> Result, SyncServerError> { + debug!("Fetching vault snapshot for vault `{vault_id}`"); + + state + .database + .vault_snapshot(&vault_id) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs new file mode 100644 index 00000000..a4b779de --- /dev/null +++ b/sync-server/src/server/endpoints/websocket.rs @@ -0,0 +1,128 @@ +use crate::{ + app_state::{ + AppState, + database::models::VaultId, + websocket::{ + broadcasts::Notification, + models::{CursorPositionFromServer, WebSocketClientMessage, WebSocketServerMessage}, + utils::{get_authenticated_handshake, send_update_over_websocket}, + }, + }, + errors::{SyncServerError, client_error, server_error}, +}; +use axum::{ + extract::{ + Path, State, + ws::{Message, WebSocket, WebSocketUpgrade}, + }, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use futures::StreamExt; +use log::{debug, info}; +use std::time::Duration; +use tokio::sync::broadcast::error::RecvError; + +use super::VaultPath; + +#[axum::debug_handler] +pub async fn websocket_handler( + ws: WebSocketUpgrade, + Path(VaultPath(vault_id)): Path, + State(state): State, +) -> Result { + debug!("Upgrading WebSocket connection for vault `{vault_id}`"); + + let Some(permit) = state.broadcasts.try_admit(&vault_id) else { + return Ok(( + StatusCode::TOO_MANY_REQUESTS, + "Vault connection limit reached", + ) + .into_response()); + }; + Ok(ws + .max_message_size(1024 * 1024) + .max_frame_size(1024 * 1024) + .on_upgrade(move |socket| async move { + let _permit = permit; + if let Err(error) = websocket(state, socket, vault_id).await { + debug!("WebSocket disconnected: {error}"); + } + })) +} + +async fn websocket( + state: AppState, + socket: WebSocket, + vault: VaultId, +) -> Result<(), SyncServerError> { + let (mut sender, mut receiver) = socket.split(); + let authenticated = get_authenticated_handshake( + &state, + &vault, + tokio::time::timeout(Duration::from_secs(5), receiver.next()) + .await + .map_err(|_| client_error(anyhow::anyhow!("WebSocket handshake deadline expired")))? + .transpose() + .unwrap_or_default(), + )?; + + let device = authenticated.handshake.device_id; + let session = state.cursors.register_connection(&vault, &device).await; + let mut last_checkpoint = None; + let mut notifications = state.broadcasts.get_receiver(vault.clone()).await; + let mut timer = tokio::time::interval(Duration::from_secs(2)); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + info!("WebSocket connected to vault {vault}"); + + // HTTP owns event replay. Poll the cheap checkpoint to recover missed + // broadcasts, including a commit followed by an interrupted notification. + let result = async { + loop { + let checkpoint = state.database.history_checkpoint(&vault).await.map_err(server_error)?; + if last_checkpoint.as_ref() != Some(&checkpoint) { + send_update_over_websocket(&WebSocketServerMessage::VaultChanged, &mut sender).await?; + last_checkpoint = Some(checkpoint); + } + + tokio::select! { + _ = timer.tick() => {}, + notification = notifications.recv() => match notification { + Ok(update) => if let Notification::Cursors(mut positions) = update { + positions.clients.retain(|c| c.device_id != device); + send_update_over_websocket(&WebSocketServerMessage::CursorPositions(positions), &mut sender).await?; + }, + + Err(RecvError::Lagged(_)) => {}, // Check the durable head on the next iteration. + Err(RecvError::Closed) => break, + }, + + incoming = receiver.next() => match incoming { + Some(Ok(Message::Text(text))) => { + let message: WebSocketClientMessage = serde_json::from_str(&text).map_err(|error| client_error(error.into()))?; + match message { + WebSocketClientMessage::Handshake(_) => return Err(client_error(anyhow::anyhow!("Unexpected handshake"))), + WebSocketClientMessage::CursorPositions(positions) => { + state.cursors.update_cursors(vault.clone(), authenticated.user.name.clone(), &device, session, positions.documents_with_cursors).await; + let clients = state.cursors.get_cursors(&vault).await.into_iter().filter(|c| c.device_id != device).collect(); + send_update_over_websocket(&WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }), &mut sender).await?; + } + } + }, + Some(Ok(Message::Ping(_) | Message::Pong(_))) => {}, + _ => break, + } + } + } + + Ok(()) + }.await; + + state + .cursors + .remove_cursors_of_device(&vault, &device, session) + .await; + + result +} diff --git a/sync-server/src/server/fetch_document_version.rs b/sync-server/src/server/fetch_document_version.rs deleted file mode 100644 index c30f1d76..00000000 --- a/sync-server/src/server/fetch_document_version.rs +++ /dev/null @@ -1,62 +0,0 @@ -use anyhow::anyhow; -use axum::{ - Json, - extract::{Path, State}, -}; -use log::debug; -use serde::Deserialize; - -use crate::{ - app_state::{ - AppState, - database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId}, - }, - errors::{SyncServerError, not_found_error, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct FetchDocumentVersionPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, - vault_update_id: VaultUpdateId, -} - -#[axum::debug_handler] -pub async fn fetch_document_version( - Path(FetchDocumentVersionPathParams { - vault_id, - document_id, - vault_update_id, - }): Path, - State(state): State, -) -> Result, SyncServerError> { - debug!( - "Fetching document version `{vault_update_id}` for document `{document_id}` in vault `{vault_id}`" - ); - - let result = state - .database - .get_document_version(&vault_id, vault_update_id, None) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Document with vault update id `{vault_update_id}` not found", - ))) - }, - Ok, - )?; - - if result.document_id != document_id { - return Err(not_found_error(anyhow!( - "Document with document id `{document_id}` does not have a version with id \ - `{vault_update_id}`", - ))); - } - - Ok(Json(result.into())) -} diff --git a/sync-server/src/server/fetch_latest_document_version.rs b/sync-server/src/server/fetch_latest_document_version.rs deleted file mode 100644 index a9973606..00000000 --- a/sync-server/src/server/fetch_latest_document_version.rs +++ /dev/null @@ -1,51 +0,0 @@ -use anyhow::anyhow; -use axum::{ - Json, - extract::{Path, State}, -}; -use log::debug; -use serde::Deserialize; - -use crate::{ - app_state::{ - AppState, - database::models::{DocumentId, DocumentVersion, VaultId}, - }, - errors::{SyncServerError, not_found_error, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct FetchLatestDocumentVersionPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, -} - -#[axum::debug_handler] -pub async fn fetch_latest_document_version( - Path(FetchLatestDocumentVersionPathParams { - vault_id, - document_id, - }): Path, - State(state): State, -) -> Result, SyncServerError> { - debug!("Fetching latest document version for document `{document_id}` in vault `{vault_id}`"); - - let latest_version = state - .database - .get_latest_document(&vault_id, &document_id, None) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Document with id `{document_id}` not found", - ))) - }, - Ok, - )?; - - Ok(Json(latest_version.into())) -} diff --git a/sync-server/src/server/fetch_latest_documents.rs b/sync-server/src/server/fetch_latest_documents.rs deleted file mode 100644 index 209374ce..00000000 --- a/sync-server/src/server/fetch_latest_documents.rs +++ /dev/null @@ -1,59 +0,0 @@ -use axum::{ - Json, - extract::{Path, Query, State}, -}; -use log::debug; -use serde::Deserialize; - -use super::responses::FetchLatestDocumentsResponse; -use crate::{ - app_state::{ - AppState, - database::models::{VaultId, VaultUpdateId}, - }, - errors::{SyncServerError, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct FetchLatestDocumentsPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} - -#[derive(Deserialize)] -pub struct QueryParams { - since_update_id: Option, -} - -#[axum::debug_handler] -pub async fn fetch_latest_documents( - Path(FetchLatestDocumentsPathParams { vault_id }): Path, - Query(QueryParams { since_update_id }): Query, - State(state): State, -) -> Result, SyncServerError> { - debug!("Fetching latest documents in vault `{vault_id}` since update ID `{since_update_id:?}`"); - - let documents = if let Some(since_update_id) = since_update_id { - state - .database - .get_latest_documents_since(&vault_id, since_update_id, None) - .await - .map_err(server_error) - } else { - state - .database - .get_latest_documents(&vault_id, None) - .await - .map_err(server_error) - }?; - - Ok(Json(FetchLatestDocumentsResponse { - last_update_id: documents - .iter() - .map(|doc| doc.vault_update_id) - .max() - .unwrap_or(since_update_id.unwrap_or(0)), - latest_documents: documents, - })) -} diff --git a/sync-server/src/server/requests.rs b/sync-server/src/server/requests.rs index 119ad467..a27e62a9 100644 --- a/sync-server/src/server/requests.rs +++ b/sync-server/src/server/requests.rs @@ -1,51 +1,149 @@ -use axum::body::Bytes; -use axum_typed_multipart::{FieldData, TryFromMultipart}; +use crate::{ + app_state::database::models::{DocumentId, FileManifestEntries, VaultUpdateId}, + utils::portable_path::validate_file_manifest, +}; use reconcile_text::NumberOrText; -use serde::{self, Deserialize}; +use serde::{ + Deserialize, Deserializer, Serialize, + de::{Error, MapAccess, Visitor}, +}; +use std::{collections::BTreeMap, fmt}; use ts_rs::TS; -use crate::app_state::database::models::{DocumentId, VaultUpdateId}; - -#[derive(TS, Debug, TryFromMultipart)] -#[ts(export)] -pub struct CreateDocumentVersion { - /// The client can decide the document id (if it wishes to) in order - /// to help with syncing. If the client does not provide a document id, - /// the server will generate one. If the client provides a document id - /// it must not already exist in the database. - pub document_id: Option, - pub relative_path: String, - - #[ts(as = "Vec")] - #[form_data(limit = "unlimited")] - pub content: FieldData, -} - -#[derive(Debug, TryFromMultipart)] -pub struct UpdateBinaryDocumentVersion { - pub parent_version_id: VaultUpdateId, - pub relative_path: String, - - #[form_data(limit = "unlimited")] - pub content: FieldData, -} - -#[derive(TS, Debug, Deserialize)] +#[derive(TS, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] #[ts(export)] -pub struct UpdateTextDocumentVersion { - #[ts(as = "i32")] - pub parent_version_id: VaultUpdateId, - - pub relative_path: String, - - #[ts(type = "Array")] - pub content: Vec, +pub struct PutFileContent { + pub request_id: uuid::Uuid, + #[ts(as = "Option")] + pub parent_version_id: Option, + pub content: PushContent, } -#[derive(TS, Debug, Deserialize)] +#[derive(TS, Debug, Deserialize, Serialize)] +#[serde(tag = "type", content = "value")] +#[serde(deny_unknown_fields)] +pub enum PushContent { + Snapshot(String), + Diff(#[ts(type = "Array")] Vec), +} + +#[derive(TS, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] #[ts(export)] -pub struct DeleteDocumentVersion { - pub relative_path: String, +pub struct PushFileManifest { + pub request_id: uuid::Uuid, + #[ts(as = "f64")] + pub parent_file_manifest_id: VaultUpdateId, + #[serde(deserialize_with = "deserialize_file_manifest_entries")] + #[ts(type = "Record")] + pub entries: FileManifestEntries, +} + +// A normal JSON map decoder silently overwrites duplicate keys. UUID parsing also +// catches differently spelled keys that denote the same identity. +fn deserialize_file_manifest_entries<'de, D: Deserializer<'de>>( + d: D, +) -> Result { + struct Entries; + + impl<'de> Visitor<'de> for Entries { + type Value = FileManifestEntries; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("unique document IDs mapped to paths") + } + + fn visit_map>(self, mut map: M) -> Result { + let mut entries = BTreeMap::new(); + + while let Some((id, path)) = map.next_entry::()? { + if entries.insert(id, path).is_some() { + return Err(M::Error::custom("Duplicate document ID")); + } + } + + Ok(entries) + } + } + + let entries = d.deserialize_map(Entries)?; + + validate_file_manifest(&entries).map_err(D::Error::custom)?; + + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::PushFileManifest; + + fn deserialize_manifest(entries: &str) -> serde_json::Result { + serde_json::from_str(&format!( + r#"{{ + "requestId": "00000000-0000-0000-0000-000000000001", + "parentFileManifestId": 0, + "entries": {entries} + }}"# + )) + } + + #[test] + fn deserializes_empty_and_valid_file_manifests() { + assert!(deserialize_manifest("{}").unwrap().entries.is_empty()); + + let push = deserialize_manifest( + r#"{ + "00000000-0000-0000-0000-000000000001": "Notes/Café.md", + "00000000-0000-0000-0000-000000000002": "Notes/日記.md" + }"#, + ) + .unwrap(); + + assert_eq!( + push.entries + .values() + .map(String::as_str) + .collect::>(), + ["Notes/Café.md", "Notes/日記.md"] + ); + } + + #[test] + fn deserialization_rejects_invalid_paths_and_manifest_conflicts() { + for (entries, expected_error) in [ + ( + r#"{"00000000-0000-0000-0000-000000000001": "../notes.md"}"#, + "Invalid path component", + ), + ( + r#"{ + "00000000-0000-0000-0000-000000000001": "Notes/a.md", + "00000000-0000-0000-0000-000000000002": "notes/b.md" + }"#, + "Conflicting path", + ), + ] { + let error = deserialize_manifest(entries).unwrap_err(); + + assert!(error.to_string().contains(expected_error), "{error}"); + } + } + + #[test] + fn deserialization_rejects_duplicate_document_ids_including_uuid_aliases() { + let id = "00000000-0000-0000-0000-000000000abc"; + + for duplicate in [id.to_owned(), id.to_uppercase()] { + let entries = format!(r#"{{"{id}": "a.md", "{duplicate}": "b.md"}}"#); + let error = deserialize_manifest(&entries).unwrap_err(); + + assert!( + error.to_string().contains("Duplicate document ID"), + "{error}" + ); + } + } } diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index a8b3fcd7..e8d85687 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -1,8 +1,8 @@ -use serde::{self, Serialize}; +use serde::{self, Deserialize, Serialize}; use ts_rs::TS; use crate::app_state::database::models::{ - DocumentVersion, DocumentVersionWithoutContent, VaultUpdateId, + DocumentVersionWithoutContent, FileManifest, VaultUpdateId, }; /// Response to a ping request. @@ -25,28 +25,26 @@ pub struct PingResponse { pub supported_api_version: u32, } -/// Response to a fetch latest documents request. -#[derive(TS, Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub struct FetchLatestDocumentsResponse { - pub latest_documents: Vec, - - /// The update ID of the latest document in the response. - pub last_update_id: VaultUpdateId, -} - /// Response to an update document request. -#[derive(TS, Debug, Clone, Serialize)] +#[derive(TS, Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] #[ts(export)] pub enum DocumentUpdateResponse { - /// Returned when the created/updated document's content is the same as was - /// sent in the create/update request and thus the response doesn't contain - /// the content because the client must already have it. - FastForwardUpdate(DocumentVersionWithoutContent), + /// The exact snapshot was committed, or this is its original acknowledgement. + Accepted(DocumentVersionWithoutContent), - /// Returned when the created/updated document's content is different from - /// what was sent in the create/update request. - MergingUpdate(DocumentVersion), + /// Nothing was written. The client must incorporate this version before retrying. + StaleBase(DocumentVersionWithoutContent), +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +#[ts(export)] +pub enum FileManifestUpdateResponse { + Accepted { + #[serde(rename = "fileManifestId")] + #[ts(as = "f64")] + file_manifest_id: VaultUpdateId, + }, + StaleBase(FileManifest), } diff --git a/sync-server/src/server/tests.rs b/sync-server/src/server/tests.rs new file mode 100644 index 00000000..01d5f842 --- /dev/null +++ b/sync-server/src/server/tests.rs @@ -0,0 +1,121 @@ +use super::*; +use axum::{ + body::{Body, to_bytes}, + http::StatusCode, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; + +async fn app(directory: &tempfile::TempDir) -> Router { + let mut config: Config = serde_json::from_value(json!({ + "users": { "user_configs": [ + { "name": "admin", "token": "admin", "vault_access": { "type": "allow_access_to_all" } }, + { "name": "mixed", "token": "mixed", "vault_access": { "type": "allow_list", "allowed": [" MyVault "] } }, + { "name": "composed", "token": "composed", "vault_access": { "type": "allow_list", "allowed": ["café"] } }, + { "name": "decomposed", "token": "decomposed", "vault_access": { "type": "allow_list", "allowed": ["cafe\u{301}"] } } + ] } + })).unwrap(); + config.database.databases_directory_path = directory.path().join("databases"); + let state = AppState::try_new(config).await.unwrap(); + get_authed_routes(state.clone()).with_state(state) +} + +async fn request(app: Router, path: &str, token: &str, body: Option) -> (StatusCode, Value) { + let response = app + .oneshot( + Request::builder() + .uri(path) + .method(if body.is_some() { + Method::PUT + } else { + Method::GET + }) + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .header(http::header::CONTENT_TYPE, "application/json") + .header("Device-Id", "test") + .body(body.map_or_else(Body::empty, |value| Body::from(value.to_string()))) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) +} + +#[tokio::test] +async fn http_rejects_encoded_traversal_and_accepts_normalized_allowlists() { + let directory = tempfile::tempdir().unwrap(); + let app = app(&directory).await; + for vault in [ + "..%2Fescaped", + "%2Fabsolute", + "a%5Cb", + "%2E%2E", + "%20", + "a%00b", + ] { + let (status, _) = request( + app.clone(), + &format!("/vaults/{vault}/vault-snapshot"), + "admin", + None, + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "vault: {vault}"); + } + assert!(!directory.path().join("escaped.sqlite").exists()); + for vault in ["MyVault", "myvault", "%20MYVAULT%20"] { + let (status, _) = request( + app.clone(), + &format!("/vaults/{vault}/file-manifest"), + "mixed", + None, + ) + .await; + assert_eq!(status, StatusCode::OK, "vault: {vault}"); + } + assert_eq!( + request(app, "/vaults/other/file-manifest", "mixed", None) + .await + .0, + StatusCode::FORBIDDEN + ); +} + +#[tokio::test] +async fn http_keeps_unicode_vaults_separate_across_restart() { + let directory = tempfile::tempdir().unwrap(); + let router = app(&directory).await; + let document = uuid::Uuid::new_v4(); + let composed = format!("/vaults/caf%C3%A9/documents/{document}"); + let decomposed = format!("/vaults/cafe%CC%81/documents/{document}"); + let push = json!({ + "requestId": uuid::Uuid::new_v4(), "parentVersionId": null, + "content": { "type": "Snapshot", "value": "aGVsbG8=" } + }); + assert_eq!( + request(router, &composed, "composed", Some(push)).await.0, + StatusCode::OK + ); + + let reopened = app(&directory).await; + assert_eq!( + request(reopened.clone(), &composed, "decomposed", None) + .await + .0, + StatusCode::FORBIDDEN + ); + assert_eq!( + request(reopened.clone(), &decomposed, "decomposed", None) + .await + .0, + StatusCode::NOT_FOUND + ); + let (status, version) = request(reopened, &composed, "composed", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(version["contentBase64"], "aGVsbG8="); +} diff --git a/sync-server/src/server/update_document.rs b/sync-server/src/server/update_document.rs deleted file mode 100644 index 00fbd008..00000000 --- a/sync-server/src/server/update_document.rs +++ /dev/null @@ -1,271 +0,0 @@ -use anyhow::{Context as _, anyhow}; -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use axum_typed_multipart::TypedMultipart; -use log::{debug, info}; -use reconcile_text::{BuiltinTokenizer, EditedText, reconcile}; -use serde::Deserialize; - -use super::{ - device_id_header::DeviceIdHeader, requests::UpdateTextDocumentVersion, - responses::DocumentUpdateResponse, -}; -use crate::{ - app_state::{ - AppState, - database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId}, - }, - config::user_config::User, - errors::{SyncServerError, client_error, not_found_error, server_error}, - server::requests::UpdateBinaryDocumentVersion, - utils::{ - find_first_available_path::find_first_available_path, is_binary::is_binary, - is_file_type_mergable::is_file_type_mergable, normalize::normalize, - sanitize_path::sanitize_path, - }, -}; - -#[derive(Deserialize)] -pub struct UpdateDocumentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - document_id: DocumentId, -} - -#[axum::debug_handler] -pub async fn update_binary( - Path(UpdateDocumentPathParams { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - TypedMultipart(request): TypedMultipart, -) -> Result, SyncServerError> { - let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?; - let content = request.content.contents.to_vec(); - - update_document( - parent_document, - vault_id, - document_id, - user, - device_id, - state, - &request.relative_path, - content, - ) - .await -} - -#[axum::debug_handler] -#[allow(clippy::too_many_lines)] -pub async fn update_text( - Path(UpdateDocumentPathParams { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - Json(request): Json, -) -> Result, SyncServerError> { - let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?; - - let edited_text = EditedText::from_diff( - str::from_utf8(&parent_document.content) - .expect("parent must be valid UTF-8 because it's a text document"), - request.content, - &*BuiltinTokenizer::Word, - ) - .context("Failed to apply given diff to parent document") - .map_err(client_error)?; - - let content = edited_text.apply().text().into_bytes(); - - update_document( - parent_document, - vault_id, - document_id, - user, - device_id, - state, - &request.relative_path, - content, - ) - .await -} - -async fn get_parent_document( - state: &AppState, - vault_id: &VaultId, - parent_version_id: VaultUpdateId, -) -> Result { - state - .database - .get_document_version(vault_id, parent_version_id, None) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Parent version with id `{parent_version_id}` not found" - ))) - }, - Ok, - ) -} - -#[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn update_document( - parent_document: StoredDocumentVersion, - vault_id: VaultId, - document_id: DocumentId, - user: User, - device_id: DeviceIdHeader, - state: AppState, - relative_path: &str, - content: Vec, -) -> Result, SyncServerError> { - debug!("Updating document `{document_id}` in vault `{vault_id}`"); - - let sanitized_relative_path = sanitize_path(relative_path); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; - - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - let latest_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Document with id `{document_id}` not found", - ))) - }, - Ok, - )?; - - if latest_version.is_deleted { - transaction - .rollback() - .await - .context("Failed to roll back transaction") - .map_err(server_error)?; - - info!("Document `{document_id}` has been deleted, ignoring update to it",); - return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( - latest_version.into(), - ))); - } - - // Return the latest version if the content and path are the same as the latest - // version - if content == latest_version.content && sanitized_relative_path == latest_version.relative_path - { - info!( - "Document content is the same as the latest version for `{document_id}`, skipping update" - ); - transaction - .rollback() - .await - .context("Failed to roll back transaction") - .map_err(server_error)?; - - return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( - latest_version.into(), - ))); - } - - let are_all_participants_mergable = is_file_type_mergable( - &sanitized_relative_path, - &state.config.server.mergeable_file_extensions, - ) && !is_binary(&parent_document.content) - && !is_binary(&latest_version.content) - && !is_binary(&content); - - let merged_content = if are_all_participants_mergable { - info!("Merging changes for document `{document_id}` in vault `{vault_id}`"); - reconcile( - str::from_utf8(&parent_document.content) - .expect("parent must be valid UTF-8 because it's not binary"), - &str::from_utf8(&latest_version.content) - .expect("latest_version must be valid UTF-8 because it's not binary") - .into(), - &str::from_utf8(&content) - .expect("content must be valid UTF-8 because it's not binary") - .into(), - &*BuiltinTokenizer::Word, - ) - .apply() - .text() - .into_bytes() - } else { - content.clone() - }; - - let is_different_from_request_content = merged_content != content; - - // We can only update the relative path if we're the first one to do so - let new_relative_path = if parent_document.relative_path == latest_version.relative_path - && latest_version.relative_path != sanitized_relative_path - { - let new_path = find_first_available_path( - &vault_id, - &sanitized_relative_path, - &state.database, - &mut transaction, - ) - .await - .map_err(server_error)?; - - if new_path != sanitized_relative_path { - info!( - "Document already exists at new location: `{sanitized_relative_path}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`" - ); - } - - new_path - } else { - latest_version.relative_path.clone() - }; - - let new_version = StoredDocumentVersion { - document_id, - vault_update_id: last_update_id + 1, - relative_path: new_relative_path, - content: merged_content, - updated_date: chrono::Utc::now(), - is_deleted: false, - user_id: user.name, - device_id: device_id.0, - has_been_merged: are_all_participants_mergable && is_different_from_request_content, - }; - - state - .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) - .await - .map_err(server_error)?; - - Ok(Json(if is_different_from_request_content { - DocumentUpdateResponse::MergingUpdate(new_version.into()) - } else { - DocumentUpdateResponse::FastForwardUpdate(new_version.into()) - })) -} diff --git a/sync-server/src/server/websocket.rs b/sync-server/src/server/websocket.rs deleted file mode 100644 index bb10b49f..00000000 --- a/sync-server/src/server/websocket.rs +++ /dev/null @@ -1,193 +0,0 @@ -use anyhow::Context; -use axum::{ - extract::{ - Path, State, - ws::{Message, WebSocket, WebSocketUpgrade}, - }, - response::Response, -}; -use futures::stream::StreamExt; -use log::{debug, info}; -use serde::Deserialize; - -use crate::{ - app_state::{ - AppState, - database::models::VaultId, - websocket::{ - models::{ - CursorPositionFromServer, WebSocketClientMessage, WebSocketServerMessage, - WebSocketVaultUpdate, - }, - utils::{ - get_authenticated_handshake, get_unseen_documents, send_update_over_websocket, - }, - }, - }, - errors::{SyncServerError, client_error, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct WebSocketPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} - -pub async fn websocket_handler( - ws: WebSocketUpgrade, - Path(WebSocketPathParams { vault_id }): Path, - State(state): State, -) -> Result { - Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id))) -} - -async fn websocket_wrapped(state: AppState, stream: WebSocket, vault_id: VaultId) { - info!("WebSocket connection opened on vault `{vault_id}`"); - - let result = websocket(state, stream, vault_id.clone()).await; - - if let Err(err) = result { - debug!("WebSocket connection error on vault `{vault_id}`: {err}"); - } -} - -#[allow(clippy::too_many_lines)] -async fn websocket( - state: AppState, - stream: WebSocket, - vault_id: VaultId, -) -> Result<(), SyncServerError> { - let (mut sender, mut websocket_receiver) = stream.split(); - - let authed_handshake = get_authenticated_handshake( - &state, - &vault_id, - websocket_receiver - .next() - .await - .transpose() - .unwrap_or_default(), - )?; - - info!( - "WebSocket handshake successful for vault `{vault_id}` for `{}`", - authed_handshake.handshake.device_id - ); - - let mut broadcast_receiver = state.broadcasts.get_receiver(vault_id.clone()).await; - - send_update_over_websocket( - &WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: get_unseen_documents( - &state, - &vault_id, - authed_handshake.handshake.last_seen_vault_update_id, - ) - .await?, - is_initial_sync: true, - }), - &mut sender, - ) - .await?; - - send_update_over_websocket( - &WebSocketServerMessage::CursorPositions(CursorPositionFromServer { - clients: state.cursors.get_cursors(&vault_id).await, - }), - &mut sender, - ) - .await?; - - let device_id = authed_handshake.handshake.device_id.clone(); - let mut send_task = tokio::spawn(async move { - while let Ok(update) = broadcast_receiver.recv().await { - if Some(&device_id) == update.origin_device_id.as_ref() { - continue; - } - - let message = match update.message { - WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }) => { - WebSocketServerMessage::CursorPositions(CursorPositionFromServer { - clients: clients - .into_iter() - .filter(|client| client.device_id != device_id) - .collect(), - }) - } - WebSocketServerMessage::VaultUpdate(_) => update.message, - }; - - send_update_over_websocket(&message, &mut sender).await?; - } - - Ok::<(), SyncServerError>(()) - }); - - let device_id = authed_handshake.handshake.device_id.clone(); - let vault_id_clone = vault_id.clone(); - let cursor_manager = state.cursors.clone(); - let mut receive_task = tokio::spawn(async move { - while let Some(Ok(Message::Text(message))) = websocket_receiver.next().await { - let message: WebSocketClientMessage = serde_json::from_str(&message) - .context("Failed to parse WebSocket message from client") - .map_err(server_error)?; - - match message { - WebSocketClientMessage::Handshake(_) => { - return Err(client_error(anyhow::anyhow!( - "Unexpected handshake message" - ))); - } - WebSocketClientMessage::CursorPositions(cursors) => { - cursor_manager - .update_cursors( - vault_id_clone.clone(), - authed_handshake.user.name.clone(), - &device_id, - cursors.documents_with_cursors, - ) - .await; - } - } - } - - Ok::<(), SyncServerError>(()) - }); - - tokio::select! { - _ = &mut send_task => receive_task.abort(), - _ = &mut receive_task => send_task.abort(), - }; - - let result: Result<(), SyncServerError> = (async { - send_task - .await - .context("WebSocket send task failed") - .map_err(client_error) - .and_then(|err| err)?; - - receive_task - .await - .context("WebSocket receive task failed") - .map_err(client_error) - .and_then(|err| err)?; - - Ok(()) - }) - .await; - - state - .cursors - .remove_cursors_of_device(&vault_id, &authed_handshake.handshake.device_id) - .await; - - if result.is_err() { - info!( - "WebSocket disconnected on vault `{vault_id}` for `{}`", - authed_handshake.handshake.device_id - ); - } - - result -} diff --git a/sync-server/src/utils.rs b/sync-server/src/utils.rs index b501ecb2..a652bec6 100644 --- a/sync-server/src/utils.rs +++ b/sync-server/src/utils.rs @@ -1,8 +1,4 @@ -pub mod dedup_paths; -pub mod find_first_available_path; -pub mod is_binary; -pub mod is_file_type_mergable; pub mod log_level; -pub mod normalize; +pub mod normalize_vault_id; +pub mod portable_path; pub mod rotating_file_writer; -pub mod sanitize_path; diff --git a/sync-server/src/utils/dedup_paths.rs b/sync-server/src/utils/dedup_paths.rs deleted file mode 100644 index bc687f6a..00000000 --- a/sync-server/src/utils/dedup_paths.rs +++ /dev/null @@ -1,150 +0,0 @@ -use regex::Regex; - -pub fn dedup_paths(path: &str) -> impl Iterator { - let mut path_parts = path.split('/').collect::>(); - let file_name = path_parts.pop().unwrap().to_owned(); - - let mut directory = path_parts.join("/"); - if !directory.is_empty() { - directory.push('/'); - } - - // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should split as ".config" + ".json" - let is_simple_dotfile = file_name.starts_with('.') && file_name.matches('.').count() == 1; - - let (stem, extension) = if is_simple_dotfile { - (file_name.clone(), String::new()) - } else { - // Regular file or dotfile with extension - let name_parts = file_name.rsplitn(2, '.').collect::>(); - let mut reverse_parts = name_parts.into_iter().rev(); - match (reverse_parts.next(), reverse_parts.next()) { - (Some(stem), maybe_extension) => ( - stem.to_owned(), - maybe_extension - .map(|ext| format!(".{ext}")) - .unwrap_or_default(), - ), - _ => unreachable!("Path must have at least one part"), - } - }; - - let regex = Regex::new(r" \((\d+)\)$").unwrap(); - let start_number = regex - .captures(&stem) - .and_then(|caps| caps.get(1)) - .and_then(|m| m.as_str().parse::().ok()) - .unwrap_or(0); - - let clean_stem = regex.replace(&stem, "").to_string(); - - (start_number..).map(move |dedup_number| { - if dedup_number == 0 { - format!("{directory}{clean_stem}{extension}") - } else { - format!("{directory}{clean_stem} ({dedup_number}){extension}") - } - }) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_dedup_paths() { - let mut deduped = dedup_paths("file.txt"); - assert_eq!(deduped.next(), Some("file.txt".to_owned())); - assert_eq!(deduped.next(), Some("file (1).txt".to_owned())); - assert_eq!(deduped.next(), Some("file (2).txt".to_owned())); - - let mut deduped = dedup_paths("file"); - assert_eq!(deduped.next(), Some("file".to_owned())); - assert_eq!(deduped.next(), Some("file (1)".to_owned())); - assert_eq!(deduped.next(), Some("file (2)".to_owned())); - - let mut deduped = dedup_paths("file (51).md"); - assert_eq!(deduped.next(), Some("file (51).md".to_owned())); - assert_eq!(deduped.next(), Some("file (52).md".to_owned())); - assert_eq!(deduped.next(), Some("file (53).md".to_owned())); - - let mut deduped = dedup_paths("file (5)"); - assert_eq!(deduped.next(), Some("file (5)".to_owned())); - assert_eq!(deduped.next(), Some("file (6)".to_owned())); - assert_eq!(deduped.next(), Some("file (7)".to_owned())); - - let mut deduped = dedup_paths("my/path.with.dots/file (5).md"); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (5).md".to_owned()) - ); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (6).md".to_owned()) - ); - - let mut deduped = dedup_paths("my/path.with.dots/file (5)"); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (5)".to_owned()) - ); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (6)".to_owned()) - ); - } - - #[test] - fn test_regex_capturing_group() { - // Single digit in parentheses - let mut deduped = dedup_paths("document (5).md"); - assert_eq!(deduped.next(), Some("document (5).md".to_owned())); - assert_eq!(deduped.next(), Some("document (6).md".to_owned())); - assert_eq!(deduped.next(), Some("document (7).md".to_owned())); - - // Multi-digit number - let mut deduped = dedup_paths("report (123).pdf"); - assert_eq!(deduped.next(), Some("report (123).pdf".to_owned())); - assert_eq!(deduped.next(), Some("report (124).pdf".to_owned())); - assert_eq!(deduped.next(), Some("report (125).pdf".to_owned())); - - // Number without extension - let mut deduped = dedup_paths("folder (99)"); - assert_eq!(deduped.next(), Some("folder (99)".to_owned())); - assert_eq!(deduped.next(), Some("folder (100)".to_owned())); - assert_eq!(deduped.next(), Some("folder (101)".to_owned())); - } - - #[test] - fn test_dedup_dotfiles() { - // Simple dotfile (no extension) - let mut deduped = dedup_paths(".gitignore"); - assert_eq!(deduped.next(), Some(".gitignore".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (1)".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (2)".to_owned())); - - // Dotfile with extension - let mut deduped = dedup_paths(".config.json"); - assert_eq!(deduped.next(), Some(".config.json".to_owned())); - assert_eq!(deduped.next(), Some(".config (1).json".to_owned())); - assert_eq!(deduped.next(), Some(".config (2).json".to_owned())); - - // Dotfile with number - let mut deduped = dedup_paths(".gitignore (5)"); - assert_eq!(deduped.next(), Some(".gitignore (5)".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (6)".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (7)".to_owned())); - - // Dotfile with extension and number - let mut deduped = dedup_paths(".config (3).json"); - assert_eq!(deduped.next(), Some(".config (3).json".to_owned())); - assert_eq!(deduped.next(), Some(".config (4).json".to_owned())); - assert_eq!(deduped.next(), Some(".config (5).json".to_owned())); - - // Dotfile in subdirectory - let mut deduped = dedup_paths("my/path/.gitignore"); - assert_eq!(deduped.next(), Some("my/path/.gitignore".to_owned())); - assert_eq!(deduped.next(), Some("my/path/.gitignore (1)".to_owned())); - assert_eq!(deduped.next(), Some("my/path/.gitignore (2)".to_owned())); - } -} diff --git a/sync-server/src/utils/find_first_available_path.rs b/sync-server/src/utils/find_first_available_path.rs deleted file mode 100644 index 7629d8f1..00000000 --- a/sync-server/src/utils/find_first_available_path.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::app_state::database::models::VaultId; -use crate::{app_state::database::Transaction, utils::dedup_paths::dedup_paths}; -use anyhow::Result; -use log::{debug, info}; - -pub async fn find_first_available_path( - vault_id: &VaultId, - sanitized_relative_path: &str, - database: &crate::app_state::database::Database, - transaction: &mut Transaction<'_>, -) -> Result { - info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`"); - for candidate in dedup_paths(sanitized_relative_path) { - debug!("Checking candidate path for deconflicting names: `{candidate}`"); - if database - .get_latest_document_by_path(vault_id, &candidate, Some(transaction)) - .await? - .is_none() - { - info!("Selected available path: `{candidate}`"); - return Ok(candidate); - } - } - - unreachable!("dedup_paths produces infinite paths"); -} diff --git a/sync-server/src/utils/is_binary.rs b/sync-server/src/utils/is_binary.rs deleted file mode 100644 index 09bfcf94..00000000 --- a/sync-server/src/utils/is_binary.rs +++ /dev/null @@ -1,26 +0,0 @@ -/// Heuristically determine if the given data is a binary or a text file's -/// content. -/// -/// Only text inputs can be reconciled using the crate's functions. -#[must_use] -pub fn is_binary(data: &[u8]) -> bool { - if data.contains(&0) { - // Even though the NUL character is valid in UTF-8, it's highly suspicious in - // human-readable text. - return true; - } - - std::str::from_utf8(data).is_err() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_binary() { - assert!(is_binary(&[0, 159, 146, 150])); - assert!(is_binary(&[0, 12])); - assert!(!is_binary(b"hello")); - } -} diff --git a/sync-server/src/utils/is_file_type_mergable.rs b/sync-server/src/utils/is_file_type_mergable.rs deleted file mode 100644 index 7aabb393..00000000 --- a/sync-server/src/utils/is_file_type_mergable.rs +++ /dev/null @@ -1,34 +0,0 @@ -pub fn is_file_type_mergable(path_or_file_name: &str, mergeable_extensions: &[String]) -> bool { - let file_extension = path_or_file_name.split('.').next_back().unwrap_or_default(); - let file_extension_lower = file_extension.to_lowercase(); - - mergeable_extensions - .iter() - .any(|ext| ext.to_lowercase() == file_extension_lower) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_file_type_mergable() { - let mergeable = vec!["md".to_owned(), "txt".to_owned()]; - - assert!(is_file_type_mergable(".md", &mergeable)); - assert!(is_file_type_mergable("hi.md", &mergeable)); - assert!(is_file_type_mergable( - "my/path/to/my/document.md", - &mergeable - )); - assert!(is_file_type_mergable("hi.MD", &mergeable)); - assert!(is_file_type_mergable( - "my/path/to/my/DOCUMENT.MD", - &mergeable - )); - - assert!(!is_file_type_mergable(".json", &mergeable)); - assert!(!is_file_type_mergable("HELLO.JSON", &mergeable)); - assert!(!is_file_type_mergable("my/config.yml", &mergeable)); - } -} diff --git a/sync-server/src/utils/normalize.rs b/sync-server/src/utils/normalize.rs deleted file mode 100644 index 6553dd25..00000000 --- a/sync-server/src/utils/normalize.rs +++ /dev/null @@ -1,13 +0,0 @@ -use serde::{Deserialize, Deserializer}; - -pub fn normalize<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - Ok(normalize_string(&s)) -} - -pub fn normalize_string(s: &str) -> String { - s.trim().to_lowercase() -} diff --git a/sync-server/src/utils/normalize_vault_id.rs b/sync-server/src/utils/normalize_vault_id.rs new file mode 100644 index 00000000..7cabf3e3 --- /dev/null +++ b/sync-server/src/utils/normalize_vault_id.rs @@ -0,0 +1,29 @@ +use anyhow::{Result, ensure}; +use serde::{Deserialize, Deserializer, de::Error}; + +pub fn normalize_vault_id<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + let vault = normalize_string(&s); + validate_vault_id(&vault).map_err(D::Error::custom)?; + Ok(vault) +} + +pub fn normalize_string(s: &str) -> String { + s.trim().to_lowercase() +} + +pub fn validate_vault_id(vault: &str) -> Result<()> { + ensure!( + !vault.is_empty() + && vault != "." + && vault != ".." + && !vault + .chars() + .any(|c| c.is_control() || matches!(c, '/' | '\\')), + "Vault names must be nonempty names without path separators, traversal components, or control characters" + ); + Ok(()) +} diff --git a/sync-server/src/utils/portable_path.rs b/sync-server/src/utils/portable_path.rs new file mode 100644 index 00000000..3c4bf4f8 --- /dev/null +++ b/sync-server/src/utils/portable_path.rs @@ -0,0 +1,231 @@ +use crate::app_state::database::models::FileManifestEntries; +use anyhow::{Result, bail, ensure}; +use std::collections::BTreeMap; +use unicode_normalization::UnicodeNormalization; + +pub const INTERNAL_DIRECTORY: &str = ".vault-link-sync"; + +// Use the same locale-independent Unicode upper-case mapping in the client. +// NFC -> uppercase -> NFC also catches ß/SS and final/ordinary sigma aliases. +pub fn path_key(path: &str) -> String { + path.nfc() + .collect::() + .to_uppercase() + .nfc() + .collect() +} + +pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { + // Store one component per node. Expanding every prefix makes a short, + // deeply nested path consume quadratic memory and CPU before any I/O. + let mut nodes: BTreeMap<(usize, String), (usize, &str, bool)> = BTreeMap::new(); + + for path in entries.values() { + ensure!(!path.is_empty(), "Path must not be empty"); + + ensure!( + path.nfc().collect::() == *path, + "Paths must be NFC normalized" + ); + + let parts: Vec<_> = path.split('/').collect(); + + ensure!( + path_key(parts[0]) != path_key(INTERNAL_DIRECTORY), + "Reserved sync directory" + ); + + let mut parent = 0; + for (index, part) in parts.iter().enumerate() { + ensure!( + !part.is_empty() && *part != "." && *part != "..", + "Invalid path component" + ); + + ensure!(!part.ends_with(['.', ' ']), "Trailing dot or space in path"); + + ensure!( + !part + .chars() + .any(|c| c.is_control() || c == '\\' || "<>:\"|?*".contains(c)), + "Non-portable filename" + ); + + let stem = part + .split('.') + .next() + .unwrap_or_default() + .trim_end_matches(' ') + .to_uppercase(); + + ensure!( + !matches!( + stem.as_str(), + "CON" | "PRN" | "AUX" | "NUL" | "CONIN$" | "CONOUT$" + ), + "Reserved filename" + ); + + if stem.starts_with("COM") || stem.starts_with("LPT") { + ensure!( + !matches!( + &stem[3..], + "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "¹" | "²" | "³" + ), + "Reserved filename" + ); + } + + let is_file = index == parts.len() - 1; + let key = (parent, path_key(part)); + + if let Some((node, spelling, previous_is_file)) = nodes.get(&key) { + if spelling != part || *previous_is_file || is_file { + bail!("Conflicting path: {path}"); + } + parent = *node; + } else { + let node = nodes.len() + 1; + nodes.insert(key, (node, part, is_file)); + parent = node; + } + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{FileManifestEntries, path_key, validate_file_manifest}; + use uuid::Uuid; + + fn manifest(paths: &[&str]) -> FileManifestEntries { + paths + .iter() + .enumerate() + .map(|(index, path)| (Uuid::from_u128(index as u128), (*path).to_owned())) + .collect() + } + + #[test] + fn path_keys_normalize_unicode_and_case_aliases() { + for (path, expected) in [ + ("Notes/Café.md", "NOTES/CAFÉ.MD"), + ("notes/Cafe\u{301}.md", "NOTES/CAFÉ.MD"), + ("Straße.md", "STRASSE.MD"), + ("σς.md", "ΣΣ.MD"), + ] { + assert_eq!(path_key(path), expected, "path: {path:?}"); + } + } + + #[test] + fn accepts_empty_manifests_and_portable_paths_with_shared_directories() { + validate_file_manifest(&manifest(&[])).unwrap(); + + let entries = manifest(&[ + "Notes/Café.md", + "Notes/日記.md", + "Notes/Archive/old.md", + ".gitkeep", + "COM10.txt", + "LPT0.txt", + ".vault-link-sync-backup/state.json", + "Notes/.vault-link-sync/state.json", + ]); + + validate_file_manifest(&entries).unwrap(); + } + + #[test] + fn does_not_impose_a_cross_platform_total_path_limit() { + for path in ["a".repeat(300), "é".repeat(200)] { + validate_file_manifest(&manifest(&[&path])).unwrap(); + } + } + + #[test] + fn deep_paths_do_not_expand_every_prefix() { + let path = vec!["a"; 10_000].join("/"); + let started = std::time::Instant::now(); + validate_file_manifest(&manifest(&[&path])).unwrap(); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "A 20 KB path exhausted the validation work budget: {:?}", + started.elapsed() + ); + } + + #[test] + fn rejects_invalid_components_characters_and_non_normalized_paths() { + for path in [ + "", + "/notes.md", + "notes.md/", + "notes//a.md", + "./notes.md", + "notes/../a.md", + "notes./a.md", + "notes /a.md", + "Cafe\u{301}.md", + ] { + assert!( + validate_file_manifest(&manifest(&[path])).is_err(), + "accepted invalid path: {path:?}" + ); + } + + for character in ['\\', '<', '>', ':', '"', '|', '?', '*', '\0', '\u{85}'] { + let path = format!("notes/a{character}b.md"); + + assert!( + validate_file_manifest(&manifest(&[&path])).is_err(), + "accepted non-portable character in path: {path:?}" + ); + } + } + + #[test] + fn rejects_reserved_sync_and_windows_names() { + for path in [ + ".vault-link-sync", + ".VAULT-LINK-SYNC/state.json", + "con", + "prn.txt", + "AUX.md", + "nul .txt", + "CONIN$", + "conout$.log", + "com1.txt", + "LPT9", + "COM¹.txt", + "lpt²/file.md", + "COM³.txt", + ] { + assert!( + validate_file_manifest(&manifest(&[path])).is_err(), + "accepted reserved path: {path:?}" + ); + } + } + + #[test] + fn rejects_path_conflicts_in_either_entry_order() { + for [first, second] in [ + ["notes.md", "notes.md"], + ["Notes.md", "notes.md"], + ["Straße.md", "STRASSE.md"], + ["ς.md", "σ.md"], + ["Notes/a.md", "notes/b.md"], + ["notes", "notes/a.md"], + ] { + for paths in [[first, second], [second, first]] { + assert!( + validate_file_manifest(&manifest(&paths)).is_err(), + "accepted conflicting paths: {paths:?}" + ); + } + } + } +} diff --git a/sync-server/src/utils/sanitize_path.rs b/sync-server/src/utils/sanitize_path.rs deleted file mode 100644 index 9703225c..00000000 --- a/sync-server/src/utils/sanitize_path.rs +++ /dev/null @@ -1,34 +0,0 @@ -/// Sanitize the document's path to allow all clients to create the same path in -/// their filesystem. If we didn't do this server-side, client's would need to -/// deal with mapping invalid names to valid ones and then back. -pub fn sanitize_path(path: &str) -> String { - let options = sanitize_filename::Options { - truncate: true, - windows: true, // Windows is the lowest common denominator - replacement: "", - }; - - path.split('/') - .map(|part| { - let proposal = sanitize_filename::sanitize_with_options(part, options.clone()); - if !part.is_empty() && proposal.is_empty() { - "_".to_owned() - } else { - proposal - } - }) - .collect::>() - .join("/") -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_sanitize_path() { - assert_eq!(sanitize_path("/my/path/what?"), "/my/path/what"); - assert_eq!(sanitize_path("file (1).md"), "file (1).md"); - assert_eq!(sanitize_path("/my/path/\\\\:?"), "/my/path/_"); - } -}