Add deterministic-tests (#190)

Reviewed-on: https://home.schmelczer.dev/git/git/andras/vault-link/pulls/190
Co-authored-by: Andras Schmelczer <andras@schmelczer.dev>
Co-committed-by: Andras Schmelczer <andras@schmelczer.dev>

frontend/deterministic-tests/README.md

@@ -0,0 +1,118 @@
+# Deterministic Tests
+
+Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case.
+
+Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios.
+
+## How it works
+
+Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one.
+
+Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process.
+
+The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit.
+
+## Step types
+
+Clients always start with syncing disabled.
+
+**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited):
+
+- `create`, `update`, `rename`, `delete`
+- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path.
+
+**Sync control:**
+
+- `sync` — wait for a specific client or all clients to finish pending operations
+- `barrier` — retry until all clients converge to identical file state (60s timeout)
+- `enable-sync` / `disable-sync` — simulate going online/offline
+- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable
+- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync`
+
+**WebSocket control** (per-client):
+
+- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client
+
+**Server control:**
+
+- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process
+- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire.
+
+**Fault injection** (per-client):
+
+- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit.
+- `wait-for-dropped-create-response` — wait until the armed drop has fired.
+
+**Assertions:**
+
+- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback
+
+## Running
+
+```sh
+# Build server first
+cd sync-server && cargo build --release && cd -
+
+# Run all tests
+cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests
+
+# Filter by name
+npm run test -w deterministic-tests -- --filter=rename
+
+# Control parallelism (default: number of CPU cores)
+npm run test -w deterministic-tests -- -j 4
+```
+
+## Adding a test
+
+1. Create `src/tests/my-scenario.test.ts`:
+
+```typescript
+import type { TestDefinition } from "../test-definition";
+
+export const myScenarioTest: TestDefinition = {
+  description:
+    "Client 0 creates A.md offline. After syncing, both clients should have the file.",
+  clients: 2,
+  steps: [
+    { type: "create", client: 0, path: "A.md", content: "hello" },
+    { type: "enable-sync", client: 0 },
+    { type: "enable-sync", client: 1 },
+    { type: "barrier" },
+    {
+      type: "assert-consistent",
+      verify: (s) => {
+        s.assertFileCount(1).assertContent("A.md", "hello");
+      }
+    }
+  ]
+};
+```
+
+The `verify` callback receives an `AssertableState` object with chainable assertion methods:
+
+```typescript
+s.assertFileCount(n);                       // exact file count
+s.assertFileExists("path");                 // file must exist
+s.assertFileNotExists("path");              // file must not exist
+s.assertContent("path", "expected");        // exact content match
+s.assertContains("path", "a", "b");         // all substrings present in file
+s.assertContainsAny("path", "a", "b");      // at least one substring present
+s.assertAnyFileContains("text");            // substring present in some file
+s.assertNoFileContains("text");             // substring absent from every file
+s.assertSubstringCount("path", "x", 3);     // substring appears exactly N times
+s.assertContentInAtMostOneFile("text");     // no duplicate content
+s.ifFileExists("path", (s) => { /* … */ }); // conditional block
+s.getContent("path");                       // raw content (or "" if missing)
+```
+
+2. Register it in `src/test-registry.ts`:
+
+```typescript
+import { myScenarioTest } from "./tests/my-scenario.test";
+
+const TESTS = {
+  // ...
+  "my-scenario": myScenarioTest
+};
+```

frontend/deterministic-tests/package.json

@@ -0,0 +1,23 @@
+{
+    "name": "deterministic-tests",
+    "version": "0.14.0",
+    "private": true,
+    "bin": {
+        "deterministic-tests": "./dist/cli.js"
+    },
+    "scripts": {
+        "dev": "webpack watch --mode development",
+        "build": "webpack --mode production",
+        "test": "npm run build && node dist/cli.js"
+    },
+    "devDependencies": {
+        "commander": "^14.0.2",
+        "@types/node": "^25.0.2",
+        "sync-client": "file:../sync-client",
+        "ts-loader": "^9.5.4",
+        "tslib": "2.8.1",
+        "typescript": "5.9.3",
+        "webpack": "^5.103.0",
+        "webpack-cli": "^6.0.1"
+    }
+}

frontend/deterministic-tests/src/cli.ts

@@ -0,0 +1,243 @@
+import { TestRunner } from "./test-runner";
+import { ServerControl } from "./server-control";
+import { ServerManager } from "./server-manager";
+import { PrefixedLogger } from "./prefixed-logger";
+import { TESTS } from "./test-registry";
+import type { TestDefinition, TestResult } from "./test-definition";
+import { parseArgs } from "./parse-args";
+import { runWithConcurrency } from "./run-with-concurrency";
+import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts";
+import * as path from "node:path";
+import * as fs from "node:fs";
+import { debugging, Logger } from "sync-client";
+
+const logger = new Logger();
+debugging.logToConsole(logger, { useColors: true });
+
+process.on("unhandledRejection", (reason) => {
+    logger.error(`Unhandled Rejection: ${reason}`);
+    process.exit(1);
+});
+
+process.on("uncaughtException", (error) => {
+    logger.error(`Uncaught Exception: ${error}`);
+    process.exit(1);
+});
+
+const serverManager = new ServerManager(logger);
+serverManager.installSignalHandlers();
+
+function testUsesPauseServer(test: TestDefinition): boolean {
+    return test.steps.some(
+        (step) =>
+            step.type === "pause-server" ||
+            step.type === "resume-server" ||
+            step.type === "resume-server-until-history-then-pause"
+    );
+}
+
+/**
+ * Walk up from the CLI binary's location until we find a directory
+ * containing `sync-server/` and `frontend/`.
+ */
+function findProjectRoot(): string {
+    let dir = path.dirname(__filename);
+    const root = path.parse(dir).root;
+    while (dir !== root) {
+        if (
+            fs.existsSync(path.join(dir, "sync-server")) &&
+            fs.existsSync(path.join(dir, "frontend"))
+        ) {
+            return dir;
+        }
+        dir = path.dirname(dir);
+    }
+    throw new Error(
+        `Could not locate project root (no ancestor of ${__filename} contains both 'sync-server' and 'frontend')`
+    );
+}
+
+interface NamedTestResult {
+    name: string;
+    result: TestResult;
+}
+
+async function runSharedServerTest(
+    name: string,
+    test: TestDefinition,
+    sharedServer: ServerControl
+): Promise<NamedTestResult> {
+    const testLogger = new PrefixedLogger(logger, name);
+    const runner = new TestRunner(
+        sharedServer,
+        testLogger,
+        TOKEN,
+        sharedServer.remoteUri
+    );
+    const result = await runner.runTest(name, test);
+    if (result.success) {
+        logger.info(`PASSED: ${name} (${result.duration}ms)`);
+    } else {
+        logger.error(`FAILED: ${name} - ${result.error}`);
+    }
+    return { name, result };
+}
+
+/**
+ * Run a test with its own dedicated server (for tests that use pause-server).
+ * SIGSTOP/SIGCONT affects the entire server process, so these tests need
+ * isolated servers to avoid interfering with other tests.
+ */
+async function runDedicatedServerTest(
+    name: string,
+    test: TestDefinition,
+    serverPath: string,
+    configPath: string
+): Promise<NamedTestResult> {
+    const testLogger = new PrefixedLogger(logger, name);
+    const server = new ServerControl(serverPath, configPath, testLogger);
+    serverManager.track(server);
+
+    try {
+        await server.start();
+        const runner = new TestRunner(
+            server,
+            testLogger,
+            TOKEN,
+            server.remoteUri
+        );
+        const result = await runner.runTest(name, test);
+        if (result.success) {
+            logger.info(`PASSED: ${name} (${result.duration}ms)`);
+        } else {
+            logger.error(`FAILED: ${name} - ${result.error}`);
+        }
+        return { name, result };
+    } finally {
+        try {
+            await server.stop();
+        } catch {
+            // best-effort cleanup
+        }
+        serverManager.untrack(server);
+    }
+}
+
+async function main(): Promise<void> {
+    const projectRoot = findProjectRoot();
+    const serverPath = path.join(projectRoot, SERVER_BINARY_PATH);
+    if (!fs.existsSync(serverPath)) {
+        logger.error(`Server binary not found at: ${serverPath}`);
+        process.exit(1);
+    }
+
+    const configPath = path.join(projectRoot, CONFIG_PATH);
+    if (!fs.existsSync(configPath)) {
+        logger.error(`Config file not found at: ${configPath}`);
+        process.exit(1);
+    }
+
+    const { filter, concurrency } = parseArgs(process.argv);
+
+    const testsToRun: [string, TestDefinition][] = [];
+    for (const [key, test] of Object.entries(TESTS)) {
+        if (test) {
+            if (
+                filter !== undefined &&
+                filter.length > 0 &&
+                !key.includes(filter)
+            ) {
+                continue;
+            }
+            testsToRun.push([key, test]);
+        }
+    }
+
+    if (testsToRun.length === 0) {
+        logger.error(
+            filter !== undefined && filter.length > 0
+                ? `No tests matched filter "${filter}"`
+                : "No tests found"
+        );
+        process.exit(1);
+    }
+
+    const regularTests = testsToRun.filter(([, t]) => !testUsesPauseServer(t));
+    const pauseTests = testsToRun.filter(([, t]) => testUsesPauseServer(t));
+
+    logger.info(`Server: ${serverPath}`);
+    logger.info(`Config: ${configPath}`);
+    logger.info(
+        `Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)`
+    );
+    logger.info(`Concurrency: ${concurrency}`);
+
+    const allResults: NamedTestResult[] = [];
+
+    if (regularTests.length > 0) {
+        logger.info(
+            `\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---`
+        );
+        const sharedServer = new ServerControl(serverPath, configPath, logger);
+        serverManager.track(sharedServer);
+
+        try {
+            await sharedServer.start();
+
+            const results = await runWithConcurrency(
+                regularTests,
+                concurrency,
+                async ([name, test]) =>
+                    runSharedServerTest(name, test, sharedServer)
+            );
+
+            allResults.push(...results);
+        } finally {
+            try {
+                await sharedServer.stop();
+            } catch (error) {
+                logger.warn(
+                    `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}`
+                );
+            }
+            serverManager.untrack(sharedServer);
+        }
+    }
+
+    if (pauseTests.length > 0) {
+        logger.info(
+            `\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---`
+        );
+
+        const results = await runWithConcurrency(
+            pauseTests,
+            concurrency,
+            async ([name, test]) =>
+                runDedicatedServerTest(name, test, serverPath, configPath)
+        );
+
+        allResults.push(...results);
+    }
+
+    const passed = allResults.filter((r) => r.result.success);
+    const failed = allResults.filter((r) => !r.result.success);
+
+    logger.info(
+        `\n--- Results: ${passed.length}/${allResults.length} passed ---`
+    );
+
+    if (failed.length > 0) {
+        for (const { name, result } of failed) {
+            logger.error(`  FAILED: ${name}: ${result.error}`);
+        }
+        process.exit(1);
+    } else {
+        logger.info("All tests passed!");
+        process.exit(0);
+    }
+}
+
+main().catch((err: unknown) => {
+    logger.error(`Unexpected error: ${err}`);
+    process.exit(1);
+});

frontend/deterministic-tests/src/consts.ts

@@ -0,0 +1,17 @@
+export const TOKEN = "test-token-change-me";
+export const SERVER_BINARY_PATH = "sync-server/target/release/sync_server";
+export const CONFIG_PATH = "sync-server/config-e2e.yml";
+
+export const STOP_TIMEOUT_MS = 5_000;
+export const CONVERGENCE_TIMEOUT_MS = 60_000;
+export const CONVERGENCE_RETRY_DELAY_MS = 500;
+export const AGENT_INIT_TIMEOUT_MS = 30_000;
+export const IS_SYNC_ENABLED_BY_DEFAULT = false;
+
+export const WAIT_TIMEOUT_MS = 60_000;
+export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000;
+export const WEBSOCKET_POLL_INTERVAL_MS = 50;
+
+export const SERVER_READY_POLL_INTERVAL_MS = 100;
+export const SERVER_READY_MAX_ATTEMPTS = 50;
+export const SERVER_START_MAX_ATTEMPTS = 5;

frontend/deterministic-tests/src/deterministic-agent.ts

@@ -0,0 +1,483 @@
+import type {
+    HistoryEntry,
+    StoredDatabase,
+    SyncSettings,
+    RelativePath,
+    TextWithCursors
+} from "sync-client";
+import {
+    SyncClient,
+    SyncResetError,
+    debugging,
+    LogLevel,
+    utils
+} from "sync-client";
+import { assert } from "./utils/assert";
+import { sleep } from "./utils/sleep";
+import { withTimeout } from "./utils/with-timeout";
+import {
+    IS_SYNC_ENABLED_BY_DEFAULT,
+    WAIT_TIMEOUT_MS,
+    WEBSOCKET_CONNECT_TIMEOUT_MS,
+    WEBSOCKET_POLL_INTERVAL_MS
+} from "./consts";
+import { ManagedWebSocketFactory } from "./managed-websocket";
+
+export class DeterministicAgent extends debugging.InMemoryFileSystem {
+    public readonly clientId: number;
+    private readonly logger: (msg: string) => void;
+    private client!: SyncClient;
+    private data: Partial<{
+        settings: Partial<SyncSettings>;
+        database: Partial<StoredDatabase>;
+    }> = {};
+    private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT;
+    private readonly syncErrors: Error[] = [];
+    private readonly pendingSyncOperations = new Set<Promise<void>>();
+    private readonly wsFactory = new ManagedWebSocketFactory();
+    private nextWriteRename:
+        | {
+            oldPath: RelativePath;
+            newPath: RelativePath;
+        }
+        | undefined;
+    private nextCreateResponseDrop:
+        | {
+            dropped: Promise<void>;
+            resolveDropped: () => void;
+        }
+        | undefined;
+
+    public constructor(
+        clientId: number,
+        initialSettings: Partial<SyncSettings>,
+        logger: (msg: string) => void
+    ) {
+        super();
+        this.clientId = clientId;
+        this.logger = logger;
+        this.data.settings = { ...initialSettings };
+    }
+
+    public async init(
+        fetchImplementation: typeof globalThis.fetch
+    ): Promise<void> {
+        this.client = await SyncClient.create({
+            fs: this,
+            persistence: {
+                load: async () => this.data,
+                save: async (data) => void (this.data = data)
+            },
+            fetch: this.wrapFetch(fetchImplementation),
+            webSocket: this.wsFactory.constructorFn
+        });
+
+        this.client.logger.onLogEmitted.add((line) => {
+            const prefix = `[Client ${this.clientId}]`;
+            switch (line.level) {
+                case LogLevel.ERROR:
+                    this.logger(`${prefix} ERROR: ${line.message}`);
+                    break;
+                case LogLevel.WARNING:
+                    this.logger(`${prefix} WARN: ${line.message}`);
+                    break;
+                case LogLevel.INFO:
+                    this.logger(`${prefix} INFO: ${line.message}`);
+                    break;
+                case LogLevel.DEBUG:
+                    this.logger(`${prefix} DEBUG: ${line.message}`);
+                    break;
+            }
+        });
+
+        await this.client.start();
+
+        const connectionCheck = await this.client.checkConnection();
+        assert(
+            connectionCheck.isSuccessful,
+            `Client ${this.clientId} connection check failed`
+        );
+
+        if (this.isSyncEnabled) {
+            await this.waitForWebSocket();
+        }
+    }
+
+    public pauseWebSocket(): void {
+        this.log("Pausing WebSocket message delivery");
+        this.wsFactory.pause();
+    }
+
+    public resumeWebSocket(): void {
+        this.log("Resuming WebSocket message delivery");
+        this.wsFactory.resume();
+    }
+
+    public dropNextCreateResponse(): void {
+        assert(
+            this.nextCreateResponseDrop === undefined,
+            `Client ${this.clientId} already has a create response drop armed`
+        );
+        let resolveDropped!: () => void;
+        const dropped = new Promise<void>((resolve) => {
+            resolveDropped = resolve;
+        });
+        this.nextCreateResponseDrop = {
+            dropped,
+            resolveDropped
+        };
+        this.log("Armed next create response drop");
+    }
+
+    public async waitForDroppedCreateResponse(): Promise<void> {
+        assert(
+            this.nextCreateResponseDrop !== undefined,
+            `Client ${this.clientId} has no create response drop armed`
+        );
+        await withTimeout(
+            this.nextCreateResponseDrop.dropped,
+            WAIT_TIMEOUT_MS,
+            `Client ${this.clientId} timed out waiting for create response drop`
+        );
+        this.log("Create response was dropped after server commit");
+    }
+
+    public async waitForHistoryEntry(
+        matches: (entry: HistoryEntry) => boolean,
+        onMatch?: (entry: HistoryEntry) => void
+    ): Promise<void> {
+        const existing = this.client.getHistoryEntries().find(matches);
+        if (existing !== undefined) {
+            onMatch?.(existing);
+            return;
+        }
+
+        await withTimeout(
+            new Promise<void>((resolve) => {
+                const unsubscribe = this.client.onSyncHistoryUpdated.add(() => {
+                    const entry = this.client
+                        .getHistoryEntries()
+                        .find(matches);
+                    if (entry === undefined) {
+                        return;
+                    }
+
+                    unsubscribe();
+                    onMatch?.(entry);
+                    resolve();
+                });
+            }),
+            WAIT_TIMEOUT_MS,
+            `Client ${this.clientId} timed out waiting for history entry`
+        );
+    }
+
+    public async waitForSync(): Promise<void> {
+        this.log("Waiting for sync to complete...");
+        // Drain agent-level sync operations first. These are the fire-and-forget
+        // promises from enqueueSync() that call into the SyncClient's methods.
+        // Without this, waitUntilFinished() might return before the SyncClient
+        // has even been told about the operation.
+        await this.drainPendingSyncOperations();
+        await withTimeout(
+            this.client.waitUntilFinished(),
+            WAIT_TIMEOUT_MS,
+            `Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms`
+        );
+        if (this.syncErrors.length > 0) {
+            const errors = this.syncErrors.splice(0);
+            throw new Error(
+                `Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}`
+            );
+        }
+        this.log("Sync complete");
+    }
+
+    public async reset(): Promise<void> {
+        this.log("Resetting client (clears tracked state, keeps disk files)");
+        await this.drainPendingSyncOperations();
+        await this.client.reset();
+        if (this.isSyncEnabled) {
+            await this.waitForWebSocket();
+        }
+    }
+
+    public async disableSync(): Promise<void> {
+        this.log("Disabling sync");
+        // Drain pending enqueued operations before disabling so the SyncClient
+        // knows about all operations that were enqueued while sync was enabled.
+        await this.drainPendingSyncOperations();
+        await this.client.setSetting("isSyncEnabled", false);
+        this.isSyncEnabled = false;
+        // Wait for in-flight operations to drain. Disabling sync triggers
+        // a reset, which aborts in-flight fetches with SyncResetError.
+        try {
+            await withTimeout(
+                this.client.waitUntilFinished(),
+                WAIT_TIMEOUT_MS,
+                `Client ${this.clientId} disableSync drain timed out`
+            );
+        } catch (error) {
+            if (error instanceof Error && error.name === "SyncResetError") {
+                this.log("Disable sync drain interrupted by reset (expected)");
+            } else {
+                throw error;
+            }
+        }
+    }
+
+    public async enableSync(): Promise<void> {
+        this.log("Enabling sync");
+        await this.client.setSetting("isSyncEnabled", true);
+        this.isSyncEnabled = true;
+        await this.waitForWebSocket();
+    }
+
+    public async getFileContent(path: string): Promise<string> {
+        const bytes = await this.read(path);
+        return new TextDecoder().decode(bytes);
+    }
+
+    public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void {
+        assert(
+            this.nextWriteRename === undefined,
+            `Client ${this.clientId} already has a next-write rename armed`
+        );
+        this.nextWriteRename = { oldPath, newPath };
+        this.log(`Armed next write rename: ${oldPath} -> ${newPath}`);
+    }
+
+    public async cleanup(): Promise<void> {
+        this.log("Cleaning up...");
+        // Guard against uninitialized client (init() failed partway).
+        // The class field uses `!:` so TS thinks this is always defined,
+        // but at runtime it can be undefined when init() throws partway.
+        const maybeClient = this.client as SyncClient | undefined;
+        if (maybeClient === undefined) {
+            this.log("Client not initialized, nothing to clean up");
+            return;
+        }
+        try {
+            await this.drainPendingSyncOperations();
+            await withTimeout(
+                this.client.waitUntilFinished(),
+                WAIT_TIMEOUT_MS,
+                `Client ${this.clientId} cleanup waitUntilFinished timed out`
+            );
+        } catch (error) {
+            if (error instanceof Error && error.name === "SyncResetError") {
+                this.log(`Cleanup interrupted by reset (expected): ${error}`);
+            } else {
+                this.log(`Cleanup waitUntilFinished failed: ${error}`);
+            }
+        }
+        // Surface any background sync errors that arrived after the last
+        // waitForSync (e.g. between the final assert-consistent and here).
+        // Without this, regressions that fault the engine during the very
+        // last step of a test would be silently swallowed.
+        const pendingErrors = this.syncErrors.splice(0);
+        await this.client.destroy();
+        this.log("Cleanup complete");
+        if (pendingErrors.length > 0) {
+            throw new Error(
+                `Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}`
+            );
+        }
+    }
+
+    public override async read(path: RelativePath): Promise<Uint8Array> {
+        await Promise.resolve();
+        return super.read(path);
+    }
+
+    public override async write(
+        path: RelativePath,
+        content: Uint8Array
+    ): Promise<void> {
+        await Promise.resolve();
+        const isNew = !this.files.has(path);
+        await super.write(path, content);
+
+        if (this.isSyncEnabled && isNew) {
+            this.enqueueSync(async () => {
+                this.client.syncLocallyCreatedFile(path);
+            });
+        }
+
+        const nextWriteRename = this.nextWriteRename;
+        if (
+            nextWriteRename !== undefined &&
+            nextWriteRename.oldPath === path
+        ) {
+            this.nextWriteRename = undefined;
+            await super.rename(
+                nextWriteRename.oldPath,
+                nextWriteRename.newPath
+            );
+            if (this.isSyncEnabled) {
+                this.enqueueSync(async () => {
+                    this.client.syncLocallyUpdatedFile({
+                        oldPath: nextWriteRename.oldPath,
+                        relativePath: nextWriteRename.newPath
+                    });
+                });
+            }
+            // The rename consumed `path`. Skip the post-update enqueue below
+            // — it would send a syncLocallyUpdatedFile for a path that no
+            // longer exists.
+            return;
+        }
+
+        if (!this.isSyncEnabled) {
+            return;
+        }
+
+        if (!isNew) {
+            this.enqueueSync(async () => {
+                this.client.syncLocallyUpdatedFile({ relativePath: path });
+            });
+        }
+    }
+
+    public override async atomicUpdateText(
+        path: RelativePath,
+        updater: (current: TextWithCursors) => TextWithCursors
+    ): Promise<string> {
+        const result = await super.atomicUpdateText(path, updater);
+        if (this.isSyncEnabled) {
+            this.enqueueSync(async () => {
+                this.client.syncLocallyUpdatedFile({ relativePath: path });
+            });
+        }
+        return result;
+    }
+
+    public override async delete(path: RelativePath): Promise<void> {
+        await super.delete(path);
+        if (this.isSyncEnabled) {
+            this.enqueueSync(async () => {
+                this.client.syncLocallyDeletedFile(path);
+            });
+        }
+    }
+
+    public override async rename(
+        oldPath: RelativePath,
+        newPath: RelativePath
+    ): Promise<void> {
+        await super.rename(oldPath, newPath);
+        if (this.isSyncEnabled) {
+            this.enqueueSync(async () => {
+                this.client.syncLocallyUpdatedFile({
+                    oldPath,
+                    relativePath: newPath
+                });
+            });
+        }
+    }
+
+    private async waitForWebSocket(): Promise<void> {
+        const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS;
+        while (!this.client.isWebSocketConnected && Date.now() < deadline) {
+            await sleep(WEBSOCKET_POLL_INTERVAL_MS);
+        }
+        assert(
+            this.client.isWebSocketConnected,
+            `Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms`
+        );
+    }
+
+    /**
+     * Wait until all agent-level enqueued sync operations have completed.
+     * Uses a loop because completing one operation can trigger new enqueues.
+     */
+    private async drainPendingSyncOperations(): Promise<void> {
+        while (this.pendingSyncOperations.size > 0) {
+            await utils.awaitAll([...this.pendingSyncOperations]);
+        }
+    }
+
+    private enqueueSync(operation: () => Promise<void>): void {
+        const promise = this.executeSyncOperation(operation).catch(
+            (error: unknown) => {
+                const err =
+                    error instanceof Error ? error : new Error(String(error));
+                this.log(`Background sync failed: ${err.message}`);
+                this.syncErrors.push(err);
+            }
+        );
+        this.pendingSyncOperations.add(promise);
+        void promise.finally(() => {
+            this.pendingSyncOperations.delete(promise);
+        });
+    }
+
+    private async executeSyncOperation(
+        operation: () => Promise<void>
+    ): Promise<void> {
+        try {
+            await operation();
+        } catch (error) {
+            if (error instanceof Error && error.name === "SyncResetError") {
+                this.log(`Sync operation interrupted by reset: ${error}`);
+                return;
+            }
+            if (
+                error instanceof Error &&
+                error.message.includes("has been destroyed")
+            ) {
+                this.log(`Sync operation interrupted by destroy: ${error}`);
+                return;
+            }
+
+            throw error;
+        }
+    }
+
+    private log(message: string): void {
+        this.logger(`[Client ${this.clientId}] ${message}`);
+    }
+
+    private wrapFetch(
+        fetchImplementation: typeof globalThis.fetch
+    ): typeof globalThis.fetch {
+        return async (input, init) => {
+            const response = await fetchImplementation(input, init);
+            const drop = this.nextCreateResponseDrop;
+            if (
+                drop !== undefined &&
+                DeterministicAgent.isCreateDocumentRequest(input, init)
+            ) {
+                this.nextCreateResponseDrop = undefined;
+                try {
+                    await response.body?.cancel();
+                } catch {
+                    // Best-effort — body may already be consumed/closed.
+                }
+                drop.resolveDropped();
+                throw new SyncResetError();
+            }
+            return response;
+        };
+    }
+
+    private static isCreateDocumentRequest(
+        input: RequestInfo | URL,
+        init: RequestInit | undefined
+    ): boolean {
+        const method =
+            init?.method ??
+            (typeof Request !== "undefined" && input instanceof Request
+                ? input.method
+                : "GET");
+        if (method.toUpperCase() !== "POST") {
+            return false;
+        }
+
+        const url =
+            input instanceof URL
+                ? input
+                : new URL(typeof input === "string" ? input : input.url);
+        return /\/documents\/?$/.test(url.pathname);
+    }
+}

frontend/deterministic-tests/src/managed-websocket.ts

@@ -0,0 +1,245 @@
+/**
+ * A WebSocket wrapper that can pause and resume message delivery.
+ * When paused, incoming messages are buffered. When resumed, buffered
+ * messages are delivered in order via the onmessage handler.
+ *
+ * Member layout follows typescript-eslint default member-ordering: all
+ * accessor properties are declared with `declare` and wired through the
+ * constructor using Object.defineProperty so we don't need conflicting
+ * get/set accessor pairs.
+ */
+class ManagedWebSocket implements WebSocket {
+    public static readonly CONNECTING = WebSocket.CONNECTING;
+    public static readonly OPEN = WebSocket.OPEN;
+    public static readonly CLOSING = WebSocket.CLOSING;
+    public static readonly CLOSED = WebSocket.CLOSED;
+
+    public readonly CONNECTING = WebSocket.CONNECTING;
+    public readonly OPEN = WebSocket.OPEN;
+    public readonly CLOSING = WebSocket.CLOSING;
+    public readonly CLOSED = WebSocket.CLOSED;
+
+    declare public readonly readyState: number;
+    declare public readonly url: string;
+    declare public readonly protocol: string;
+    declare public readonly extensions: string;
+    declare public readonly bufferedAmount: number;
+    declare public binaryType: BinaryType;
+    declare public onopen: ((this: WebSocket, ev: Event) => unknown) | null;
+    declare public onclose:
+        | ((this: WebSocket, ev: CloseEvent) => unknown)
+        | null;
+    declare public onerror: ((this: WebSocket, ev: Event) => unknown) | null;
+    declare public onmessage:
+        | ((this: WebSocket, ev: MessageEvent) => unknown)
+        | null;
+
+    private readonly ws: WebSocket;
+    private readonly bufferedMessages: MessageEvent[] = [];
+    private paused = false;
+    private externalOnMessage: ((event: MessageEvent) => unknown) | null = null;
+
+    public constructor(url: string | URL, protocols?: string | string[]) {
+        this.ws = new WebSocket(url, protocols);
+
+        const { ws } = this;
+        Object.defineProperties(this, {
+            readyState: {
+                get: (): number => ws.readyState,
+                enumerable: true,
+                configurable: true
+            },
+            url: {
+                get: (): string => ws.url,
+                enumerable: true,
+                configurable: true
+            },
+            protocol: {
+                get: (): string => ws.protocol,
+                enumerable: true,
+                configurable: true
+            },
+            extensions: {
+                get: (): string => ws.extensions,
+                enumerable: true,
+                configurable: true
+            },
+            bufferedAmount: {
+                get: (): number => ws.bufferedAmount,
+                enumerable: true,
+                configurable: true
+            },
+            binaryType: {
+                get: (): BinaryType => ws.binaryType,
+                set: (v: BinaryType): void => {
+                    ws.binaryType = v;
+                },
+                enumerable: true,
+                configurable: true
+            },
+            onopen: {
+                get: (): ((this: WebSocket, ev: Event) => unknown) | null =>
+                    ws.onopen,
+                set: (
+                    h: ((this: WebSocket, ev: Event) => unknown) | null
+                ): void => {
+                    ws.onopen = h;
+                },
+                enumerable: true,
+                configurable: true
+            },
+            onclose: {
+                get: ():
+                    | ((this: WebSocket, ev: CloseEvent) => unknown)
+                    | null => ws.onclose,
+                set: (
+                    h: ((this: WebSocket, ev: CloseEvent) => unknown) | null
+                ): void => {
+                    ws.onclose = h;
+                },
+                enumerable: true,
+                configurable: true
+            },
+            onerror: {
+                get: (): ((this: WebSocket, ev: Event) => unknown) | null =>
+                    ws.onerror,
+                set: (
+                    h: ((this: WebSocket, ev: Event) => unknown) | null
+                ): void => {
+                    ws.onerror = h;
+                },
+                enumerable: true,
+                configurable: true
+            },
+            onmessage: {
+                get: ():
+                    | ((this: WebSocket, ev: MessageEvent) => unknown)
+                    | null => this.externalOnMessage,
+                set: (
+                    h: ((this: WebSocket, ev: MessageEvent) => unknown) | null
+                ): void => {
+                    this.externalOnMessage = h;
+                },
+                enumerable: true,
+                configurable: true
+            }
+        });
+
+        this.ws.onmessage = (event: MessageEvent): void => {
+            if (this.paused) {
+                this.bufferedMessages.push(event);
+            } else {
+                this.externalOnMessage?.(event);
+            }
+        };
+    }
+
+    public pause(): void {
+        this.paused = true;
+    }
+
+    public resume(): void {
+        // Drain buffered messages BEFORE flipping `paused` to false.
+        // If `externalOnMessage` is async (its return type is `unknown`),
+        // dispatch yields control between buffered messages, and a fresh
+        // live `ws.onmessage` event firing during that yield would jump
+        // ahead of unprocessed buffered messages — silently reordering
+        // events relative to the wire. Keeping `paused = true` during the
+        // drain forces the live handler to keep buffering, so we splice
+        // those late arrivals onto the tail and dispatch them in order.
+        while (this.bufferedMessages.length > 0) {
+            const messages = this.bufferedMessages.splice(0);
+            for (const msg of messages) {
+                this.externalOnMessage?.(msg);
+            }
+        }
+        this.paused = false;
+    }
+
+    public send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
+        this.ws.send(data);
+    }
+
+    public close(code?: number, reason?: string): void {
+        this.ws.close(code, reason);
+    }
+
+    public addEventListener(
+        ...args: Parameters<WebSocket["addEventListener"]>
+    ): void {
+        // Only the `.onmessage` setter routes through the pause buffer.
+        // If sync-client ever attaches "message" listeners via
+        // addEventListener instead, those messages would bypass pause/resume
+        // and deterministic tests would silently lose their fault injection.
+        if (args[0] === "message") {
+            throw new Error(
+                "ManagedWebSocket: addEventListener('message') bypasses the " +
+                    "pause buffer. Use the .onmessage setter instead, or " +
+                    "extend ManagedWebSocket to route message listeners."
+            );
+        }
+        this.ws.addEventListener(...args);
+    }
+
+    public removeEventListener(
+        ...args: Parameters<WebSocket["removeEventListener"]>
+    ): void {
+        this.ws.removeEventListener(...args);
+    }
+
+    public dispatchEvent(event: Event): boolean {
+        return this.ws.dispatchEvent(event);
+    }
+}
+
+/**
+ * Factory that creates ManagedWebSocket instances and tracks them
+ * for pause/resume control from the test harness
+ */
+export class ManagedWebSocketFactory {
+    // Append-only: closed sockets stay tracked. Bounded per test (one
+    // factory per agent, each test discards its agents on cleanup), so
+    // not a real leak — but iterating over closed instances on
+    // pause/resume is a deliberate no-op since their `.onmessage` is
+    // already detached.
+    private readonly instances: ManagedWebSocket[] = [];
+    // Sticky pause state: applied to current instances on `pause()` AND
+    // to any new instance created later (e.g. WS reconnect after a
+    // `disable-sync` / `reset` cycle). Without this, a test pausing the
+    // WS before the agent reconnects would silently see the new socket
+    // start un-paused and miss the messages it meant to buffer.
+    private currentlyPaused = false;
+
+    public get constructorFn(): typeof globalThis.WebSocket {
+        const trackInstance = (instance: ManagedWebSocket): void => {
+            this.instances.push(instance);
+            if (this.currentlyPaused) {
+                instance.pause();
+            }
+        };
+        class TrackedManagedWebSocket extends ManagedWebSocket {
+            public constructor(
+                url: string | URL,
+                protocols?: string | string[]
+            ) {
+                super(url, protocols);
+                trackInstance(this);
+            }
+        }
+        return TrackedManagedWebSocket;
+    }
+
+    public pause(): void {
+        this.currentlyPaused = true;
+        for (const ws of this.instances) {
+            ws.pause();
+        }
+    }
+
+    public resume(): void {
+        this.currentlyPaused = false;
+        for (const ws of this.instances) {
+            ws.resume();
+        }
+    }
+}

frontend/deterministic-tests/src/parse-args.ts

@@ -0,0 +1,43 @@
+import * as os from "node:os";
+import { Command, InvalidArgumentError } from "commander";
+
+export interface CliArgs {
+    filter: string | undefined;
+    concurrency: number;
+}
+
+function parsePositiveInt(value: string): number {
+    const n = parseInt(value, 10);
+    if (isNaN(n) || n <= 0) {
+        throw new InvalidArgumentError("must be a positive integer");
+    }
+    return n;
+}
+
+export function parseArgs(argv: string[]): CliArgs {
+    const program = new Command();
+
+    program
+        .name("deterministic-tests")
+        .description("Scripted multi-client sync tests against a real server")
+        .option(
+            "-f, --filter <substring>",
+            "Run only tests whose name contains this substring"
+        )
+        .option(
+            "-j, --concurrency <number>",
+            "Number of tests to run in parallel",
+            parsePositiveInt,
+            os.cpus().length
+        );
+
+    program.parse(argv);
+
+    /* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
+    const opts = program.opts();
+    const filter = opts.filter as string | undefined;
+    const concurrency = opts.concurrency as number;
+    /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */
+
+    return { filter, concurrency };
+}

frontend/deterministic-tests/src/prefixed-logger.ts

@@ -0,0 +1,28 @@
+import { Logger } from "sync-client";
+
+export class PrefixedLogger extends Logger {
+    private readonly base: Logger;
+    private readonly prefix: string;
+
+    public constructor(base: Logger, prefix: string) {
+        super();
+        this.base = base;
+        this.prefix = prefix;
+    }
+
+    public override debug(message: string): void {
+        this.base.debug(`[${this.prefix}] ${message}`);
+    }
+
+    public override info(message: string): void {
+        this.base.info(`[${this.prefix}] ${message}`);
+    }
+
+    public override warn(message: string): void {
+        this.base.warn(`[${this.prefix}] ${message}`);
+    }
+
+    public override error(message: string): void {
+        this.base.error(`[${this.prefix}] ${message}`);
+    }
+}

frontend/deterministic-tests/src/run-with-concurrency.ts

@@ -0,0 +1,33 @@
+export async function runWithConcurrency<T, R>(
+    items: T[],
+    concurrency: number,
+    fn: (item: T) => Promise<R>
+): Promise<R[]> {
+    const results: R[] = [];
+    const errors: unknown[] = [];
+    const executing = new Set<Promise<void>>();
+
+    for (let i = 0; i < items.length; i++) {
+        const index = i;
+        const p = fn(items[index])
+            .then((result) => {
+                results[index] = result;
+            })
+            .catch((error: unknown) => {
+                errors.push(error);
+            })
+            .finally(() => executing.delete(p));
+        executing.add(p);
+        if (executing.size >= concurrency) {
+            await Promise.race(executing);
+        }
+    }
+
+    // eslint-disable-next-line no-restricted-properties
+    await Promise.all(executing);
+
+    if (errors.length > 0) {
+        throw errors[0];
+    }
+    return results;
+}

frontend/deterministic-tests/src/server-control.ts

@@ -0,0 +1,296 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { sleep } from "./utils/sleep";
+import { findFreePort } from "./utils/find-free-port";
+import type { Logger } from "sync-client";
+import {
+    STOP_TIMEOUT_MS,
+    SERVER_READY_POLL_INTERVAL_MS,
+    SERVER_READY_MAX_ATTEMPTS,
+    SERVER_START_MAX_ATTEMPTS
+} from "./consts";
+
+export class ServerControl {
+    private process: ChildProcess | null = null;
+    private readonly serverPath: string;
+    private readonly baseConfigPath: string;
+    private readonly logger: Logger;
+    private _port: number | undefined;
+    private tempDir: string | undefined;
+    private _isPaused = false;
+
+    public constructor(serverPath: string, configPath: string, logger: Logger) {
+        this.serverPath = serverPath;
+        this.baseConfigPath = configPath;
+        this.logger = logger;
+    }
+
+    public get port(): number {
+        if (this._port === undefined) {
+            throw new Error("Server has not been started yet");
+        }
+        return this._port;
+    }
+
+    public get remoteUri(): string {
+        return `http://localhost:${this.port}`;
+    }
+
+    public async start(): Promise<void> {
+        if (this.process !== null) {
+            throw new Error("Server is already running");
+        }
+
+        // Retry on bind failure: findFreePort closes its probe before we
+        // spawn, so under heavy parallelism another process can grab the
+        // same port. Each attempt picks a fresh port.
+        let lastError: unknown;
+        for (let attempt = 1; attempt <= SERVER_START_MAX_ATTEMPTS; attempt++) {
+            try {
+                await this.startOnce();
+                return;
+            } catch (error) {
+                lastError = error;
+                this.logger.warn(
+                    `Server start attempt ${attempt}/${SERVER_START_MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : String(error)}`
+                );
+                // startOnce already cleaned up its child + tempdir on failure.
+            }
+        }
+        throw new Error(
+            `Server failed to start after ${SERVER_START_MAX_ATTEMPTS} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`,
+            { cause: lastError instanceof Error ? lastError : undefined }
+        );
+    }
+
+    private async startOnce(): Promise<void> {
+        const reservation = await findFreePort();
+        this._port = reservation.port;
+        const tmpBase = os.tmpdir();
+        this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-"));
+        const tempConfigPath = path.join(this.tempDir, "config.yml");
+        const dbDir = path.join(this.tempDir, "databases");
+
+        this.writeConfigFile(tempConfigPath, dbDir);
+
+        this.logger.info(
+            `Starting server: ${this.serverPath} (port ${this._port})`
+        );
+
+        // Release the port reservation right before spawning to minimize
+        // the TOCTOU window between port discovery and server binding.
+        reservation.release();
+
+        this.process = spawn(this.serverPath, [tempConfigPath], {
+            stdio: ["ignore", "pipe", "pipe"],
+            detached: false
+        });
+
+        this.process.stdout?.on("data", (data: Buffer) => {
+            this.logger.info(`[SERVER] ${data.toString().trim()}`);
+        });
+
+        this.process.stderr?.on("data", (data: Buffer) => {
+            this.logger.info(`[SERVER] ${data.toString().trim()}`);
+        });
+
+        this.process.on("error", (err) => {
+            this.logger.error(`[SERVER] Process error: ${err.message}`);
+        });
+
+        const currentProcess = this.process;
+        currentProcess.on("exit", (code, signal) => {
+            this.logger.info(
+                `Server exited with code ${code}, signal ${signal}`
+            );
+            // Only clear state if this handler is for the current process.
+            // A fast stop→start cycle could create a new process before this
+            // handler fires — clearing state here would corrupt the new one.
+            if (this.process === currentProcess) {
+                this.process = null;
+                this._isPaused = false;
+            }
+        });
+
+        try {
+            await this.waitForReady();
+        } catch (error) {
+            // Kill the spawned process if it failed to become ready,
+            // preventing a zombie process from lingering.
+            try {
+                await this.stop();
+            } catch {
+                // Best-effort cleanup
+            }
+            throw error;
+        }
+    }
+
+    public async waitForReady(
+        maxAttempts: number = SERVER_READY_MAX_ATTEMPTS
+    ): Promise<void> {
+        const pingUrl = `${this.remoteUri}/vaults/test/ping`;
+        for (let i = 0; i < maxAttempts; i++) {
+            if (this.process?.exitCode !== null) {
+                throw new Error(
+                    "Server process died while waiting for it to become ready"
+                );
+            }
+            try {
+                const response = await fetch(pingUrl);
+                if (response.ok) {
+                    this.logger.info("[SERVER] Ready");
+                    return;
+                }
+            } catch {
+                // Server not ready yet, continue polling
+            }
+            await sleep(SERVER_READY_POLL_INTERVAL_MS);
+        }
+        throw new Error("Server failed to start within timeout");
+    }
+
+    public pause(): void {
+        if (this.process?.pid === undefined) {
+            throw new Error("Server is not running");
+        }
+        if (this._isPaused) {
+            this.logger.warn("Server is already paused, skipping double-pause");
+            return;
+        }
+        this.logger.info("Server pausing...");
+        try {
+            process.kill(this.process.pid, "SIGSTOP");
+            this._isPaused = true;
+            this.logger.info("Server paused (SIGSTOP sent)");
+        } catch (error) {
+            throw new Error(
+                `Failed to pause server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
+            );
+        }
+    }
+
+    public resume(): void {
+        if (this.process?.pid === undefined) {
+            throw new Error("Server is not running");
+        }
+        if (!this._isPaused) {
+            return;
+        }
+        this.logger.info("Server resuming...");
+        try {
+            process.kill(this.process.pid, "SIGCONT");
+            this._isPaused = false;
+            this.logger.info("Server resumed (SIGCONT sent)");
+        } catch (error) {
+            throw new Error(
+                `Failed to resume server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
+            );
+        }
+    }
+
+    public async stop(): Promise<void> {
+        const proc = this.process;
+        if (proc?.pid === undefined) {
+            this.cleanupTempDir();
+            return;
+        }
+
+        // Resume if paused — a SIGSTOP'd process ignores SIGKILL
+        if (this._isPaused) {
+            try {
+                process.kill(proc.pid, "SIGCONT");
+            } catch {
+                // Process may already be gone
+            }
+            this._isPaused = false;
+        }
+
+        this.logger.info("Server stopping...");
+
+        // Set up a promise that resolves when the process actually exits.
+        const exitPromise = new Promise<void>((resolve) => {
+            if (proc.exitCode !== null) {
+                resolve();
+                return;
+            }
+            proc.on("exit", () => {
+                resolve();
+            });
+        });
+
+        try {
+            process.kill(proc.pid, "SIGKILL");
+        } catch {
+            // Process already gone
+        }
+
+        // Wait for the process to actually exit before cleaning up,
+        // with a 5s safety timeout to avoid hanging forever.
+        await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]);
+
+        this.process = null;
+        this._isPaused = false;
+        this.cleanupTempDir();
+    }
+
+    public isRunning(): boolean {
+        const proc = this.process;
+        return (
+            proc !== null &&
+            proc.pid !== undefined &&
+            proc.exitCode === null &&
+            proc.signalCode === null
+        );
+    }
+
+    /**
+     * Synchronously SIGCONT-then-SIGKILL the child process. Safe to call
+     * from a `process.on("exit", ...)` handler, where async work cannot
+     * run. Used as a last-resort cleanup so a SIGSTOP'd server doesn't
+     * outlive the test runner and wedge the next CI invocation.
+     */
+    public forceKillSync(): void {
+        const proc = this.process;
+        if (proc?.pid === undefined) {
+            return;
+        }
+        try {
+            process.kill(proc.pid, "SIGCONT");
+        } catch {
+            // Process may already be gone or never paused.
+        }
+        try {
+            process.kill(proc.pid, "SIGKILL");
+        } catch {
+            // Process already gone.
+        }
+    }
+
+    private writeConfigFile(destPath: string, dbDir: string): void {
+        // Assumes config-e2e.yml has exactly one 2-space-indented `port:` and
+        // one `databases_directory_path:` (under `server:` and `database:`
+        // respectively)
+        const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8");
+        const config = baseConfig
+            .replace(/^\s*port:\s*\d+/m, `  port: ${this._port}`)
+            .replace(
+                /^\s*databases_directory_path:\s*.+/m,
+                `  databases_directory_path: ${dbDir}`
+            );
+        fs.writeFileSync(destPath, config);
+    }
+
+    private cleanupTempDir(): void {
+        if (this.tempDir !== undefined) {
+            try {
+                fs.rmSync(this.tempDir, { recursive: true, force: true });
+            } catch {
+                // Best-effort cleanup
+            }
+            this.tempDir = undefined;
+        }
+    }
+}

frontend/deterministic-tests/src/server-manager.ts

@@ -0,0 +1,71 @@
+import type { ServerControl } from "./server-control";
+import type { Logger } from "sync-client";
+
+export class ServerManager {
+    private readonly activeServers = new Set<ServerControl>();
+    private readonly logger: Logger;
+    private isShuttingDown = false;
+
+    public constructor(logger: Logger) {
+        this.logger = logger;
+    }
+
+    public track(server: ServerControl): void {
+        this.activeServers.add(server);
+    }
+
+    public untrack(server: ServerControl): void {
+        this.activeServers.delete(server);
+    }
+
+    public async stopAll(): Promise<void> {
+        if (this.isShuttingDown) {
+            return;
+        }
+        this.isShuttingDown = true;
+
+        const servers = Array.from(this.activeServers);
+        // eslint-disable-next-line no-restricted-properties
+        await Promise.all(
+            servers.map(async (server) => {
+                try {
+                    await server.stop();
+                } catch {
+                    // Best-effort cleanup during shutdown
+                }
+            })
+        );
+    }
+
+    public installSignalHandlers(): void {
+        process.on("SIGINT", () => {
+            this.logger.info("Received SIGINT, shutting down...");
+            void this.stopAll()
+                .catch(() => {
+                    /* no-op */
+                })
+                .then(() => process.exit(130));
+        });
+
+        process.on("SIGTERM", () => {
+            this.logger.info("Received SIGTERM, shutting down...");
+            void this.stopAll()
+                .catch(() => {
+                    /* no-op */
+                })
+                .then(() => process.exit(143));
+        });
+
+        // Last-resort synchronous cleanup. Runs even when the process is
+        // exiting via process.exit() from unhandledRejection /
+        // uncaughtException — paths where async stopAll() cannot complete.
+        // SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the
+        // kernel keeps them as zombies holding the test's tmpdir, and the
+        // next CI run can't reuse the port.
+        process.on("exit", () => {
+            for (const server of this.activeServers) {
+                server.forceKillSync();
+            }
+        });
+    }
+}

frontend/deterministic-tests/src/test-definition.ts

@@ -0,0 +1,49 @@
+import type { AssertableState } from "./utils/assertable-state";
+
+export interface ClientState {
+    files: Map<string, string>;
+    clientFiles: Map<string, string>[];
+}
+
+export type TestStep =
+    | { type: "create"; client: number; path: string; content: string }
+    | { type: "update"; client: number; path: string; content: string }
+    | { type: "rename"; client: number; oldPath: string; newPath: string }
+    | {
+          type: "rename-next-write";
+          client: number;
+          oldPath: string;
+          newPath: string;
+      }
+    | { type: "delete"; client: number; path: string }
+    | { type: "sync"; client?: number }
+    | { type: "disable-sync"; client: number }
+    | { type: "enable-sync"; client: number }
+    | { type: "pause-server" }
+    | { type: "resume-server" }
+    | {
+          type: "resume-server-until-history-then-pause";
+          client: number;
+          syncType: "CREATE" | "UPDATE" | "DELETE";
+          path: string;
+      }
+    | { type: "barrier" }
+    | { type: "assert-consistent"; verify?: (state: AssertableState) => void }
+    | { type: "pause-websocket"; client: number }
+    | { type: "resume-websocket"; client: number }
+    | { type: "drop-next-create-response"; client: number }
+    | { type: "wait-for-dropped-create-response"; client: number }
+    | { type: "sleep"; ms: number }
+    | { type: "reset"; client: number };
+
+export interface TestDefinition {
+    description?: string;
+    clients: number;
+    steps: TestStep[];
+}
+
+export interface TestResult {
+    success: boolean;
+    error?: string;
+    duration?: number;
+}

frontend/deterministic-tests/src/test-registry.ts

@@ -0,0 +1,245 @@
+import type { TestDefinition } from "./test-definition";
+import { renameCreateConflictTest } from "./tests/rename-create-conflict.test";
+import { renameChainTest } from "./tests/rename-chain.test";
+import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test";
+import { deleteRenameConflictTest } from "./tests/delete-rename-conflict.test";
+import { multiFileOperationsTest } from "./tests/multi-file-operations.test";
+import { deleteRecreateSamePathTest } from "./tests/delete-recreate-same-path.test";
+import { offlineRenameAndEditTest } from "./tests/offline-rename-and-edit.test";
+import { simultaneousCreateDeleteSamePathTest } from "./tests/simultaneous-create-delete-same-path.test";
+import { idempotencyAfterServerPauseTest } from "./tests/idempotency-after-server-pause.test";
+import { sequentialCreateDuplicateContentTest } from "./tests/sequential-create-duplicate-content.test";
+import { mcThreeClientRenameOfflineUpdateTest } from "./tests/mc-three-client-rename-offline-update.test";
+import { mcMultiDeleteOfflineRenameTest } from "./tests/mc-multi-delete-offline-rename.test";
+import { mcCrossCreateRenameSameTargetTest } from "./tests/mc-cross-create-rename-same-target.test";
+import { mcDeleteThenOfflineRenameTest } from "./tests/mc-delete-then-offline-rename.test";
+import { offlineMixedOperationsTest } from "./tests/offline-mixed-operations.test";
+import { offlineConcurrentRenamesTest } from "./tests/offline-concurrent-renames.test";
+import { offlineMultipleEditsTest } from "./tests/offline-multiple-edits.test";
+import { serverPauseBothClientsCreateTest } from "./tests/server-pause-both-clients-create.test";
+import { serverPauseUpdateAndCreateTest } from "./tests/server-pause-update-and-create.test";
+import { renameSwapTest } from "./tests/rename-swap.test";
+import { renameCircularTest } from "./tests/rename-circular.test";
+import { renameRoundtripTest } from "./tests/rename-roundtrip.test";
+import { offlineRenameRemoteCreateOldPathTest } from "./tests/offline-rename-remote-create-old-path.test";
+import { offlineEditRemoteRenameTest } from "./tests/offline-edit-remote-rename.test";
+import { renameChainThenDeleteTest } from "./tests/rename-chain-then-delete.test";
+import { offlineDeleteRemoteRenameTest } from "./tests/offline-delete-remote-rename.test";
+import { overlappingEditsSameSectionTest } from "./tests/overlapping-edits-same-section.test";
+import { rapidUpdatesAfterMergeTest } from "./tests/rapid-updates-after-merge.test";
+import { deleteRecreateConcurrentUpdateTest } from "./tests/delete-recreate-concurrent-update.test";
+import { moveAndConcurrentRemoteUpdateTest } from "./tests/move-and-concurrent-remote-update.test";
+import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remote-update.test";
+import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test";
+import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test";
+import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test";
+import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-mergeable.test";
+import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test";
+import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test";
+import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test";
+import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test";
+import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test";
+import { serverPauseBothEditSameFileTest } from "./tests/server-pause-both-edit-same-file.test";
+import { deleteRecreateDifferentContentTest } from "./tests/delete-recreate-different-content.test";
+import { updateDuringCreateProcessingTest } from "./tests/update-during-create-processing.test";
+import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remote-delete.test";
+import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test";
+import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test";
+import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test";
+import { updateDoesNotSurviveRemoteDeleteTest } from "./tests/update-does-not-survive-remote-delete.test";
+import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test";
+import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test";
+import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test";
+import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test";
+import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test";
+import { renameToPendingPathFallbackTest } from "./tests/rename-to-pending-path-fallback.test";
+import { moveRemoteUpdateRevertsRenameTest } from "./tests/move-remote-update-reverts-rename.test";
+import { localEditLostDuringCreateMergeTest } from "./tests/local-edit-lost-during-create-merge.test";
+import { renamePendingCreateBeforeResponseTest } from "./tests/rename-pending-create-before-response.test";
+import { createRenameResponseSkipsFileTest } from "./tests/create-rename-response-skips-file.test";
+import { onlineCreateRenameConcurrentCreateOrphanTest } from "./tests/online-create-rename-concurrent-create-orphan.test";
+import { concurrentRenameFirstWinsTest } from "./tests/concurrent-rename-first-wins.test";
+import { binaryToTextTransitionTest } from "./tests/binary-to-text-transition.test";
+import { textPendingCreateNotDisplacedTest } from "./tests/text-pending-create-not-displaced.test";
+import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test";
+import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test";
+import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test";
+import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test";
+import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test";
+import { concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/concurrent-rename-and-create-at-target-rename-first.test";
+import { concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/concurrent-rename-and-create-at-target-create-first.test";
+import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test";
+import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test";
+import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test";
+import { createDeleteNoopTest } from "./tests/create-delete-noop.test";
+import { createMergeDeleteTest } from "./tests/create-merge-delete.test";
+import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test";
+import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test";
+import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test";
+import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test";
+import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test";
+import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test";
+import { deleteByOtherClientThenRecreateTest } from "./tests/delete-by-other-client-then-recreate.test";
+import { onlineDeleteRecreateRapidCycleTest } from "./tests/online-delete-recreate-rapid-cycle.test";
+import { onlineEditVsDeleteConvergenceTest } from "./tests/online-edit-vs-delete-convergence.test";
+import { rapidEditDeleteOnlineConvergenceTest } from "./tests/rapid-edit-delete-online-convergence.test";
+import { serverPauseDeleteRecreateTest } from "./tests/server-pause-delete-recreate.test";
+import { onlineBothCreateSamePathDeconflictTest } from "./tests/online-both-create-same-path-deconflict.test";
+import { onlineCreateUpdateWhileOtherCreatesSamePathTest } from "./tests/online-create-update-while-other-creates-same-path.test";
+import { displacedFileNotMarkedDeletedTest } from "./tests/displaced-file-not-marked-deleted.test";
+import { remoteUpdateResurrectsDeletedDocTest } from "./tests/remote-update-resurrects-deleted-doc.test";
+import { localUpdateSurvivesRemoteRenameTest } from "./tests/local-update-survives-remote-rename.test";
+import { mergingUpdateResponseSurvivesUserRenameTest } from "./tests/merging-update-response-survives-user-rename.test";
+import { catchupCreateAndUpdateNotSkippedTest } from "./tests/catchup-create-and-update-not-skipped.test";
+import { localRenameSurvivesRemoteRenameTest } from "./tests/local-rename-survives-remote-rename.test";
+import { renameChainDuringPendingCreateTest } from "./tests/rename-chain-during-pending-create.test";
+import { remoteRenameCollidesWithPendingLocalCreateTest } from "./tests/remote-rename-collides-with-pending-local-create.test";
+import { remoteUpdateSurvivesUserRenameTest } from "./tests/remote-update-survives-user-rename.test";
+import { sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest } from "./tests/same-doc-id-collapse-on-local-create-after-remote-create.test";
+import { sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest } from "./tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test";
+import { renameOverwritesPendingCreateThenDeleteTest } from "./tests/rename-overwrites-pending-create-then-delete.test";
+import { deleteRecreatedPendingCreateWithStaleDeletingRecordTest } from "./tests/delete-recreated-pending-create-with-stale-deleting-record.test";
+import { queuedCreateDeleteDoesNotHijackReusedPathTest } from "./tests/queued-create-delete-does-not-hijack-reused-path.test";
+import { renamedPendingCreateReusedPathThenDeleteTest } from "./tests/renamed-pending-create-reused-path-then-delete.test";
+import { renamePendingCreateOntoPendingDeletePathTest } from "./tests/rename-pending-create-onto-pending-delete-path.test";
+import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-write-rename-before-record.test";
+import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test";
+
+export const TESTS: Partial<Record<string, TestDefinition>> = {
+    "rename-create-conflict": renameCreateConflictTest,
+    "rename-chain": renameChainTest,
+    "rename-update-conflict": renameUpdateConflictTest,
+    "delete-rename-conflict": deleteRenameConflictTest,
+    "multi-file-operations": multiFileOperationsTest,
+    "delete-recreate-same-path": deleteRecreateSamePathTest,
+    "offline-rename-and-edit": offlineRenameAndEditTest,
+    "simultaneous-create-delete-same-path":
+        simultaneousCreateDeleteSamePathTest,
+    "idempotency-after-server-pause": idempotencyAfterServerPauseTest,
+    "sequential-create-duplicate-content": sequentialCreateDuplicateContentTest,
+    "mc-three-client-rename-offline-update":
+        mcThreeClientRenameOfflineUpdateTest,
+    "mc-multi-delete-offline-rename": mcMultiDeleteOfflineRenameTest,
+    "mc-cross-create-rename-same-target": mcCrossCreateRenameSameTargetTest,
+    "mc-delete-then-offline-rename": mcDeleteThenOfflineRenameTest,
+    "offline-mixed-operations": offlineMixedOperationsTest,
+    "offline-concurrent-renames": offlineConcurrentRenamesTest,
+    "offline-multiple-edits": offlineMultipleEditsTest,
+    "server-pause-both-clients-create": serverPauseBothClientsCreateTest,
+    "server-pause-update-and-create": serverPauseUpdateAndCreateTest,
+    "rename-swap": renameSwapTest,
+    "rename-circular": renameCircularTest,
+    "rename-roundtrip": renameRoundtripTest,
+    "offline-rename-remote-create-old-path":
+        offlineRenameRemoteCreateOldPathTest,
+    "offline-edit-remote-rename": offlineEditRemoteRenameTest,
+    "rename-chain-then-delete": renameChainThenDeleteTest,
+    "offline-delete-remote-rename": offlineDeleteRemoteRenameTest,
+    "overlapping-edits-same-section": overlappingEditsSameSectionTest,
+    "rapid-updates-after-merge": rapidUpdatesAfterMergeTest,
+    "delete-recreate-concurrent-update": deleteRecreateConcurrentUpdateTest,
+    "move-and-concurrent-remote-update": moveAndConcurrentRemoteUpdateTest,
+    "double-offline-cycle": doubleOfflineCycleTest,
+    "server-pause-rename-edit-resume": serverPauseRenameEditResumeTest,
+    "offline-update-both-then-delete-one": offlineUpdateBothThenDeleteOneTest,
+    "offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest,
+    "delete-during-pending-create": deleteDuringPendingCreateTest,
+    "three-client-rename-create-delete": threeClientRenameCreateDeleteTest,
+    "rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest,
+    "offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest,
+    "rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest,
+    "server-pause-both-edit-same-file": serverPauseBothEditSameFileTest,
+    "delete-recreate-different-content": deleteRecreateDifferentContentTest,
+    "update-during-create-processing": updateDuringCreateProcessingTest,
+    "offline-move-then-remote-delete": offlineMoveThenRemoteDeleteTest,
+    "reset-clears-recently-deleted-resurrection":
+        resetClearsRecentlyDeletedResurrectionTest,
+    "move-then-delete-stale-path": moveThenDeleteStalePathTest,
+    "offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest,
+    "interrupted-delete-retry": interruptedDeleteRetryTest,
+    "update-does-not-survive-remote-delete": updateDoesNotSurviveRemoteDeleteTest,
+    "move-preserves-remote-update": movePreservesRemoteUpdateTest,
+    "recently-deleted-cleared-on-reconnect":
+        recentlyDeletedClearedOnReconnectTest,
+    "watermark-advances-on-skip": watermarkAdvancesOnSkipTest,
+    "watermark-gap-remote-update-not-recorded":
+        watermarkGapRemoteUpdateNotRecordedTest,
+    "queue-reset-loses-coalesced-local-edit":
+        queueResetLosesCoalescedLocalEditTest,
+    "rename-to-pending-path-fallback": renameToPendingPathFallbackTest,
+    "move-remote-update-reverts-rename": moveRemoteUpdateRevertsRenameTest,
+    "local-edit-lost-during-create-merge": localEditLostDuringCreateMergeTest,
+    "rename-pending-create-before-response":
+        renamePendingCreateBeforeResponseTest,
+    "create-rename-response-skips-file": createRenameResponseSkipsFileTest,
+    "online-create-rename-concurrent-create-orphan":
+        onlineCreateRenameConcurrentCreateOrphanTest,
+    "concurrent-rename-first-wins": concurrentRenameFirstWinsTest,
+    "binary-to-text-transition": binaryToTextTransitionTest,
+    "text-pending-create-not-displaced": textPendingCreateNotDisplacedTest,
+    "binary-pending-create-not-displaced": binaryPendingCreateNotDisplacedTest,
+    "coalesce-update-remote-update-data-loss":
+        coalesceUpdateRemoteUpdateDataLossTest,
+    "coalesced-remote-update-watermark-loss":
+        coalescedRemoteUpdateWatermarkLossTest,
+    "concurrent-delete-during-remote-update":
+        concurrentDeleteDuringRemoteUpdateTest,
+    "concurrent-edit-exact-same-position": concurrentEditExactSamePositionTest,
+    "concurrent-rename-and-create-at-target-rename-first":
+        concurrentRenameAndCreateAtTargetRenameFirstTest,
+    "concurrent-rename-and-create-at-target-create-first":
+        concurrentRenameAndCreateAtTargetCreateFirstTest,
+    "concurrent-rename-same-target": concurrentRenameSameTargetTest,
+    "concurrent-update-diff-consistency": concurrentUpdateDiffConsistencyTest,
+    "user-parenthesized-file-not-deleted": userParenthesizedFileNotDeletedTest,
+    "create-delete-noop": createDeleteNoopTest,
+    "create-merge-delete": createMergeDeleteTest,
+    "move-identical-content-ambiguity": moveIdenticalContentAmbiguityTest,
+    "create-update-coalesce-server-pause": createUpdateCoalesceServerPauseTest,
+    "create-during-reconciliation": createDuringReconciliationTest,
+    "create-merge-preserves-renamed-update":
+        createMergePreservesRenamedUpdateTest,
+    "create-rename-create-same-path": createRenameCreateSamePathTest,
+    "move-chain-three-files": moveChainThreeFilesTest,
+    "delete-by-other-client-then-recreate": deleteByOtherClientThenRecreateTest,
+    "online-delete-recreate-rapid-cycle": onlineDeleteRecreateRapidCycleTest,
+    "online-edit-vs-delete-convergence": onlineEditVsDeleteConvergenceTest,
+    "rapid-edit-delete-online-convergence":
+        rapidEditDeleteOnlineConvergenceTest,
+    "server-pause-delete-recreate": serverPauseDeleteRecreateTest,
+    "online-both-create-same-path-deconflict":
+        onlineBothCreateSamePathDeconflictTest,
+    "online-create-update-while-other-creates-same-path":
+        onlineCreateUpdateWhileOtherCreatesSamePathTest,
+    "displaced-file-not-marked-deleted": displacedFileNotMarkedDeletedTest,
+    "remote-update-resurrects-deleted-doc":
+        remoteUpdateResurrectsDeletedDocTest,
+    "local-update-survives-remote-rename": localUpdateSurvivesRemoteRenameTest,
+    "merging-update-response-survives-user-rename":
+        mergingUpdateResponseSurvivesUserRenameTest,
+    "catchup-create-and-update-not-skipped":
+        catchupCreateAndUpdateNotSkippedTest,
+    "local-rename-survives-remote-rename": localRenameSurvivesRemoteRenameTest,
+    "rename-chain-during-pending-create": renameChainDuringPendingCreateTest,
+    "remote-rename-collides-with-pending-local-create":
+        remoteRenameCollidesWithPendingLocalCreateTest,
+    "remote-update-survives-user-rename": remoteUpdateSurvivesUserRenameTest,
+    "same-doc-id-collapse-on-local-create-after-remote-create":
+        sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest,
+    "renamed-pending-create-reused-path-then-delete":
+        renamedPendingCreateReusedPathThenDeleteTest,
+    "rename-pending-create-onto-pending-delete-path":
+        renamePendingCreateOntoPendingDeletePathTest,
+    "rename-overwrites-pending-create-then-delete":
+        renameOverwritesPendingCreateThenDeleteTest,
+    "same-doc-id-collapse-after-remote-quick-write-and-pending-rename":
+        sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest,
+    "delete-recreated-pending-create-with-stale-deleting-record":
+        deleteRecreatedPendingCreateWithStaleDeletingRecordTest,
+    "queued-create-delete-does-not-hijack-reused-path":
+        queuedCreateDeleteDoesNotHijackReusedPathTest,
+    "remote-quick-write-rename-before-record":
+        remoteQuickWriteRenameBeforeRecordTest,
+    "self-merge-pending-rename-aliases-second-create":
+        selfMergePendingRenameAliasesSecondCreateTest
+};

frontend/deterministic-tests/src/test-runner.ts

@@ -0,0 +1,399 @@
+import type { TestDefinition, TestResult, TestStep } from "./test-definition";
+import { DeterministicAgent } from "./deterministic-agent";
+import type { ServerControl } from "./server-control";
+import type { SyncSettings, Logger } from "sync-client";
+import { assert } from "./utils/assert";
+import { AssertableState } from "./utils/assertable-state";
+import { sleep } from "./utils/sleep";
+import { withTimeout } from "./utils/with-timeout";
+import {
+    CONVERGENCE_TIMEOUT_MS,
+    CONVERGENCE_RETRY_DELAY_MS,
+    AGENT_INIT_TIMEOUT_MS,
+    IS_SYNC_ENABLED_BY_DEFAULT
+} from "./consts";
+import { randomUUID } from "node:crypto";
+
+export class TestRunner {
+    private agents: DeterministicAgent[] = [];
+    private readonly serverControl: ServerControl;
+    private readonly token: string;
+    private readonly remoteUri: string;
+    private readonly logger: Logger;
+
+    public constructor(
+        serverControl: ServerControl,
+        logger: Logger,
+        token: string,
+        remoteUri: string
+    ) {
+        this.serverControl = serverControl;
+        this.logger = logger;
+        this.token = token;
+        this.remoteUri = remoteUri;
+    }
+
+    public async runTest(
+        name: string,
+        test: TestDefinition
+    ): Promise<TestResult> {
+        const startTime = Date.now();
+        this.logger.info(`Running test: ${name}`);
+        if (test.description !== undefined && test.description !== "") {
+            this.logger.info(`Description: ${test.description}`);
+        }
+        this.logger.info(`Clients: ${test.clients}`);
+        this.logger.info(`Steps: ${test.steps.length}`);
+
+        try {
+            assert(
+                this.serverControl.isRunning(),
+                "Server is not running before test start"
+            );
+
+            await this.initializeAgents(test.clients);
+
+            for (let i = 0; i < test.steps.length; i++) {
+                const step = test.steps[i];
+                this.logger.info(
+                    `Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
+                );
+                await this.executeStep(step);
+            }
+
+            await this.cleanup();
+
+            const duration = Date.now() - startTime;
+            this.logger.info(`\n✓ Test passed: ${name} (${duration}ms)`);
+
+            return {
+                success: true,
+                duration
+            };
+        } catch (error) {
+            const duration = Date.now() - startTime;
+            const errorMessage =
+                error instanceof Error ? error.message : String(error);
+            this.logger.info(`\n✗ Test failed: ${name}`);
+            this.logger.info(`Error: ${errorMessage}`);
+
+            await this.cleanup();
+
+            return {
+                success: false,
+                error: errorMessage,
+                duration
+            };
+        }
+    }
+
+    private async initializeAgents(count: number): Promise<void> {
+        assert(count > 0, `Client count must be positive, got ${count}`);
+        const vaultName = `test-${randomUUID()}`;
+        this.logger.info(
+            `Initializing ${count} agents with vault: ${vaultName}`
+        );
+
+        for (let i = 0; i < count; i++) {
+            const settings: Partial<SyncSettings> = {
+                isSyncEnabled: IS_SYNC_ENABLED_BY_DEFAULT,
+                token: this.token,
+                vaultName,
+                remoteUri: this.remoteUri
+            };
+
+            const agent = new DeterministicAgent(i, settings, (msg) => {
+                this.logger.info(msg);
+            });
+
+            // Push before init so cleanup() handles this agent if init fails
+            this.agents.push(agent);
+            await withTimeout(
+                agent.init(fetch),
+                AGENT_INIT_TIMEOUT_MS,
+                `Client ${i} init timed out after ${AGENT_INIT_TIMEOUT_MS}ms`
+            );
+            this.logger.info(`Initialized client ${i}`);
+        }
+
+        this.logger.info("All agents initialized");
+    }
+
+    private getAgent(index: number): DeterministicAgent {
+        assert(
+            index >= 0 && index < this.agents.length,
+            `Client index ${index} out of bounds (have ${this.agents.length} agents)`
+        );
+        return this.agents[index];
+    }
+
+    private async executeStep(step: TestStep): Promise<void> {
+        switch (step.type) {
+            case "create":
+            case "update":
+                await this.getAgent(step.client).write(
+                    step.path,
+                    new TextEncoder().encode(step.content)
+                );
+                break;
+
+            case "rename":
+                await this.getAgent(step.client).rename(
+                    step.oldPath,
+                    step.newPath
+                );
+                break;
+
+            case "rename-next-write":
+                this.getAgent(step.client).renameNextWrite(
+                    step.oldPath,
+                    step.newPath
+                );
+                break;
+
+            case "delete":
+                await this.getAgent(step.client).delete(step.path);
+                break;
+
+            case "sync":
+                if (step.client !== undefined) {
+                    await this.getAgent(step.client).waitForSync();
+                } else {
+                    for (const agent of this.agents) {
+                        await agent.waitForSync();
+                    }
+                }
+                break;
+
+            case "disable-sync":
+                await this.getAgent(step.client).disableSync();
+                break;
+
+            case "enable-sync":
+                await this.getAgent(step.client).enableSync();
+                break;
+
+            case "pause-server":
+                this.serverControl.pause();
+                break;
+
+            case "resume-server":
+                this.serverControl.resume();
+                // Verify the server is actually responsive before proceeding.
+                // This replaces relying solely on hardcoded waits.
+                await this.serverControl.waitForReady();
+                break;
+
+            case "resume-server-until-history-then-pause": {
+                const agent = this.getAgent(step.client);
+                const historySeen = agent.waitForHistoryEntry(
+                    (entry) =>
+                        entry.details.type === step.syncType &&
+                        entry.details.relativePath === step.path,
+                    () => this.serverControl.pause()
+                );
+                this.serverControl.resume();
+                await historySeen;
+                break;
+            }
+
+            case "barrier":
+                await this.waitForConvergence();
+                break;
+
+            case "assert-consistent":
+                await this.assertConsistent(step.verify);
+                break;
+
+            case "pause-websocket":
+                this.getAgent(step.client).pauseWebSocket();
+                break;
+
+            case "resume-websocket":
+                this.getAgent(step.client).resumeWebSocket();
+                break;
+
+            case "drop-next-create-response":
+                this.getAgent(step.client).dropNextCreateResponse();
+                break;
+
+            case "wait-for-dropped-create-response":
+                await this.getAgent(step.client).waitForDroppedCreateResponse();
+                break;
+
+            case "sleep":
+                await sleep(step.ms);
+                break;
+
+            case "reset":
+                await this.getAgent(step.client).reset();
+                break;
+
+            default: {
+                const unknownStep = step as { type: string };
+                throw new Error(`Unknown step type: ${unknownStep.type}`);
+            }
+        }
+    }
+
+    /**
+     * Wait for all agents to reach a consistent state.
+     *
+     * Waiting for agents is done in two full rounds: the first round
+     * drains in-flight operations, but completing those operations can
+     * trigger new work on OTHER agents via server broadcasts. The second
+     * round waits for that cascading work to settle. Deeper cascades
+     * are handled by the outer retry loop.
+     */
+    private async waitForConvergence(): Promise<void> {
+        this.logger.info("Barrier: waiting for convergence...");
+
+        const deadline = Date.now() + CONVERGENCE_TIMEOUT_MS;
+        let lastError: Error | undefined = undefined;
+
+        while (Date.now() < deadline) {
+            await this.waitAllAgentsSettled();
+
+            try {
+                await this.assertConsistent();
+                this.logger.info("Barrier complete: all clients converged");
+                return;
+            } catch (error) {
+                lastError =
+                    error instanceof Error ? error : new Error(String(error));
+                this.logger.info("Barrier: not yet converged, retrying...");
+                await sleep(CONVERGENCE_RETRY_DELAY_MS);
+            }
+        }
+
+        throw new Error(
+            `Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${lastError?.message ?? "no consistency check ran"}`,
+            { cause: lastError }
+        );
+    }
+
+    /**
+     * Wait for all agents to be simultaneously idle.
+     *
+     * Completing work on agent A can trigger a server broadcast that
+     * enqueues new work on agent B, which can cascade further. With N
+     * agents the worst-case cascade depth is N (a chain A→B→C→…→A),
+     * so we run N+1 sequential passes to drain it. Extra passes are
+     * essentially free when there is no outstanding work.
+     *
+     * The outer {@link waitForConvergence} loop with consistency checks
+     * remains the ultimate guarantee — this method just minimizes how
+     * many slow retry iterations are needed.
+     */
+    private async waitAllAgentsSettled(): Promise<void> {
+        const rounds = this.agents.length + 1;
+        for (let round = 0; round < rounds; round++) {
+            for (const agent of this.agents) {
+                await agent.waitForSync();
+            }
+        }
+    }
+
+    private async assertConsistent(
+        verify?: (state: AssertableState) => void
+    ): Promise<void> {
+        this.logger.info("Asserting all clients are consistent...");
+        assert(
+            this.agents.length >= 2,
+            "Need at least 2 agents for consistency check"
+        );
+
+        // Snapshot all agents' file states upfront to minimize the window
+        // where background sync could mutate state between reads.
+        const clientFiles: Map<string, string>[] = [];
+        for (const agent of this.agents) {
+            const sortedFiles = (await agent.listFilesRecursively()).sort();
+            const fileMap = new Map<string, string>();
+            for (const file of sortedFiles) {
+                const content = await agent.getFileContent(file);
+                fileMap.set(file, content);
+            }
+            clientFiles.push(fileMap);
+        }
+
+        const referenceFiles = Array.from(clientFiles[0].keys());
+
+        this.logger.info(
+            `Reference client has ${referenceFiles.length} files: ${referenceFiles.join(", ")}`
+        );
+
+        for (let i = 1; i < clientFiles.length; i++) {
+            const agentFileKeys = Array.from(clientFiles[i].keys());
+
+            this.logger.info(
+                `Client ${i} has ${agentFileKeys.length} files: ${agentFileKeys.join(", ")}`
+            );
+
+            assert(
+                agentFileKeys.length === referenceFiles.length,
+                `File count mismatch: client 0 has ${referenceFiles.length} files, client ${i} has ${agentFileKeys.length} files`
+            );
+
+            for (let j = 0; j < agentFileKeys.length; j++) {
+                assert(
+                    agentFileKeys[j] === referenceFiles[j],
+                    `File list mismatch at index ${j}: client 0 has "${referenceFiles[j]}", client ${i} has "${agentFileKeys[j]}"`
+                );
+            }
+
+            for (const file of referenceFiles) {
+                const referenceContent = clientFiles[0].get(file);
+                const agentContent = clientFiles[i].get(file);
+
+                assert(
+                    referenceContent === agentContent,
+                    `Content mismatch for ${file}:\nClient 0: "${referenceContent}"\nClient ${i}: "${agentContent}"`
+                );
+            }
+        }
+
+        this.logger.info("✓ All clients are consistent");
+
+        if (verify) {
+            this.logger.info("Running custom verification...");
+            try {
+                verify(
+                    new AssertableState({
+                        files: clientFiles[0],
+                        clientFiles
+                    })
+                );
+            } catch (error) {
+                const msg =
+                    error instanceof Error ? error.message : String(error);
+                throw new Error(`Custom verification failed: ${msg}`);
+            }
+            this.logger.info("✓ Custom verification passed");
+        }
+    }
+
+    private async cleanup(): Promise<void> {
+        // Always resume the server in case a test paused it and then
+        // failed before reaching the resume step. Without this, all
+        // subsequent tests would hang because the server process is
+        // frozen (SIGSTOP) and can't respond to HTTP or WebSocket.
+        try {
+            this.serverControl.resume();
+        } catch {
+            // Server wasn't paused or isn't running — safe to ignore
+        }
+
+        this.logger.info("\nCleaning up agents...");
+        for (const agent of this.agents) {
+            try {
+                await agent.cleanup();
+            } catch (error) {
+                this.logger.warn(
+                    `Agent cleanup error: ${error instanceof Error ? error.message : String(error)}`
+                );
+            }
+        }
+        this.agents = [];
+        this.logger.info("Cleanup complete");
+    }
+}

frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts

@@ -0,0 +1,40 @@
+import type { AssertableState } from "../utils/assertable-state";
+import type { TestDefinition } from "../test-definition";
+
+export const binaryPendingCreateNotDisplacedTest: TestDefinition = {
+    description:
+        "Two clients each create a binary file at the same path while offline. " +
+        "After syncing, both files should exist on both clients at separate paths.",
+    clients: 2,
+    steps: [
+        {
+            type: "create",
+            client: 0,
+            path: "data.bin",
+            content: "binary data from client 0"
+        },
+        {
+            type: "create",
+            client: 1,
+            path: "data.bin",
+            content: "binary data from client 1"
+        },
+
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertFileCount(2)
+                    .assertFileExists("data.bin")
+                    .assertFileExists("data (1).bin")
+                    .assertAnyFileContains(
+                        "binary data from client 0",
+                        "binary data from client 1"
+                    );
+            }
+        }
+    ]
+};

frontend/deterministic-tests/src/tests/binary-to-text-transition.test.ts

@@ -0,0 +1,97 @@
+import type { AssertableState } from "../utils/assertable-state";
+import type { TestDefinition } from "../test-definition";
+
+export const binaryToTextTransitionTest: TestDefinition = {
+    description:
+        "A .bin file is created and synced. Both clients edit it offline " +
+        "(binary last-write-wins), then client 0 renames it to .md and " +
+        "writes a clean text baseline. Both clients edit different sections " +
+        "offline. The text merge should preserve both edits.",
+    clients: 2,
+    steps: [
+        {
+            type: "create",
+            client: 0,
+            path: "data.bin",
+            content: "original content"
+        },
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertContent("data.bin", "original content");
+            }
+        },
+
+        { type: "disable-sync", client: 0 },
+        { type: "disable-sync", client: 1 },
+
+        { type: "update", client: 0, path: "data.bin", content: "version A" },
+        { type: "update", client: 1, path: "data.bin", content: "version B" },
+
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertFileCount(1).assertContainsAny(
+                    "data.bin",
+                    "version A",
+                    "version B"
+                );
+            }
+        },
+
+        { type: "disable-sync", client: 1 },
+        { type: "rename", client: 0, oldPath: "data.bin", newPath: "data.md" },
+        {
+            type: "update",
+            client: 0,
+            path: "data.md",
+            content: "top line\nmiddle line\nbottom line"
+        },
+        { type: "sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertContent(
+                    "data.md",
+                    "top line\nmiddle line\nbottom line"
+                );
+            }
+        },
+
+        { type: "disable-sync", client: 0 },
+        { type: "disable-sync", client: 1 },
+
+        {
+            type: "update",
+            client: 0,
+            path: "data.md",
+            content: "alpha\nmiddle line\nbottom line"
+        },
+        {
+            type: "update",
+            client: 1,
+            path: "data.md",
+            content: "top line\nmiddle line\nbeta"
+        },
+
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertFileCount(1).assertContains("data.md", "alpha", "beta");
+            }
+        }
+    ]
+};

frontend/deterministic-tests/src/tests/catchup-create-and-update-not-skipped.test.ts

@@ -0,0 +1,66 @@
+import type { AssertableState } from "../utils/assertable-state";
+import type { TestDefinition } from "../test-definition";
+
+export const catchupCreateAndUpdateNotSkippedTest: TestDefinition = {
+    description:
+        "Client 1 disconnects (sync disabled). Client 0 creates a doc and " +
+        "then updates it. When Client 1 reconnects, the server's catch-up " +
+        "stream sends only the doc's *latest* version (the update), not the " +
+        "full history. Pre-fix the wire's `is_new_file` was set to " +
+        "`creation == latest_version`, so the catch-up flagged the doc as " +
+        "non-new even though Client 1 had never seen its creation. Client " +
+        "1's `processRemoteChange` then dropped it as a 'stale RemoteChange " +
+        "for untracked, non-new document' and the doc was silently lost. " +
+        "Post-fix `is_new_file` in the catch-up stream means 'new relative " +
+        "to the recipient's watermark' (`creation > last_seen_vault_update_id`).",
+    clients: 2,
+    steps: [
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        // Establish a baseline so Client 1's last_seen is non-zero before
+        // we take it offline. This makes the bug genuinely about catch-up
+        // missing the create rather than just an empty-vault first sync.
+        { type: "create", client: 0, path: "warmup.md", content: "w\n" },
+        { type: "barrier" },
+
+        // Client 1 goes offline.
+        { type: "disable-sync", client: 1 },
+
+        // Client 0 creates the doc (vault_update_id v_C, after Client 1's
+        // watermark). Client 1 doesn't see this because it's offline.
+        { type: "create", client: 0, path: "doc.md", content: "v1\n" },
+        // Wait for the create's HTTP to land before the update; otherwise
+        // both writes are coalesced into a single POST and the server
+        // never sees the doc as "create followed by update".
+        { type: "sync", client: 0 },
+
+        // Client 0 updates the doc (vault_update_id v_X > v_C). The
+        // server's `latest_document_versions` view now returns the
+        // *update* row — its `creation_vault_update_id != vault_update_id`.
+        {
+            type: "update",
+            client: 0,
+            path: "doc.md",
+            content: "v1\nupdate\n"
+        },
+        { type: "sync", client: 0 },
+
+        // Client 1 reconnects. Server's catch-up replays docs with
+        // `vault_update_id > last_seen`. For doc.md it sends v_X with
+        // `is_new_file` derived from `creation_vault_update_id >
+        // last_seen_vault_update_id` (post-fix) — so Client 1 treats it
+        // as a fresh create and downloads the latest content.
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (state: AssertableState): void => {
+                state.assertFileCount(2);
+                state.assertFileExists("doc.md");
+                state.assertContent("doc.md", "v1\nupdate\n");
+                state.assertContent("warmup.md", "w\n");
+            }
+        }
+    ]
+};

frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts

@@ -0,0 +1,59 @@
+import type { AssertableState } from "../utils/assertable-state";
+import type { TestDefinition } from "../test-definition";
+
+export const coalesceUpdateRemoteUpdateDataLossTest: TestDefinition = {
+    description:
+        "Divergent offline edits with text-merge expectation. Client 0's " +
+        "remote update fully lands before Client 1 reconnects (`sync`-after " +
+        "the c0 update enforces this), so Client 1's offline edit merges " +
+        "against a server-known version, not a coalesced batch. Both " +
+        "additions must survive in the final merged content. (Filename's " +
+        "'coalesce' framing is aspirational — a true update-coalesce test " +
+        "would skip the c0 sync and queue overlapping local + remote " +
+        "updates against the same parent version.)",
+    clients: 2,
+    steps: [
+        {
+            type: "create",
+            client: 0,
+            path: "doc.md",
+            content: "line 1\nline 2\nline 3"
+        },
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        { type: "disable-sync", client: 1 },
+
+        {
+            type: "update",
+            client: 0,
+            path: "doc.md",
+            content: "line 1\nline 2\nline 3\nclient 0 addition"
+        },
+        { type: "sync", client: 0 },
+
+        {
+            type: "update",
+            client: 1,
+            path: "doc.md",
+            content: "client 1 addition\nline 1\nline 2\nline 3"
+        },
+
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (state: AssertableState): void => {
+                state
+                    .assertFileCount(1)
+                    .assertContains(
+                        "doc.md",
+                        "client 0 addition",
+                        "client 1 addition"
+                    );
+            }
+        }
+    ]
+};

frontend/deterministic-tests/src/tests/coalesced-remote-update-watermark-loss.test.ts

@@ -0,0 +1,53 @@
+import type { AssertableState } from "../utils/assertable-state";
+import type { TestDefinition } from "../test-definition";
+
+export const coalescedRemoteUpdateWatermarkLossTest: TestDefinition = {
+    description:
+        "Client 0 sends three rapid updates. After syncing, both clients " +
+        "disconnect and reconnect twice. Content should remain correct " +
+        "after each reconnect.",
+    clients: 2,
+    steps: [
+        { type: "create", client: 0, path: "doc.md", content: "original" },
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        { type: "update", client: 0, path: "doc.md", content: "update 1" },
+        { type: "update", client: 0, path: "doc.md", content: "update 2" },
+        { type: "update", client: 0, path: "doc.md", content: "final update" },
+
+        { type: "barrier" },
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertFileCount(1).assertContent("doc.md", "final update");
+            }
+        },
+
+        { type: "disable-sync", client: 0 },
+        { type: "disable-sync", client: 1 },
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertFileCount(1).assertContent("doc.md", "final update");
+            }
+        },
+
+        { type: "disable-sync", client: 0 },
+        { type: "disable-sync", client: 1 },
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+        {
+            type: "assert-consistent",
+            verify: (s: AssertableState): void => {
+                s.assertFileCount(1).assertContent("doc.md", "final update");
+            }
+        }
+    ]
+};

frontend/deterministic-tests/src/tests/concurrent-delete-during-remote-update.test.ts

@@ -0,0 +1,32 @@
+import type { AssertableState } from "../utils/assertable-state";
+import type { TestDefinition } from "../test-definition";
+
+export const concurrentDeleteDuringRemoteUpdateTest: TestDefinition = {
+    description:
+        "One client updates a file while the other deletes it at the same " +
+        "time. Both clients should converge without errors.",
+    clients: 2,
+    steps: [
+        { type: "create", client: 0, path: "doc.md", content: "original" },
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        { type: "disable-sync", client: 0 },
+        { type: "disable-sync", client: 1 },
+
+        { type: "update", client: 0, path: "doc.md", content: "updated by 0" },
+        { type: "delete", client: 1, path: "doc.md" },
+
+        { type: "enable-sync", client: 0 },
+        { type: "enable-sync", client: 1 },
+        { type: "barrier" },
+
+        {
+            type: "assert-consistent",
+            verify: (state: AssertableState): void => {
+                state.assertFileCount(0);
+            }
+        }
+    ]
+};