mor

.forgejo/workflows/e2e.yml

@@ -17,6 +17,7 @@ env:
   RUSTFLAGS: "-Dwarnings"
   E2E_ARTIFACTS: logs/e2e
   E2E_WORKERS: 8
+  # Each campaign records a fresh random seed; concrete corpus traces always run.
 
 jobs:
   build:
@@ -42,6 +43,10 @@ jobs:
           toolchain: "1.92.0"
           components: clippy, rustfmt
 
+      - name: Expand scheduled fuzz campaigns
+        if: github.event_name == 'schedule'
+        run: echo "E2E_ITERATIONS=100" >> "$GITHUB_ENV"
+
       - name: E2E tests
         run: scripts/e2e.sh
 

frontend/deterministic-tests/README.md

@@ -7,10 +7,10 @@ E2E_WORKERS=2 /path/to/vault-link/scripts/e2e.sh
 ```
 
 The `E2E_WORKERS` variable is the number of fuzz workers / simultaneous scripted tests.
-The runner builds the server and the three in-scope frontend packages, then runs
+The runner builds the server and all frontend packages, then runs core/adapter
-the harness self-tests, recovery regressions, protocol/real-filesystem tests,
+unit tests, harness self-tests, recovery regressions, protocol/real-filesystem tests,
-scripted scenarios, and seeded fuzz workers. It does not build the Obsidian plugin
+scripted scenarios, concrete regression traces, and seeded fuzz workers.
-or CLI application. Install frontend dependencies with `npm ci` first. Node's
+Install frontend dependencies with `npm ci` first. Node's
 built-in WebSocket, Rust, and a C compiler (`cc`, for test-only exclusive rename
 and OS-lock primitives on macOS/Linux) are required.
 
@@ -29,8 +29,9 @@ before teardown so cleanup failures retain the original evidence.
 Environment options:
 
 - `E2E_WORKERS=1`: number of fuzz workers / simultaneous scripted tests (max 32).
-- `E2E_SEED=1`: first uint32 seed; workers use successive seeds.
+- `E2E_SEED`: optional first uint32 seed; otherwise a fresh random seed is recorded.
-- `E2E_ITERATIONS=25`: generated conflict scenarios per worker.
+  Workers use successive seeds, with no uint32 overflow.
+- `E2E_ITERATIONS=25`: generated rounds per worker; scheduled CI uses 100.
 - `E2E_TIMEOUT_SECONDS=900`: timeout for each build/suite/worker command.
 - `E2E_ARTIFACTS=/absolute/path`: optional output directory; defaults to `mkdtemp`.
 
@@ -49,7 +50,7 @@ npm run test:harness
 # and host filesystem contract checks.
 npm run test:protocol
 
-npm run build --workspace sync-client --workspace test-client
+npm run build --workspace sync-client --workspace deterministic-tests --workspace test-client
 node test-client/dist/cli.js --seed 42 --iterations 25
 node test-client/dist/cli.js --replay /path/to/trace-42.json
 ```
@@ -78,7 +79,10 @@ contiguous event log, and absence of pending requests.
 Internal recovery files are excluded from the user-visible file list, but are
 present in failure disk images. Scenario assertions separately check intended
 content, document identity, or preservation markers; equality alone is not an
-adequate data-loss oracle.
+adequate data-loss oracle. Custom `assert-consistent` verifiers must check exact
+content for the entire file set; partial substring assertions alone fail. Harness
+self-tests verify that shared corruption, duplicate files and identity reuse are
+rejected by these oracles.
 
 Clients start disabled. `enable-sync` waits for the initial sync attempt/retries,
 except while the server is intentionally paused: that attempt is tracked so a
@@ -101,6 +105,11 @@ distinct UUIDs at one path.
   checkpoint must be exercised and released, and held reads respect cancellation.
 - `delay-notifications`, `flush-notifications`: delay editor notifications without
   disabling preservation assertions.
+- `hold-request`, `wait-for-request`, `release-request`: gate create/content/manifest
+  CAS, content downloads or event reads before sending or after receiving a reply.
+  After-response gates can require a response type. `enable-sync` with `wait: false`
+  allows the script to reach and release a gate. Missing or unreleased gates fail.
+- `assert-response`: prove a required CAS response (for example `StaleBase`) occurred.
 - `drop-response`: `{client, kind: "create" | "content" | "manifest", point:
 "before" | "after"}`. After faults fire only on an `Accepted` CAS response.
 - `wait-for-response-drop`: explicit checkpoint; every armed fault must fire.
@@ -111,7 +120,15 @@ distinct UUIDs at one path.
 - `pause-server`, `resume-server`: SIGSTOP/SIGCONT, not a crash.
 - `crash-server`, `restart-server`: SIGKILL and restart with the same database/port.
   Server-control scenarios have dedicated processes.
+- `crash-client`, `restart-client`: terminate a worker at an accepted response or
+  filesystem/persistence boundary, then start a fresh runtime from saved state.
+  Prepare the client's work while disabled. Post-crash disk edits are supported;
+  interrupted CAS requests must retain their request ID, target and payload.
+- `fail-save`, `wait-for-save-failure`: reject before or after durable metadata
+  replacement and require that the injection actually fired.
 - `remember-identity`, `assert-identity`, `assert-files`, `assert-markers`.
+- `remember-local-identity`: bind an offline create's UUID from its durable journal
+  before network reconciliation. `{{key}}` in a path resolves a bound identity.
 - `assert-documents`: a complete expected set of `{key, path, content, conflict?}`
   records. Content is text or a byte array. Keys bind UUIDs on the first successful
   check and must retain them on later checks; new keys cannot reuse deleted UUIDs.
@@ -119,8 +136,8 @@ distinct UUIDs at one path.
 - `assert-consistent` can provide `verify(state)`: exact text/bytes, UUID lookups,
   `assertIdentity`, and `conflictPath(original)` for UUID-derived conflict names.
 
-Legacy timing scenarios are retained. Prefer explicit fault checkpoints and
+Scripted race scenarios use observable checkpoints instead of sleep steps.
-barriers for new tests. Only known coherent-scan races and actually injected
+Only known coherent-scan races and actually injected
 transient network errors may retry; they still must converge before the deadline.
 Other background errors, unhandled rejections, and cleanup failures fail the run.
 
@@ -128,6 +145,10 @@ Other background errors, unhandled rejections, and cleanup failures fail the run
 
 Seeds and JSON traces reproduce harness choices and logical actions. They do not
 control the OS scheduler, server thread scheduling, or production-generated UUIDs.
+Results retain the exercised checkpoint schedule; replay requires the selected
+boundaries and responses to occur again. The seeded client automatically reduces
+failing programs while retaining complete independent expectations. See
+[`../test-client/README.md`](../test-client/README.md) for corpus and coverage details.
 The recovery sweep enumerates adapter/save boundaries; worker termination proves
 abrupt runtime loss. Neither is a proof covering every real filesystem or hardware
 power-loss behavior. Run multiple seeds, repeat timing-sensitive cases, and keep

frontend/deterministic-tests/src/crash-worker.ts

@@ -18,6 +18,11 @@ export interface CrashWorkerInput {
     stored: StoredClient;
     stopAt?: RequestKind;
     editAfterSync?: { path: string; content: string };
+    stopAtBoundary?: {
+        type: "persistence" | "filesystem";
+        label: string;
+        afterAccepted?: RequestKind;
+    };
 }
 const input = workerData as CrashWorkerInput;
 const disk = new MemoryDisk();
@@ -25,13 +30,35 @@ disk.restore(input.disk);
 const persistence = new MemoryPersistence(input.stored);
 const requests: RequestRecord[] = [];
 let editedBase: { documentId: string; vaultUpdateId: number } | undefined;
+let acceptedKind: RequestKind | undefined;
+let boundary: string | undefined;
 const snapshot = () => ({
     disk: disk.image(),
     stored: persistence.snapshot(),
     requests,
-    editedBase
+    editedBase,
+    boundary
 });
 
+for (const [type, target] of [
+    ["filesystem", disk],
+    ["persistence", persistence]
+] as const) {
+    target.boundary = async (label) => {
+        const stop = input.stopAtBoundary;
+        if (
+            stop?.type === type &&
+            stop.label === label &&
+            (stop.afterAccepted === undefined ||
+                acceptedKind === stop.afterAccepted)
+        ) {
+            boundary = label;
+            parentPort!.postMessage({ type: "checkpoint", ...snapshot() });
+            await new Promise<never>(() => {});
+        }
+    };
+}
+
 async function main() {
     const client = await SyncClient.create({
         fs: disk.session(),
@@ -40,11 +67,16 @@ async function main() {
             const record = classifyRequest(url, init);
             if (record) requests.push(record);
             const response = await fetch(url, init);
-            if (record && response.ok && input.stopAt === record.kind) {
+            if (record && response.ok) {
                 const receipt = (await response.clone().json()) as {
                     type: string;
                 };
-                if (receipt.type === "Accepted") {
+                record.responseType = receipt.type;
+                if (receipt.type === "Accepted") acceptedKind = record.kind;
+                if (
+                    receipt.type === "Accepted" &&
+                    input.stopAt === record.kind
+                ) {
                     parentPort!.postMessage({
                         type: "checkpoint",
                         ...snapshot()

frontend/deterministic-tests/src/deterministic-agent.ts

@@ -11,6 +11,8 @@ import { sleep } from "./utils/sleep";
 import { withTimeout } from "./utils/with-timeout";
 import { WAIT_TIMEOUT_MS, WEBSOCKET_CONNECT_TIMEOUT_MS } from "./consts";
 import { ManagedWebSocketFactory } from "./managed-websocket";
+import { terminateAtCheckpoint } from "./worker-control";
+import type { CrashWorkerInput } from "./crash-worker";
 
 /** User actions live here; the engine only receives disk.session(). */
 export class DeterministicAgent {
@@ -20,7 +22,15 @@ export class DeterministicAgent {
     private client?: SyncClient;
     private readonly errors: Error[] = [];
     private readonly pending = new Set<Promise<void>>();
-    private readonly wsFactory = new ManagedWebSocketFactory();
+    private wsFactory = new ManagedWebSocketFactory();
+    private crashed = false;
+    private interruptedRequest?: import("../../test-support/network").RequestRecord;
+    private saveFailure?: {
+        fired: boolean;
+        reached: Promise<void>;
+        resolve: () => void;
+        message: string;
+    };
     private nextWriteRename?: { oldPath: string; newPath: string };
     private delayedNotifications = false;
     private disposed = false;
@@ -131,6 +141,7 @@ export class DeterministicAgent {
         for (const notify of this.notifications.splice(0)) this.enqueue(notify);
     }
     private notify(operation: () => Promise<void>): void {
+        if (this.crashed) return;
         // Offline means transport disabled, not that editor identity events vanish.
         if (this.delayedNotifications) this.notifications.push(operation);
         else this.enqueue(operation);
@@ -145,13 +156,15 @@ export class DeterministicAgent {
         this.pending.add(promise);
         void promise.then(() => this.pending.delete(promise));
     }
-    private async drain(): Promise<void> {
+    public async drain(): Promise<void> {
         while (this.pending.size) await Promise.all([...this.pending]);
     }
     private isRetryableFailure(message: string): boolean {
         // A coherent scan rejecting an editor race is a required adapter
         // behavior. It must subsequently converge; never blanket-ignore errors.
         return (
+            (this.saveFailure?.fired === true &&
+                message.includes(this.saveFailure.message)) ||
             this.network.isExpectedFailure(message) ||
             /^(Sync paused for retry: )?Error: File changed during scan: .+$/.test(
                 message
@@ -220,6 +233,99 @@ export class DeterministicAgent {
         await this.drain();
         await this.client!.reset();
     }
+    public async setSettings(settings: Partial<SyncSettings>): Promise<void> {
+        await this.client!.setSettings(settings);
+    }
+    public failSave(point: "before" | "durable"): void {
+        assert(
+            !this.saveFailure || this.saveFailure.fired,
+            "Save fault never fired"
+        );
+        const reached = Promise.withResolvers<void>();
+        const fault = {
+            fired: false,
+            reached: reached.promise,
+            resolve: () => reached.resolve(),
+            message: `Injected ${point} persistence failure`
+        };
+        this.saveFailure = fault;
+        this.persistence.boundary = (label) => {
+            if (label !== `${point}:save` || fault.fired) return;
+            fault.fired = true;
+            this.persistence.boundary = () => {};
+            this.network.schedule.push({ type: "save-failure", detail: point });
+            fault.resolve();
+            throw new Error(fault.message);
+        };
+    }
+    public async waitForSaveFailure(): Promise<void> {
+        assert(this.saveFailure !== undefined, "Save fault not armed");
+        await withTimeout(
+            this.saveFailure!.reached,
+            WAIT_TIMEOUT_MS,
+            "Save fault never fired"
+        );
+        await this.drain();
+    }
+    public async crashDuringSync(
+        at: Pick<CrashWorkerInput, "stopAt" | "stopAtBoundary">,
+        powerLoss = false
+    ): Promise<void> {
+        assert(
+            !this.crashed && this.client !== undefined,
+            "Client already crashed"
+        );
+        await this.drain();
+        assert(
+            !this.client.getSettings().isSyncEnabled,
+            "Prepare crash work with transport disabled"
+        );
+        this.network.assertConsumed();
+        await this.client.destroy();
+        await this.wsFactory.finish();
+        const stored = this.persistence.snapshot();
+        stored.settings = { ...stored.settings, isSyncEnabled: true };
+        const snapshot = await terminateAtCheckpoint({
+            disk: this.disk.image(),
+            stored,
+            ...at
+        });
+        this.disk.restore(snapshot.disk);
+        this.disk.crash(powerLoss);
+        await this.persistence.save(snapshot.stored);
+        this.network.requests.push(...snapshot.requests);
+        this.network.schedule.push({
+            type: "crash",
+            detail: { at, powerLoss, boundary: snapshot.boundary }
+        });
+        this.interruptedRequest = snapshot.stored.database?.pending
+            ? snapshot.requests.at(-1)
+            : undefined;
+        this.client = undefined;
+        this.crashed = true;
+    }
+    public async restartAfterCrash(): Promise<void> {
+        assert(this.crashed, "Restart requires a crashed client");
+        this.wsFactory = new ManagedWebSocketFactory();
+        this.crashed = false;
+        await this.init();
+        await this.waitForSync();
+        if (this.interruptedRequest) {
+            const request = this.interruptedRequest;
+            const retries = this.network.requests.filter(
+                (r) => r.body.requestId === request.body.requestId
+            );
+            assert(retries.length >= 2, "Crashed CAS was never retried");
+            for (const retry of retries)
+                assert(
+                    JSON.stringify(retry.body) ===
+                        JSON.stringify(request.body) &&
+                        retry.url === request.url,
+                    "Crash retry changed request identity, payload or target"
+                );
+        }
+        this.interruptedRequest = undefined;
+    }
     public pauseWebSocket(): void {
         this.wsFactory.pause();
     }
@@ -291,9 +397,20 @@ export class DeterministicAgent {
         }
     }
     public async cleanup(): Promise<void> {
+        assert(!this.crashed, "Crashed client was never restarted");
+        assert(
+            !this.saveFailure || this.saveFailure.fired,
+            "Save fault never fired"
+        );
         if (!this.client) return;
         const errors: unknown[] = [];
+        try {
+            this.network.assertCheckpointsConsumed();
+        } catch (error) {
+            errors.push(error);
+        }
         this.resumeObservation();
+        this.network.releaseRequests();
         this.flushNotifications();
         try {
             await this.waitForSync();

frontend/deterministic-tests/src/test-definition.ts

@@ -1,4 +1,5 @@
 import type { AssertableState } from "./utils/assertable-state";
+import type { RequestCheckpoint } from "../../test-support/network";
 
 export interface ClientState {
     files: Map<string, string>;
@@ -15,6 +16,7 @@ export interface ExpectedDocument {
     path: string;
     conflict?: boolean;
     content: string | number[];
+    alternatives?: (string | number[])[];
 }
 
 export type TestStep =
@@ -30,7 +32,40 @@ export type TestStep =
     | { type: "delete"; client: number; path: string }
     | { type: "sync"; client?: number }
     | { type: "disable-sync"; client: number }
-    | { type: "enable-sync"; client: number }
+    | { type: "enable-sync"; client: number; wait?: boolean }
+    | ({ type: "hold-request"; client: number } & RequestCheckpoint)
+    | { type: "wait-for-request" | "release-request"; client: number }
+    | {
+          type: "remember-local-identity";
+          client: number;
+          path: string;
+          key: string;
+      }
+    | {
+          type: "set-settings";
+          client: number;
+          settings: Partial<import("sync-client").SyncSettings>;
+      }
+    | {
+          type: "crash-client";
+          client: number;
+          at: Pick<
+              import("./crash-worker").CrashWorkerInput,
+              "stopAt" | "stopAtBoundary"
+          >;
+          powerLoss?: boolean;
+      }
+    | { type: "restart-client"; client: number }
+    | { type: "fail-save"; client: number; point: "before" | "durable" }
+    | { type: "wait-for-save-failure"; client: number }
+    | {
+          type: "assert-response";
+          client: number;
+          kind: "create" | "content" | "manifest";
+          responseType: string;
+          count?: number;
+          sinceCheckpoint?: boolean;
+      }
     | { type: "pause-server" }
     | { type: "resume-server" }
     | {
@@ -90,4 +125,9 @@ export interface TestResult {
     error?: string;
     duration?: number;
     diagnostics?: unknown;
+    coverage?: Record<string, number>;
+    schedule?: {
+        client: number;
+        events: { type: string; detail: unknown }[];
+    }[];
 }

frontend/deterministic-tests/src/test-runner.ts

@@ -24,6 +24,7 @@ export class TestRunner {
     private readonly logger: Logger;
     private vaultName = "";
     private readonly identities = new Map<string, string>();
+    private readonly coverage: Record<string, number> = {};
 
     public constructor(
         serverControl: ServerControl,
@@ -49,7 +50,9 @@ export class TestRunner {
         this.logger.info(`Clients: ${test.clients}`);
         this.logger.info(`Steps: ${test.steps.length}`);
         this.identities.clear();
+        for (const key of Object.keys(this.coverage)) delete this.coverage[key];
         let diagnostics: TestResult["diagnostics"];
+        let evidence: Pick<TestResult, "coverage" | "schedule"> | undefined;
 
         try {
             assert(
@@ -64,7 +67,9 @@ export class TestRunner {
                 this.logger.info(
                     `Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
                 );
-                await this.executeStep(step);
+                this.coverage[`step:${step.type}`] =
+                    (this.coverage[`step:${step.type}`] ?? 0) + 1;
+                await this.executeStep(this.resolveStep(step));
             }
 
             assert(
@@ -72,6 +77,7 @@ export class TestRunner {
                 "Server died before test completion"
             );
             diagnostics = this.captureDiagnostics();
+            evidence = this.captureEvidence();
             await this.cleanup();
 
             const duration = Date.now() - startTime;
@@ -79,10 +85,12 @@ export class TestRunner {
 
             return {
                 success: true,
-                duration
+                duration,
+                ...evidence
             };
         } catch (error) {
             diagnostics ??= this.captureDiagnostics();
+            evidence ??= this.captureEvidence();
             const duration = Date.now() - startTime;
             const errorMessage =
                 error instanceof Error ? error.message : String(error);
@@ -96,7 +104,8 @@ export class TestRunner {
                     success: false,
                     error: `${errorMessage}\nCleanup: ${String(cleanupError)}`,
                     duration,
-                    diagnostics
+                    diagnostics,
+                    ...evidence
                 };
             }
 
@@ -104,17 +113,86 @@ export class TestRunner {
                 success: false,
                 error: errorMessage,
                 diagnostics,
-                duration
+                duration,
+                ...evidence
             };
         }
     }
 
+    private resolvePath(path: string): string {
+        return path.replace(/\{\{([^}]+)\}\}/g, (_, key: string) => {
+            const id = this.identities.get(key);
+            assert(id !== undefined, `Unknown path identity ${key}`);
+            return id;
+        });
+    }
+
+    private captureEvidence(): Pick<TestResult, "coverage" | "schedule"> {
+        const coverage = { ...this.coverage };
+        const count = (key: string) => {
+            coverage[key] = (coverage[key] ?? 0) + 1;
+        };
+        for (const agent of this.agents) {
+            for (const request of agent.network.requests) {
+                count(
+                    `response:${request.kind}:${request.responseType ?? request.status ?? "interrupted"}`
+                );
+                const payload = request.body.content as
+                    | { type?: string }
+                    | undefined;
+                if (payload?.type) count(`payload:${payload.type}`);
+            }
+            for (const event of agent.network.schedule) {
+                count(`checkpoint:${event.type}`);
+                if (event.type === "reached") {
+                    const { spec } = event.detail as {
+                        spec: { kind: string; point: string };
+                    };
+                    count(`held:${spec.kind}:${spec.point}`);
+                } else if (event.type === "crash") {
+                    const { at, boundary } = event.detail as {
+                        at: { stopAt?: string };
+                        boundary?: string;
+                    };
+                    count(
+                        `crash:${at.stopAt ?? boundary?.split(":").slice(0, 2).join(":")}`
+                    );
+                } else if (event.type === "save-failure")
+                    count(`save-failure:${event.detail}`);
+            }
+        }
+        return {
+            coverage,
+            schedule: this.agents.map((agent) => ({
+                client: agent.clientId,
+                events: structuredClone(agent.network.schedule)
+            }))
+        };
+    }
+
+    private resolveStep(step: TestStep): TestStep {
+        const next = { ...step };
+        for (const field of ["path", "oldPath", "newPath"] as const)
+            if (field in next)
+                (next as unknown as Record<string, unknown>)[field] =
+                    this.resolvePath(
+                        (next as unknown as Record<string, string>)[field]
+                    );
+        if (next.type === "assert-documents")
+            next.expected = next.expected.map((doc) => ({
+                ...doc,
+                path: this.resolvePath(doc.path)
+            }));
+        return next;
+    }
+
     private captureDiagnostics(): unknown {
         return this.agents.map((agent) => ({
             client: agent.clientId,
             database: agent.database(),
             disk: agent.disk.image(),
-            requests: structuredClone(agent.network.requests)
+            requests: structuredClone(agent.network.requests),
+            schedule: structuredClone(agent.network.schedule)
         }));
     }
 
@@ -161,6 +239,88 @@ export class TestRunner {
 
     private async executeStep(step: TestStep): Promise<void> {
         switch (step.type) {
+            case "fail-save":
+                this.getAgent(step.client).failSave(step.point);
+                break;
+            case "wait-for-save-failure":
+                await this.getAgent(step.client).waitForSaveFailure();
+                break;
+            case "crash-client":
+                await this.getAgent(step.client).crashDuringSync(
+                    step.at,
+                    step.powerLoss
+                );
+                break;
+            case "restart-client":
+                await this.getAgent(step.client).restartAfterCrash();
+                break;
+            case "set-settings":
+                await this.getAgent(step.client).setSettings(step.settings);
+                break;
+            case "remember-local-identity": {
+                const agent = this.getAgent(step.client);
+                await agent.drain();
+                const stored = agent.persistence.snapshot();
+                const created = stored.localChanges
+                    ?.filter(
+                        (change) =>
+                            change.type === "create" &&
+                            change.path === step.path
+                    )
+                    .at(-1);
+                const id =
+                    (created?.type === "create"
+                        ? created.documentId
+                        : undefined) ??
+                    Object.entries(agent.database()?.local ?? {}).find(
+                        ([, path]) => path === step.path
+                    )?.[0];
+                assert(id !== undefined, `No local identity at ${step.path}`);
+                const previous = this.identities.get(step.key);
+                assert(
+                    previous === undefined || previous === id,
+                    `Identity ${step.key} changed before publication`
+                );
+                assert(
+                    previous !== undefined ||
+                        ![...this.identities.values()].includes(id),
+                    `New document ${step.key} reused an identity`
+                );
+                this.identities.set(step.key, id);
+                break;
+            }
+            case "hold-request":
+                this.getAgent(step.client).network.hold(step);
+                break;
+            case "wait-for-request":
+                await withTimeout(
+                    this.getAgent(step.client).network.waitForHeldRequest(),
+                    CONVERGENCE_TIMEOUT_MS,
+                    "Request checkpoint never reached"
+                );
+                break;
+            case "release-request":
+                this.getAgent(step.client).network.resumeRequest();
+                break;
+            case "assert-response": {
+                const network = this.getAgent(step.client).network;
+                const matching = (
+                    step.sinceCheckpoint
+                        ? network.requestsSinceCheckpoint()
+                        : network.requests
+                ).filter(
+                    (r) =>
+                        r.kind === step.kind &&
+                        r.responseType === step.responseType
+                );
+                assert(
+                    step.count === undefined
+                        ? matching.length > 0
+                        : matching.length === step.count,
+                    `Expected ${step.count ?? "at least one"} ${step.kind} ${step.responseType} response, got ${matching.length}`
+                );
+                break;
+            }
             case "pause-observation":
                 this.getAgent(step.client).pauseObservation();
                 break;
@@ -284,7 +444,7 @@ export class TestRunner {
 
             case "enable-sync":
                 await this.getAgent(step.client).enableSync(
-                    !this.serverControl.isPaused
+                    step.wait ?? !this.serverControl.isPaused
                 );
                 break;
 
@@ -319,7 +479,14 @@ export class TestRunner {
                 break;
 
             case "assert-consistent":
-                await this.assertConsistent(step.verify);
+                await this.assertConsistent(
+                    step.verify
+                        ? (state) => {
+                              step.verify!(state);
+                              state.assertComplete();
+                          }
+                        : undefined
+                );
                 break;
 
             case "pause-websocket":

frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts

@@ -34,10 +34,9 @@ export const binaryPendingCreateNotDisplacedTest: TestDefinition = {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
                 s.assertFileCount(2)
-                    .assertFileExists("data.bin")
+                    .assertContent("data.bin", "binary data from client 0")
-                    .assertFileExists(s.conflictPath("data.bin"))
+                    .assertContent(
-                    .assertAnyFileContains(
+                        s.conflictPath("data.bin"),
-                        "binary data from client 0",
                         "binary data from client 1"
                     );
             }

frontend/deterministic-tests/src/tests/binary-to-text-transition.test.ts

@@ -38,11 +38,7 @@ export const binaryToTextTransitionTest: TestDefinition = {
         {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
-                s.assertFileCount(1).assertContainsAny(
+                s.assertFileCount(1).assertContent("data.bin", "version A");
-                    "data.bin",
-                    "version A",
-                    "version B"
-                );
             }
         },
 
@@ -90,7 +86,10 @@ export const binaryToTextTransitionTest: TestDefinition = {
         {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
-                s.assertFileCount(1).assertContains("data.md", "alpha", "beta");
+                s.assertFileCount(1).assertContent(
+                    "data.md",
+                    "alpha\nmiddle line\nbeta"
+                );
             }
         }
     ]

frontend/deterministic-tests/src/tests/coalesce-update-remote-update-data-loss.test.ts

@@ -45,13 +45,11 @@ export const coalesceUpdateRemoteUpdateDataLossTest: TestDefinition = {
 
         {
             type: "assert-consistent",
-            verify: (state: AssertableState): void => {
+            verify: (s: AssertableState): void => {
-                state
+                s.assertFileCount(1);
-                    .assertFileCount(1)
+                s.assertContent(
-                    .assertContains(
                     "doc.md",
-                        "client 0 addition",
+                    "client 1 addition\nline 1\nline 2\nline 3\nclient 0 addition"
-                        "client 1 addition"
                 );
             }
         }

frontend/deterministic-tests/src/tests/concurrent-edit-exact-same-position.test.ts

@@ -42,7 +42,11 @@ export const concurrentEditExactSamePositionTest: TestDefinition = {
             verify: (state: AssertableState): void => {
                 state
                     .assertFileCount(1)
-                    .assertContains("doc.md", "slow", "fast", "brown fox");
+                    .assertContentOneOf(
+                        "doc.md",
+                        "the fast slow brown fox",
+                        "the slow fast brown fox"
+                    );
             }
         }
     ]

frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts

@@ -1,7 +1,8 @@
 import type { AssertableState } from "../utils/assertable-state";
 import type { TestDefinition } from "../test-definition";
 
-export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = {
+export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition =
+    {
         description:
             "One client renames X to Y while another creates a new file at Y, " +
             "both offline. After syncing, Y should contain merged content from " +
@@ -38,11 +39,13 @@ export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition =
 
             {
                 type: "assert-consistent",
-            verify: (state: AssertableState): void => {
+                verify: (s: AssertableState): void => {
-                state
+                    s.assertFileCount(2)
-                    .assertFileCount(2)
+                        .assertContent("Y.md", "brand new Y content")
-                    .assertContains(state.conflictPath("Y.md"), "original file X")
+                        .assertContent(
-                    .assertContains("Y.md", "brand new Y content");
+                            s.conflictPath("Y.md"),
+                            "original file X"
+                        );
                 }
             }
         ]

frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts

@@ -1,7 +1,8 @@
 import type { AssertableState } from "../utils/assertable-state";
 import type { TestDefinition } from "../test-definition";
 
-export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = {
+export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition =
+    {
         description:
             "One client renames X to Y while another creates a new file at Y, " +
             "both offline. We can't merge the create because it would result in a cycle",
@@ -37,13 +38,11 @@ export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition =
 
             {
                 type: "assert-consistent",
-            verify: (state: AssertableState): void => {
+                verify: (s: AssertableState): void => {
-                state
+                    s.assertFileCount(2)
-                    .assertFileNotExists("X.md")
+                        .assertContent("Y.md", "original file X")
-                    .assertFileExists("Y.md")
+                        .assertContent(
-                    .assertFileExists(state.conflictPath("Y.md"))
+                            s.conflictPath("Y.md"),
-                    .assertAnyFileContains(
-                        "original file X",
                             "brand new Y content"
                         );
                 }

frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts

@@ -25,14 +25,10 @@ export const concurrentRenameSameTargetTest: TestDefinition = {
 
         {
             type: "assert-consistent",
-            verify: (state: AssertableState): void => {
+            verify: (s: AssertableState): void => {
-                state
+                s.assertFileCount(2)
-                    .assertFileCount(2)
+                    .assertContent("C.md", "content-a")
-                    .assertFileNotExists("A.md")
+                    .assertContent(s.conflictPath("C.md"), "content-b");
-                    .assertFileNotExists("B.md")
-                    .assertFileExists("C.md")
-                    .assertFileExists(state.conflictPath("C.md"))
-                    .assertAnyFileContains("content-a", "content-b");
             }
         }
     ]

frontend/deterministic-tests/src/tests/create-delete-noop.test.ts

@@ -20,7 +20,7 @@ export const createDeleteNoopTest: TestDefinition = {
         {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
-                s.assertFileNotExists("temp.md");
+                s.assertFileCount(0).assertFileNotExists("temp.md");
             }
         }
     ]

frontend/deterministic-tests/src/tests/create-merge-delete.test.ts

@@ -17,10 +17,14 @@ export const createMergeDeleteTest: TestDefinition = {
 
         {
             type: "assert-consistent",
-            verify: (state: AssertableState): void => {
+            verify: (s: AssertableState): void => {
-                state
+                s.assertFileCount(1).assertContentOneOf(
-                    .assertFileCount(1)
+                    "A.md",
-                    .assertContains("A.md", "from-zero", "from-one");
+                    "from-zerofrom-one",
+                    "from-onefrom-zero",
+                    "from-zero from-one",
+                    "from-one from-zero"
+                );
             }
         },
 

frontend/deterministic-tests/src/tests/create-merge-preserves-renamed-update.test.ts

@@ -17,8 +17,14 @@ export const createMergePreservesRenamedUpdateTest: TestDefinition = {
 
         {
             type: "assert-consistent",
-            verify: (state: AssertableState): void => {
+            verify: (s: AssertableState): void => {
-                state.assertContains("doc.md", "alpha", "beta");
+                s.assertFileCount(1).assertContentOneOf(
+                    "doc.md",
+                    "alphabeta",
+                    "betaalpha",
+                    "alpha beta",
+                    "beta alpha"
+                );
             }
         },
 

frontend/deterministic-tests/src/tests/create-rename-response-skips-file.test.ts

@@ -29,7 +29,8 @@ export const createRenameResponseSkipsFileTest: TestDefinition = {
         {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
-                s.assertFileCount(1).assertAnyFileContains("the-content");
+                s.assertFileCount(1);
+                s.assertContent("renamed.md", "the-content");
             }
         }
     ]

frontend/deterministic-tests/src/tests/delete-by-other-client-then-recreate.test.ts

@@ -18,7 +18,7 @@ export const deleteByOtherClientThenRecreateTest: TestDefinition = {
         {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
-                s.assertFileNotExists("A.md");
+                s.assertFileCount(0).assertFileNotExists("A.md");
             }
         },
 

frontend/deterministic-tests/src/tests/delete-during-pending-create.test.ts

@@ -3,15 +3,15 @@ import type { TestDefinition } from "../test-definition";
 
 export const deleteDuringPendingCreateTest: TestDefinition = {
     description:
-        "Client 0 creates a file while the server is paused, then deletes it before the server resumes. " +
+        "Client 0 creates a file while its create request is held, then deletes it before releasing the request. " +
-        "After resume, the file should end up deleted on both clients.",
+        "After release, the file should end up deleted on both clients.",
     clients: 2,
     steps: [
         { type: "enable-sync", client: 0 },
         { type: "enable-sync", client: 1 },
         { type: "barrier" },
 
-        { type: "pause-server" },
+        { type: "hold-request", client: 0, kind: "create", point: "before" },
 
         {
             type: "create",
@@ -19,10 +19,11 @@ export const deleteDuringPendingCreateTest: TestDefinition = {
             path: "ephemeral.md",
             content: "this will be deleted"
         },
+        { type: "wait-for-request", client: 0 },
 
         { type: "delete", client: 0, path: "ephemeral.md" },
 
-        { type: "resume-server" },
+        { type: "release-request", client: 0 },
         { type: "barrier" },
 
         {

frontend/deterministic-tests/src/tests/delete-recreate-concurrent-update.test.ts

@@ -35,7 +35,8 @@ export const deleteRecreateConcurrentUpdateTest: TestDefinition = {
         {
             type: "assert-consistent",
             verify: (s: AssertableState): void => {
-                s.assertFileExists("A.md").assertContains("A.md", "recreated");
+                s.assertFileCount(1);
+                s.assertContent("A.md", "recreated by client 0");
             }
         }
     ]