Add deterministic-tests (#190)
Reviewed-on: https://home.schmelczer.dev/git/git/andras/vault-link/pulls/190 Co-authored-by: Andras Schmelczer <andras@schmelczer.dev> Co-committed-by: Andras Schmelczer <andras@schmelczer.dev>
This commit is contained in:
parent
4482e0155f
commit
0e3132f96c
127 changed files with 7722 additions and 1 deletions
118
frontend/deterministic-tests/README.md
Normal file
118
frontend/deterministic-tests/README.md
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
# Deterministic Tests
|
||||
|
||||
Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case.
|
||||
|
||||
Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios.
|
||||
|
||||
## How it works
|
||||
|
||||
Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one.
|
||||
|
||||
Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process.
|
||||
|
||||
The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit.
|
||||
|
||||
## Step types
|
||||
|
||||
Clients always start with syncing disabled.
|
||||
|
||||
**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited):
|
||||
|
||||
- `create`, `update`, `rename`, `delete`
|
||||
- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path.
|
||||
|
||||
**Sync control:**
|
||||
|
||||
- `sync` — wait for a specific client or all clients to finish pending operations
|
||||
- `barrier` — retry until all clients converge to identical file state (60s timeout)
|
||||
- `enable-sync` / `disable-sync` — simulate going online/offline
|
||||
- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable
|
||||
- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync`
|
||||
|
||||
**WebSocket control** (per-client):
|
||||
|
||||
- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client
|
||||
|
||||
**Server control:**
|
||||
|
||||
- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process
|
||||
- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire.
|
||||
|
||||
**Fault injection** (per-client):
|
||||
|
||||
- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit.
|
||||
- `wait-for-dropped-create-response` — wait until the armed drop has fired.
|
||||
|
||||
**Assertions:**
|
||||
|
||||
- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback
|
||||
|
||||
## Running
|
||||
|
||||
```sh
|
||||
# Build server first
|
||||
cd sync-server && cargo build --release && cd -
|
||||
|
||||
# Run all tests
|
||||
cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests
|
||||
|
||||
# Filter by name
|
||||
npm run test -w deterministic-tests -- --filter=rename
|
||||
|
||||
# Control parallelism (default: number of CPU cores)
|
||||
npm run test -w deterministic-tests -- -j 4
|
||||
```
|
||||
|
||||
## Adding a test
|
||||
|
||||
1. Create `src/tests/my-scenario.test.ts`:
|
||||
|
||||
```typescript
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const myScenarioTest: TestDefinition = {
|
||||
description:
|
||||
"Client 0 creates A.md offline. After syncing, both clients should have the file.",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{ type: "create", client: 0, path: "A.md", content: "hello" },
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s) => {
|
||||
s.assertFileCount(1).assertContent("A.md", "hello");
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
```
|
||||
|
||||
The `verify` callback receives an `AssertableState` object with chainable assertion methods:
|
||||
|
||||
```typescript
|
||||
s.assertFileCount(n); // exact file count
|
||||
s.assertFileExists("path"); // file must exist
|
||||
s.assertFileNotExists("path"); // file must not exist
|
||||
s.assertContent("path", "expected"); // exact content match
|
||||
s.assertContains("path", "a", "b"); // all substrings present in file
|
||||
s.assertContainsAny("path", "a", "b"); // at least one substring present
|
||||
s.assertAnyFileContains("text"); // substring present in some file
|
||||
s.assertNoFileContains("text"); // substring absent from every file
|
||||
s.assertSubstringCount("path", "x", 3); // substring appears exactly N times
|
||||
s.assertContentInAtMostOneFile("text"); // no duplicate content
|
||||
s.ifFileExists("path", (s) => { /* … */ }); // conditional block
|
||||
s.getContent("path"); // raw content (or "" if missing)
|
||||
```
|
||||
|
||||
2. Register it in `src/test-registry.ts`:
|
||||
|
||||
```typescript
|
||||
import { myScenarioTest } from "./tests/my-scenario.test";
|
||||
|
||||
const TESTS = {
|
||||
// ...
|
||||
"my-scenario": myScenarioTest
|
||||
};
|
||||
```
|
||||
23
frontend/deterministic-tests/package.json
Normal file
23
frontend/deterministic-tests/package.json
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"name": "deterministic-tests",
|
||||
"version": "0.14.0",
|
||||
"private": true,
|
||||
"bin": {
|
||||
"deterministic-tests": "./dist/cli.js"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "webpack watch --mode development",
|
||||
"build": "webpack --mode production",
|
||||
"test": "npm run build && node dist/cli.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"commander": "^14.0.2",
|
||||
"@types/node": "^25.0.2",
|
||||
"sync-client": "file:../sync-client",
|
||||
"ts-loader": "^9.5.4",
|
||||
"tslib": "2.8.1",
|
||||
"typescript": "5.9.3",
|
||||
"webpack": "^5.103.0",
|
||||
"webpack-cli": "^6.0.1"
|
||||
}
|
||||
}
|
||||
243
frontend/deterministic-tests/src/cli.ts
Normal file
243
frontend/deterministic-tests/src/cli.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import { TestRunner } from "./test-runner";
|
||||
import { ServerControl } from "./server-control";
|
||||
import { ServerManager } from "./server-manager";
|
||||
import { PrefixedLogger } from "./prefixed-logger";
|
||||
import { TESTS } from "./test-registry";
|
||||
import type { TestDefinition, TestResult } from "./test-definition";
|
||||
import { parseArgs } from "./parse-args";
|
||||
import { runWithConcurrency } from "./run-with-concurrency";
|
||||
import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts";
|
||||
import * as path from "node:path";
|
||||
import * as fs from "node:fs";
|
||||
import { debugging, Logger } from "sync-client";
|
||||
|
||||
const logger = new Logger();
|
||||
debugging.logToConsole(logger, { useColors: true });
|
||||
|
||||
process.on("unhandledRejection", (reason) => {
|
||||
logger.error(`Unhandled Rejection: ${reason}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("uncaughtException", (error) => {
|
||||
logger.error(`Uncaught Exception: ${error}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const serverManager = new ServerManager(logger);
|
||||
serverManager.installSignalHandlers();
|
||||
|
||||
function testUsesPauseServer(test: TestDefinition): boolean {
|
||||
return test.steps.some(
|
||||
(step) =>
|
||||
step.type === "pause-server" ||
|
||||
step.type === "resume-server" ||
|
||||
step.type === "resume-server-until-history-then-pause"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk up from the CLI binary's location until we find a directory
|
||||
* containing `sync-server/` and `frontend/`.
|
||||
*/
|
||||
function findProjectRoot(): string {
|
||||
let dir = path.dirname(__filename);
|
||||
const root = path.parse(dir).root;
|
||||
while (dir !== root) {
|
||||
if (
|
||||
fs.existsSync(path.join(dir, "sync-server")) &&
|
||||
fs.existsSync(path.join(dir, "frontend"))
|
||||
) {
|
||||
return dir;
|
||||
}
|
||||
dir = path.dirname(dir);
|
||||
}
|
||||
throw new Error(
|
||||
`Could not locate project root (no ancestor of ${__filename} contains both 'sync-server' and 'frontend')`
|
||||
);
|
||||
}
|
||||
|
||||
interface NamedTestResult {
|
||||
name: string;
|
||||
result: TestResult;
|
||||
}
|
||||
|
||||
async function runSharedServerTest(
|
||||
name: string,
|
||||
test: TestDefinition,
|
||||
sharedServer: ServerControl
|
||||
): Promise<NamedTestResult> {
|
||||
const testLogger = new PrefixedLogger(logger, name);
|
||||
const runner = new TestRunner(
|
||||
sharedServer,
|
||||
testLogger,
|
||||
TOKEN,
|
||||
sharedServer.remoteUri
|
||||
);
|
||||
const result = await runner.runTest(name, test);
|
||||
if (result.success) {
|
||||
logger.info(`PASSED: ${name} (${result.duration}ms)`);
|
||||
} else {
|
||||
logger.error(`FAILED: ${name} - ${result.error}`);
|
||||
}
|
||||
return { name, result };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a test with its own dedicated server (for tests that use pause-server).
|
||||
* SIGSTOP/SIGCONT affects the entire server process, so these tests need
|
||||
* isolated servers to avoid interfering with other tests.
|
||||
*/
|
||||
async function runDedicatedServerTest(
|
||||
name: string,
|
||||
test: TestDefinition,
|
||||
serverPath: string,
|
||||
configPath: string
|
||||
): Promise<NamedTestResult> {
|
||||
const testLogger = new PrefixedLogger(logger, name);
|
||||
const server = new ServerControl(serverPath, configPath, testLogger);
|
||||
serverManager.track(server);
|
||||
|
||||
try {
|
||||
await server.start();
|
||||
const runner = new TestRunner(
|
||||
server,
|
||||
testLogger,
|
||||
TOKEN,
|
||||
server.remoteUri
|
||||
);
|
||||
const result = await runner.runTest(name, test);
|
||||
if (result.success) {
|
||||
logger.info(`PASSED: ${name} (${result.duration}ms)`);
|
||||
} else {
|
||||
logger.error(`FAILED: ${name} - ${result.error}`);
|
||||
}
|
||||
return { name, result };
|
||||
} finally {
|
||||
try {
|
||||
await server.stop();
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
serverManager.untrack(server);
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const projectRoot = findProjectRoot();
|
||||
const serverPath = path.join(projectRoot, SERVER_BINARY_PATH);
|
||||
if (!fs.existsSync(serverPath)) {
|
||||
logger.error(`Server binary not found at: ${serverPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const configPath = path.join(projectRoot, CONFIG_PATH);
|
||||
if (!fs.existsSync(configPath)) {
|
||||
logger.error(`Config file not found at: ${configPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { filter, concurrency } = parseArgs(process.argv);
|
||||
|
||||
const testsToRun: [string, TestDefinition][] = [];
|
||||
for (const [key, test] of Object.entries(TESTS)) {
|
||||
if (test) {
|
||||
if (
|
||||
filter !== undefined &&
|
||||
filter.length > 0 &&
|
||||
!key.includes(filter)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
testsToRun.push([key, test]);
|
||||
}
|
||||
}
|
||||
|
||||
if (testsToRun.length === 0) {
|
||||
logger.error(
|
||||
filter !== undefined && filter.length > 0
|
||||
? `No tests matched filter "${filter}"`
|
||||
: "No tests found"
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const regularTests = testsToRun.filter(([, t]) => !testUsesPauseServer(t));
|
||||
const pauseTests = testsToRun.filter(([, t]) => testUsesPauseServer(t));
|
||||
|
||||
logger.info(`Server: ${serverPath}`);
|
||||
logger.info(`Config: ${configPath}`);
|
||||
logger.info(
|
||||
`Tests: ${testsToRun.length} total (${regularTests.length} regular, ${pauseTests.length} server-pause)`
|
||||
);
|
||||
logger.info(`Concurrency: ${concurrency}`);
|
||||
|
||||
const allResults: NamedTestResult[] = [];
|
||||
|
||||
if (regularTests.length > 0) {
|
||||
logger.info(
|
||||
`\n--- Running ${regularTests.length} regular tests (shared server, concurrency ${concurrency}) ---`
|
||||
);
|
||||
const sharedServer = new ServerControl(serverPath, configPath, logger);
|
||||
serverManager.track(sharedServer);
|
||||
|
||||
try {
|
||||
await sharedServer.start();
|
||||
|
||||
const results = await runWithConcurrency(
|
||||
regularTests,
|
||||
concurrency,
|
||||
async ([name, test]) =>
|
||||
runSharedServerTest(name, test, sharedServer)
|
||||
);
|
||||
|
||||
allResults.push(...results);
|
||||
} finally {
|
||||
try {
|
||||
await sharedServer.stop();
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
`Error stopping shared server: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
}
|
||||
serverManager.untrack(sharedServer);
|
||||
}
|
||||
}
|
||||
|
||||
if (pauseTests.length > 0) {
|
||||
logger.info(
|
||||
`\n--- Running ${pauseTests.length} server-pause tests (dedicated servers, concurrency ${concurrency}) ---`
|
||||
);
|
||||
|
||||
const results = await runWithConcurrency(
|
||||
pauseTests,
|
||||
concurrency,
|
||||
async ([name, test]) =>
|
||||
runDedicatedServerTest(name, test, serverPath, configPath)
|
||||
);
|
||||
|
||||
allResults.push(...results);
|
||||
}
|
||||
|
||||
const passed = allResults.filter((r) => r.result.success);
|
||||
const failed = allResults.filter((r) => !r.result.success);
|
||||
|
||||
logger.info(
|
||||
`\n--- Results: ${passed.length}/${allResults.length} passed ---`
|
||||
);
|
||||
|
||||
if (failed.length > 0) {
|
||||
for (const { name, result } of failed) {
|
||||
logger.error(` FAILED: ${name}: ${result.error}`);
|
||||
}
|
||||
process.exit(1);
|
||||
} else {
|
||||
logger.info("All tests passed!");
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err: unknown) => {
|
||||
logger.error(`Unexpected error: ${err}`);
|
||||
process.exit(1);
|
||||
});
|
||||
17
frontend/deterministic-tests/src/consts.ts
Normal file
17
frontend/deterministic-tests/src/consts.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
export const TOKEN = "test-token-change-me";
|
||||
export const SERVER_BINARY_PATH = "sync-server/target/release/sync_server";
|
||||
export const CONFIG_PATH = "sync-server/config-e2e.yml";
|
||||
|
||||
export const STOP_TIMEOUT_MS = 5_000;
|
||||
export const CONVERGENCE_TIMEOUT_MS = 60_000;
|
||||
export const CONVERGENCE_RETRY_DELAY_MS = 500;
|
||||
export const AGENT_INIT_TIMEOUT_MS = 30_000;
|
||||
export const IS_SYNC_ENABLED_BY_DEFAULT = false;
|
||||
|
||||
export const WAIT_TIMEOUT_MS = 60_000;
|
||||
export const WEBSOCKET_CONNECT_TIMEOUT_MS = 10_000;
|
||||
export const WEBSOCKET_POLL_INTERVAL_MS = 50;
|
||||
|
||||
export const SERVER_READY_POLL_INTERVAL_MS = 100;
|
||||
export const SERVER_READY_MAX_ATTEMPTS = 50;
|
||||
export const SERVER_START_MAX_ATTEMPTS = 5;
|
||||
483
frontend/deterministic-tests/src/deterministic-agent.ts
Normal file
483
frontend/deterministic-tests/src/deterministic-agent.ts
Normal file
|
|
@ -0,0 +1,483 @@
|
|||
import type {
|
||||
HistoryEntry,
|
||||
StoredDatabase,
|
||||
SyncSettings,
|
||||
RelativePath,
|
||||
TextWithCursors
|
||||
} from "sync-client";
|
||||
import {
|
||||
SyncClient,
|
||||
SyncResetError,
|
||||
debugging,
|
||||
LogLevel,
|
||||
utils
|
||||
} from "sync-client";
|
||||
import { assert } from "./utils/assert";
|
||||
import { sleep } from "./utils/sleep";
|
||||
import { withTimeout } from "./utils/with-timeout";
|
||||
import {
|
||||
IS_SYNC_ENABLED_BY_DEFAULT,
|
||||
WAIT_TIMEOUT_MS,
|
||||
WEBSOCKET_CONNECT_TIMEOUT_MS,
|
||||
WEBSOCKET_POLL_INTERVAL_MS
|
||||
} from "./consts";
|
||||
import { ManagedWebSocketFactory } from "./managed-websocket";
|
||||
|
||||
export class DeterministicAgent extends debugging.InMemoryFileSystem {
|
||||
public readonly clientId: number;
|
||||
private readonly logger: (msg: string) => void;
|
||||
private client!: SyncClient;
|
||||
private data: Partial<{
|
||||
settings: Partial<SyncSettings>;
|
||||
database: Partial<StoredDatabase>;
|
||||
}> = {};
|
||||
private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT;
|
||||
private readonly syncErrors: Error[] = [];
|
||||
private readonly pendingSyncOperations = new Set<Promise<void>>();
|
||||
private readonly wsFactory = new ManagedWebSocketFactory();
|
||||
private nextWriteRename:
|
||||
| {
|
||||
oldPath: RelativePath;
|
||||
newPath: RelativePath;
|
||||
}
|
||||
| undefined;
|
||||
private nextCreateResponseDrop:
|
||||
| {
|
||||
dropped: Promise<void>;
|
||||
resolveDropped: () => void;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
public constructor(
|
||||
clientId: number,
|
||||
initialSettings: Partial<SyncSettings>,
|
||||
logger: (msg: string) => void
|
||||
) {
|
||||
super();
|
||||
this.clientId = clientId;
|
||||
this.logger = logger;
|
||||
this.data.settings = { ...initialSettings };
|
||||
}
|
||||
|
||||
public async init(
|
||||
fetchImplementation: typeof globalThis.fetch
|
||||
): Promise<void> {
|
||||
this.client = await SyncClient.create({
|
||||
fs: this,
|
||||
persistence: {
|
||||
load: async () => this.data,
|
||||
save: async (data) => void (this.data = data)
|
||||
},
|
||||
fetch: this.wrapFetch(fetchImplementation),
|
||||
webSocket: this.wsFactory.constructorFn
|
||||
});
|
||||
|
||||
this.client.logger.onLogEmitted.add((line) => {
|
||||
const prefix = `[Client ${this.clientId}]`;
|
||||
switch (line.level) {
|
||||
case LogLevel.ERROR:
|
||||
this.logger(`${prefix} ERROR: ${line.message}`);
|
||||
break;
|
||||
case LogLevel.WARNING:
|
||||
this.logger(`${prefix} WARN: ${line.message}`);
|
||||
break;
|
||||
case LogLevel.INFO:
|
||||
this.logger(`${prefix} INFO: ${line.message}`);
|
||||
break;
|
||||
case LogLevel.DEBUG:
|
||||
this.logger(`${prefix} DEBUG: ${line.message}`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
await this.client.start();
|
||||
|
||||
const connectionCheck = await this.client.checkConnection();
|
||||
assert(
|
||||
connectionCheck.isSuccessful,
|
||||
`Client ${this.clientId} connection check failed`
|
||||
);
|
||||
|
||||
if (this.isSyncEnabled) {
|
||||
await this.waitForWebSocket();
|
||||
}
|
||||
}
|
||||
|
||||
public pauseWebSocket(): void {
|
||||
this.log("Pausing WebSocket message delivery");
|
||||
this.wsFactory.pause();
|
||||
}
|
||||
|
||||
public resumeWebSocket(): void {
|
||||
this.log("Resuming WebSocket message delivery");
|
||||
this.wsFactory.resume();
|
||||
}
|
||||
|
||||
public dropNextCreateResponse(): void {
|
||||
assert(
|
||||
this.nextCreateResponseDrop === undefined,
|
||||
`Client ${this.clientId} already has a create response drop armed`
|
||||
);
|
||||
let resolveDropped!: () => void;
|
||||
const dropped = new Promise<void>((resolve) => {
|
||||
resolveDropped = resolve;
|
||||
});
|
||||
this.nextCreateResponseDrop = {
|
||||
dropped,
|
||||
resolveDropped
|
||||
};
|
||||
this.log("Armed next create response drop");
|
||||
}
|
||||
|
||||
public async waitForDroppedCreateResponse(): Promise<void> {
|
||||
assert(
|
||||
this.nextCreateResponseDrop !== undefined,
|
||||
`Client ${this.clientId} has no create response drop armed`
|
||||
);
|
||||
await withTimeout(
|
||||
this.nextCreateResponseDrop.dropped,
|
||||
WAIT_TIMEOUT_MS,
|
||||
`Client ${this.clientId} timed out waiting for create response drop`
|
||||
);
|
||||
this.log("Create response was dropped after server commit");
|
||||
}
|
||||
|
||||
public async waitForHistoryEntry(
|
||||
matches: (entry: HistoryEntry) => boolean,
|
||||
onMatch?: (entry: HistoryEntry) => void
|
||||
): Promise<void> {
|
||||
const existing = this.client.getHistoryEntries().find(matches);
|
||||
if (existing !== undefined) {
|
||||
onMatch?.(existing);
|
||||
return;
|
||||
}
|
||||
|
||||
await withTimeout(
|
||||
new Promise<void>((resolve) => {
|
||||
const unsubscribe = this.client.onSyncHistoryUpdated.add(() => {
|
||||
const entry = this.client
|
||||
.getHistoryEntries()
|
||||
.find(matches);
|
||||
if (entry === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
unsubscribe();
|
||||
onMatch?.(entry);
|
||||
resolve();
|
||||
});
|
||||
}),
|
||||
WAIT_TIMEOUT_MS,
|
||||
`Client ${this.clientId} timed out waiting for history entry`
|
||||
);
|
||||
}
|
||||
|
||||
public async waitForSync(): Promise<void> {
|
||||
this.log("Waiting for sync to complete...");
|
||||
// Drain agent-level sync operations first. These are the fire-and-forget
|
||||
// promises from enqueueSync() that call into the SyncClient's methods.
|
||||
// Without this, waitUntilFinished() might return before the SyncClient
|
||||
// has even been told about the operation.
|
||||
await this.drainPendingSyncOperations();
|
||||
await withTimeout(
|
||||
this.client.waitUntilFinished(),
|
||||
WAIT_TIMEOUT_MS,
|
||||
`Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms`
|
||||
);
|
||||
if (this.syncErrors.length > 0) {
|
||||
const errors = this.syncErrors.splice(0);
|
||||
throw new Error(
|
||||
`Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}`
|
||||
);
|
||||
}
|
||||
this.log("Sync complete");
|
||||
}
|
||||
|
||||
public async reset(): Promise<void> {
|
||||
this.log("Resetting client (clears tracked state, keeps disk files)");
|
||||
await this.drainPendingSyncOperations();
|
||||
await this.client.reset();
|
||||
if (this.isSyncEnabled) {
|
||||
await this.waitForWebSocket();
|
||||
}
|
||||
}
|
||||
|
||||
public async disableSync(): Promise<void> {
|
||||
this.log("Disabling sync");
|
||||
// Drain pending enqueued operations before disabling so the SyncClient
|
||||
// knows about all operations that were enqueued while sync was enabled.
|
||||
await this.drainPendingSyncOperations();
|
||||
await this.client.setSetting("isSyncEnabled", false);
|
||||
this.isSyncEnabled = false;
|
||||
// Wait for in-flight operations to drain. Disabling sync triggers
|
||||
// a reset, which aborts in-flight fetches with SyncResetError.
|
||||
try {
|
||||
await withTimeout(
|
||||
this.client.waitUntilFinished(),
|
||||
WAIT_TIMEOUT_MS,
|
||||
`Client ${this.clientId} disableSync drain timed out`
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "SyncResetError") {
|
||||
this.log("Disable sync drain interrupted by reset (expected)");
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async enableSync(): Promise<void> {
|
||||
this.log("Enabling sync");
|
||||
await this.client.setSetting("isSyncEnabled", true);
|
||||
this.isSyncEnabled = true;
|
||||
await this.waitForWebSocket();
|
||||
}
|
||||
|
||||
public async getFileContent(path: string): Promise<string> {
|
||||
const bytes = await this.read(path);
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
|
||||
public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void {
|
||||
assert(
|
||||
this.nextWriteRename === undefined,
|
||||
`Client ${this.clientId} already has a next-write rename armed`
|
||||
);
|
||||
this.nextWriteRename = { oldPath, newPath };
|
||||
this.log(`Armed next write rename: ${oldPath} -> ${newPath}`);
|
||||
}
|
||||
|
||||
public async cleanup(): Promise<void> {
|
||||
this.log("Cleaning up...");
|
||||
// Guard against uninitialized client (init() failed partway).
|
||||
// The class field uses `!:` so TS thinks this is always defined,
|
||||
// but at runtime it can be undefined when init() throws partway.
|
||||
const maybeClient = this.client as SyncClient | undefined;
|
||||
if (maybeClient === undefined) {
|
||||
this.log("Client not initialized, nothing to clean up");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await this.drainPendingSyncOperations();
|
||||
await withTimeout(
|
||||
this.client.waitUntilFinished(),
|
||||
WAIT_TIMEOUT_MS,
|
||||
`Client ${this.clientId} cleanup waitUntilFinished timed out`
|
||||
);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "SyncResetError") {
|
||||
this.log(`Cleanup interrupted by reset (expected): ${error}`);
|
||||
} else {
|
||||
this.log(`Cleanup waitUntilFinished failed: ${error}`);
|
||||
}
|
||||
}
|
||||
// Surface any background sync errors that arrived after the last
|
||||
// waitForSync (e.g. between the final assert-consistent and here).
|
||||
// Without this, regressions that fault the engine during the very
|
||||
// last step of a test would be silently swallowed.
|
||||
const pendingErrors = this.syncErrors.splice(0);
|
||||
await this.client.destroy();
|
||||
this.log("Cleanup complete");
|
||||
if (pendingErrors.length > 0) {
|
||||
throw new Error(
|
||||
`Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public override async read(path: RelativePath): Promise<Uint8Array> {
|
||||
await Promise.resolve();
|
||||
return super.read(path);
|
||||
}
|
||||
|
||||
public override async write(
|
||||
path: RelativePath,
|
||||
content: Uint8Array
|
||||
): Promise<void> {
|
||||
await Promise.resolve();
|
||||
const isNew = !this.files.has(path);
|
||||
await super.write(path, content);
|
||||
|
||||
if (this.isSyncEnabled && isNew) {
|
||||
this.enqueueSync(async () => {
|
||||
this.client.syncLocallyCreatedFile(path);
|
||||
});
|
||||
}
|
||||
|
||||
const nextWriteRename = this.nextWriteRename;
|
||||
if (
|
||||
nextWriteRename !== undefined &&
|
||||
nextWriteRename.oldPath === path
|
||||
) {
|
||||
this.nextWriteRename = undefined;
|
||||
await super.rename(
|
||||
nextWriteRename.oldPath,
|
||||
nextWriteRename.newPath
|
||||
);
|
||||
if (this.isSyncEnabled) {
|
||||
this.enqueueSync(async () => {
|
||||
this.client.syncLocallyUpdatedFile({
|
||||
oldPath: nextWriteRename.oldPath,
|
||||
relativePath: nextWriteRename.newPath
|
||||
});
|
||||
});
|
||||
}
|
||||
// The rename consumed `path`. Skip the post-update enqueue below
|
||||
// — it would send a syncLocallyUpdatedFile for a path that no
|
||||
// longer exists.
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.isSyncEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isNew) {
|
||||
this.enqueueSync(async () => {
|
||||
this.client.syncLocallyUpdatedFile({ relativePath: path });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public override async atomicUpdateText(
|
||||
path: RelativePath,
|
||||
updater: (current: TextWithCursors) => TextWithCursors
|
||||
): Promise<string> {
|
||||
const result = await super.atomicUpdateText(path, updater);
|
||||
if (this.isSyncEnabled) {
|
||||
this.enqueueSync(async () => {
|
||||
this.client.syncLocallyUpdatedFile({ relativePath: path });
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public override async delete(path: RelativePath): Promise<void> {
|
||||
await super.delete(path);
|
||||
if (this.isSyncEnabled) {
|
||||
this.enqueueSync(async () => {
|
||||
this.client.syncLocallyDeletedFile(path);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public override async rename(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath
|
||||
): Promise<void> {
|
||||
await super.rename(oldPath, newPath);
|
||||
if (this.isSyncEnabled) {
|
||||
this.enqueueSync(async () => {
|
||||
this.client.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: newPath
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async waitForWebSocket(): Promise<void> {
|
||||
const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS;
|
||||
while (!this.client.isWebSocketConnected && Date.now() < deadline) {
|
||||
await sleep(WEBSOCKET_POLL_INTERVAL_MS);
|
||||
}
|
||||
assert(
|
||||
this.client.isWebSocketConnected,
|
||||
`Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until all agent-level enqueued sync operations have completed.
|
||||
* Uses a loop because completing one operation can trigger new enqueues.
|
||||
*/
|
||||
private async drainPendingSyncOperations(): Promise<void> {
|
||||
while (this.pendingSyncOperations.size > 0) {
|
||||
await utils.awaitAll([...this.pendingSyncOperations]);
|
||||
}
|
||||
}
|
||||
|
||||
private enqueueSync(operation: () => Promise<void>): void {
|
||||
const promise = this.executeSyncOperation(operation).catch(
|
||||
(error: unknown) => {
|
||||
const err =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
this.log(`Background sync failed: ${err.message}`);
|
||||
this.syncErrors.push(err);
|
||||
}
|
||||
);
|
||||
this.pendingSyncOperations.add(promise);
|
||||
void promise.finally(() => {
|
||||
this.pendingSyncOperations.delete(promise);
|
||||
});
|
||||
}
|
||||
|
||||
private async executeSyncOperation(
|
||||
operation: () => Promise<void>
|
||||
): Promise<void> {
|
||||
try {
|
||||
await operation();
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "SyncResetError") {
|
||||
this.log(`Sync operation interrupted by reset: ${error}`);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes("has been destroyed")
|
||||
) {
|
||||
this.log(`Sync operation interrupted by destroy: ${error}`);
|
||||
return;
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private log(message: string): void {
|
||||
this.logger(`[Client ${this.clientId}] ${message}`);
|
||||
}
|
||||
|
||||
private wrapFetch(
|
||||
fetchImplementation: typeof globalThis.fetch
|
||||
): typeof globalThis.fetch {
|
||||
return async (input, init) => {
|
||||
const response = await fetchImplementation(input, init);
|
||||
const drop = this.nextCreateResponseDrop;
|
||||
if (
|
||||
drop !== undefined &&
|
||||
DeterministicAgent.isCreateDocumentRequest(input, init)
|
||||
) {
|
||||
this.nextCreateResponseDrop = undefined;
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// Best-effort — body may already be consumed/closed.
|
||||
}
|
||||
drop.resolveDropped();
|
||||
throw new SyncResetError();
|
||||
}
|
||||
return response;
|
||||
};
|
||||
}
|
||||
|
||||
private static isCreateDocumentRequest(
|
||||
input: RequestInfo | URL,
|
||||
init: RequestInit | undefined
|
||||
): boolean {
|
||||
const method =
|
||||
init?.method ??
|
||||
(typeof Request !== "undefined" && input instanceof Request
|
||||
? input.method
|
||||
: "GET");
|
||||
if (method.toUpperCase() !== "POST") {
|
||||
return false;
|
||||
}
|
||||
|
||||
const url =
|
||||
input instanceof URL
|
||||
? input
|
||||
: new URL(typeof input === "string" ? input : input.url);
|
||||
return /\/documents\/?$/.test(url.pathname);
|
||||
}
|
||||
}
|
||||
245
frontend/deterministic-tests/src/managed-websocket.ts
Normal file
245
frontend/deterministic-tests/src/managed-websocket.ts
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
/**
|
||||
* A WebSocket wrapper that can pause and resume message delivery.
|
||||
* When paused, incoming messages are buffered. When resumed, buffered
|
||||
* messages are delivered in order via the onmessage handler.
|
||||
*
|
||||
* Member layout follows typescript-eslint default member-ordering: all
|
||||
* accessor properties are declared with `declare` and wired through the
|
||||
* constructor using Object.defineProperty so we don't need conflicting
|
||||
* get/set accessor pairs.
|
||||
*/
|
||||
class ManagedWebSocket implements WebSocket {
|
||||
public static readonly CONNECTING = WebSocket.CONNECTING;
|
||||
public static readonly OPEN = WebSocket.OPEN;
|
||||
public static readonly CLOSING = WebSocket.CLOSING;
|
||||
public static readonly CLOSED = WebSocket.CLOSED;
|
||||
|
||||
public readonly CONNECTING = WebSocket.CONNECTING;
|
||||
public readonly OPEN = WebSocket.OPEN;
|
||||
public readonly CLOSING = WebSocket.CLOSING;
|
||||
public readonly CLOSED = WebSocket.CLOSED;
|
||||
|
||||
declare public readonly readyState: number;
|
||||
declare public readonly url: string;
|
||||
declare public readonly protocol: string;
|
||||
declare public readonly extensions: string;
|
||||
declare public readonly bufferedAmount: number;
|
||||
declare public binaryType: BinaryType;
|
||||
declare public onopen: ((this: WebSocket, ev: Event) => unknown) | null;
|
||||
declare public onclose:
|
||||
| ((this: WebSocket, ev: CloseEvent) => unknown)
|
||||
| null;
|
||||
declare public onerror: ((this: WebSocket, ev: Event) => unknown) | null;
|
||||
declare public onmessage:
|
||||
| ((this: WebSocket, ev: MessageEvent) => unknown)
|
||||
| null;
|
||||
|
||||
private readonly ws: WebSocket;
|
||||
private readonly bufferedMessages: MessageEvent[] = [];
|
||||
private paused = false;
|
||||
private externalOnMessage: ((event: MessageEvent) => unknown) | null = null;
|
||||
|
||||
public constructor(url: string | URL, protocols?: string | string[]) {
|
||||
this.ws = new WebSocket(url, protocols);
|
||||
|
||||
const { ws } = this;
|
||||
Object.defineProperties(this, {
|
||||
readyState: {
|
||||
get: (): number => ws.readyState,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
url: {
|
||||
get: (): string => ws.url,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
protocol: {
|
||||
get: (): string => ws.protocol,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
extensions: {
|
||||
get: (): string => ws.extensions,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
bufferedAmount: {
|
||||
get: (): number => ws.bufferedAmount,
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
binaryType: {
|
||||
get: (): BinaryType => ws.binaryType,
|
||||
set: (v: BinaryType): void => {
|
||||
ws.binaryType = v;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
onopen: {
|
||||
get: (): ((this: WebSocket, ev: Event) => unknown) | null =>
|
||||
ws.onopen,
|
||||
set: (
|
||||
h: ((this: WebSocket, ev: Event) => unknown) | null
|
||||
): void => {
|
||||
ws.onopen = h;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
onclose: {
|
||||
get: ():
|
||||
| ((this: WebSocket, ev: CloseEvent) => unknown)
|
||||
| null => ws.onclose,
|
||||
set: (
|
||||
h: ((this: WebSocket, ev: CloseEvent) => unknown) | null
|
||||
): void => {
|
||||
ws.onclose = h;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
onerror: {
|
||||
get: (): ((this: WebSocket, ev: Event) => unknown) | null =>
|
||||
ws.onerror,
|
||||
set: (
|
||||
h: ((this: WebSocket, ev: Event) => unknown) | null
|
||||
): void => {
|
||||
ws.onerror = h;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
},
|
||||
onmessage: {
|
||||
get: ():
|
||||
| ((this: WebSocket, ev: MessageEvent) => unknown)
|
||||
| null => this.externalOnMessage,
|
||||
set: (
|
||||
h: ((this: WebSocket, ev: MessageEvent) => unknown) | null
|
||||
): void => {
|
||||
this.externalOnMessage = h;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true
|
||||
}
|
||||
});
|
||||
|
||||
this.ws.onmessage = (event: MessageEvent): void => {
|
||||
if (this.paused) {
|
||||
this.bufferedMessages.push(event);
|
||||
} else {
|
||||
this.externalOnMessage?.(event);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public pause(): void {
|
||||
this.paused = true;
|
||||
}
|
||||
|
||||
public resume(): void {
|
||||
// Drain buffered messages BEFORE flipping `paused` to false.
|
||||
// If `externalOnMessage` is async (its return type is `unknown`),
|
||||
// dispatch yields control between buffered messages, and a fresh
|
||||
// live `ws.onmessage` event firing during that yield would jump
|
||||
// ahead of unprocessed buffered messages — silently reordering
|
||||
// events relative to the wire. Keeping `paused = true` during the
|
||||
// drain forces the live handler to keep buffering, so we splice
|
||||
// those late arrivals onto the tail and dispatch them in order.
|
||||
while (this.bufferedMessages.length > 0) {
|
||||
const messages = this.bufferedMessages.splice(0);
|
||||
for (const msg of messages) {
|
||||
this.externalOnMessage?.(msg);
|
||||
}
|
||||
}
|
||||
this.paused = false;
|
||||
}
|
||||
|
||||
public send(data: string | ArrayBufferLike | Blob | ArrayBufferView): void {
|
||||
this.ws.send(data);
|
||||
}
|
||||
|
||||
public close(code?: number, reason?: string): void {
|
||||
this.ws.close(code, reason);
|
||||
}
|
||||
|
||||
public addEventListener(
|
||||
...args: Parameters<WebSocket["addEventListener"]>
|
||||
): void {
|
||||
// Only the `.onmessage` setter routes through the pause buffer.
|
||||
// If sync-client ever attaches "message" listeners via
|
||||
// addEventListener instead, those messages would bypass pause/resume
|
||||
// and deterministic tests would silently lose their fault injection.
|
||||
if (args[0] === "message") {
|
||||
throw new Error(
|
||||
"ManagedWebSocket: addEventListener('message') bypasses the " +
|
||||
"pause buffer. Use the .onmessage setter instead, or " +
|
||||
"extend ManagedWebSocket to route message listeners."
|
||||
);
|
||||
}
|
||||
this.ws.addEventListener(...args);
|
||||
}
|
||||
|
||||
public removeEventListener(
|
||||
...args: Parameters<WebSocket["removeEventListener"]>
|
||||
): void {
|
||||
this.ws.removeEventListener(...args);
|
||||
}
|
||||
|
||||
public dispatchEvent(event: Event): boolean {
|
||||
return this.ws.dispatchEvent(event);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory that creates ManagedWebSocket instances and tracks them
|
||||
* for pause/resume control from the test harness
|
||||
*/
|
||||
export class ManagedWebSocketFactory {
|
||||
// Append-only: closed sockets stay tracked. Bounded per test (one
|
||||
// factory per agent, each test discards its agents on cleanup), so
|
||||
// not a real leak — but iterating over closed instances on
|
||||
// pause/resume is a deliberate no-op since their `.onmessage` is
|
||||
// already detached.
|
||||
private readonly instances: ManagedWebSocket[] = [];
|
||||
// Sticky pause state: applied to current instances on `pause()` AND
|
||||
// to any new instance created later (e.g. WS reconnect after a
|
||||
// `disable-sync` / `reset` cycle). Without this, a test pausing the
|
||||
// WS before the agent reconnects would silently see the new socket
|
||||
// start un-paused and miss the messages it meant to buffer.
|
||||
private currentlyPaused = false;
|
||||
|
||||
public get constructorFn(): typeof globalThis.WebSocket {
|
||||
const trackInstance = (instance: ManagedWebSocket): void => {
|
||||
this.instances.push(instance);
|
||||
if (this.currentlyPaused) {
|
||||
instance.pause();
|
||||
}
|
||||
};
|
||||
class TrackedManagedWebSocket extends ManagedWebSocket {
|
||||
public constructor(
|
||||
url: string | URL,
|
||||
protocols?: string | string[]
|
||||
) {
|
||||
super(url, protocols);
|
||||
trackInstance(this);
|
||||
}
|
||||
}
|
||||
return TrackedManagedWebSocket;
|
||||
}
|
||||
|
||||
public pause(): void {
|
||||
this.currentlyPaused = true;
|
||||
for (const ws of this.instances) {
|
||||
ws.pause();
|
||||
}
|
||||
}
|
||||
|
||||
public resume(): void {
|
||||
this.currentlyPaused = false;
|
||||
for (const ws of this.instances) {
|
||||
ws.resume();
|
||||
}
|
||||
}
|
||||
}
|
||||
43
frontend/deterministic-tests/src/parse-args.ts
Normal file
43
frontend/deterministic-tests/src/parse-args.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
import * as os from "node:os";
|
||||
import { Command, InvalidArgumentError } from "commander";
|
||||
|
||||
export interface CliArgs {
|
||||
filter: string | undefined;
|
||||
concurrency: number;
|
||||
}
|
||||
|
||||
function parsePositiveInt(value: string): number {
|
||||
const n = parseInt(value, 10);
|
||||
if (isNaN(n) || n <= 0) {
|
||||
throw new InvalidArgumentError("must be a positive integer");
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): CliArgs {
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name("deterministic-tests")
|
||||
.description("Scripted multi-client sync tests against a real server")
|
||||
.option(
|
||||
"-f, --filter <substring>",
|
||||
"Run only tests whose name contains this substring"
|
||||
)
|
||||
.option(
|
||||
"-j, --concurrency <number>",
|
||||
"Number of tests to run in parallel",
|
||||
parsePositiveInt,
|
||||
os.cpus().length
|
||||
);
|
||||
|
||||
program.parse(argv);
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
|
||||
const opts = program.opts();
|
||||
const filter = opts.filter as string | undefined;
|
||||
const concurrency = opts.concurrency as number;
|
||||
/* eslint-enable @typescript-eslint/no-unsafe-type-assertion */
|
||||
|
||||
return { filter, concurrency };
|
||||
}
|
||||
28
frontend/deterministic-tests/src/prefixed-logger.ts
Normal file
28
frontend/deterministic-tests/src/prefixed-logger.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import { Logger } from "sync-client";
|
||||
|
||||
export class PrefixedLogger extends Logger {
|
||||
private readonly base: Logger;
|
||||
private readonly prefix: string;
|
||||
|
||||
public constructor(base: Logger, prefix: string) {
|
||||
super();
|
||||
this.base = base;
|
||||
this.prefix = prefix;
|
||||
}
|
||||
|
||||
public override debug(message: string): void {
|
||||
this.base.debug(`[${this.prefix}] ${message}`);
|
||||
}
|
||||
|
||||
public override info(message: string): void {
|
||||
this.base.info(`[${this.prefix}] ${message}`);
|
||||
}
|
||||
|
||||
public override warn(message: string): void {
|
||||
this.base.warn(`[${this.prefix}] ${message}`);
|
||||
}
|
||||
|
||||
public override error(message: string): void {
|
||||
this.base.error(`[${this.prefix}] ${message}`);
|
||||
}
|
||||
}
|
||||
33
frontend/deterministic-tests/src/run-with-concurrency.ts
Normal file
33
frontend/deterministic-tests/src/run-with-concurrency.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
export async function runWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
fn: (item: T) => Promise<R>
|
||||
): Promise<R[]> {
|
||||
const results: R[] = [];
|
||||
const errors: unknown[] = [];
|
||||
const executing = new Set<Promise<void>>();
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const index = i;
|
||||
const p = fn(items[index])
|
||||
.then((result) => {
|
||||
results[index] = result;
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
errors.push(error);
|
||||
})
|
||||
.finally(() => executing.delete(p));
|
||||
executing.add(p);
|
||||
if (executing.size >= concurrency) {
|
||||
await Promise.race(executing);
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
await Promise.all(executing);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw errors[0];
|
||||
}
|
||||
return results;
|
||||
}
|
||||
296
frontend/deterministic-tests/src/server-control.ts
Normal file
296
frontend/deterministic-tests/src/server-control.ts
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { sleep } from "./utils/sleep";
|
||||
import { findFreePort } from "./utils/find-free-port";
|
||||
import type { Logger } from "sync-client";
|
||||
import {
|
||||
STOP_TIMEOUT_MS,
|
||||
SERVER_READY_POLL_INTERVAL_MS,
|
||||
SERVER_READY_MAX_ATTEMPTS,
|
||||
SERVER_START_MAX_ATTEMPTS
|
||||
} from "./consts";
|
||||
|
||||
export class ServerControl {
|
||||
private process: ChildProcess | null = null;
|
||||
private readonly serverPath: string;
|
||||
private readonly baseConfigPath: string;
|
||||
private readonly logger: Logger;
|
||||
private _port: number | undefined;
|
||||
private tempDir: string | undefined;
|
||||
private _isPaused = false;
|
||||
|
||||
public constructor(serverPath: string, configPath: string, logger: Logger) {
|
||||
this.serverPath = serverPath;
|
||||
this.baseConfigPath = configPath;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public get port(): number {
|
||||
if (this._port === undefined) {
|
||||
throw new Error("Server has not been started yet");
|
||||
}
|
||||
return this._port;
|
||||
}
|
||||
|
||||
public get remoteUri(): string {
|
||||
return `http://localhost:${this.port}`;
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
if (this.process !== null) {
|
||||
throw new Error("Server is already running");
|
||||
}
|
||||
|
||||
// Retry on bind failure: findFreePort closes its probe before we
|
||||
// spawn, so under heavy parallelism another process can grab the
|
||||
// same port. Each attempt picks a fresh port.
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= SERVER_START_MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
await this.startOnce();
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
this.logger.warn(
|
||||
`Server start attempt ${attempt}/${SERVER_START_MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
// startOnce already cleaned up its child + tempdir on failure.
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`Server failed to start after ${SERVER_START_MAX_ATTEMPTS} attempts: ${lastError instanceof Error ? lastError.message : String(lastError)}`,
|
||||
{ cause: lastError instanceof Error ? lastError : undefined }
|
||||
);
|
||||
}
|
||||
|
||||
private async startOnce(): Promise<void> {
|
||||
const reservation = await findFreePort();
|
||||
this._port = reservation.port;
|
||||
const tmpBase = os.tmpdir();
|
||||
this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-"));
|
||||
const tempConfigPath = path.join(this.tempDir, "config.yml");
|
||||
const dbDir = path.join(this.tempDir, "databases");
|
||||
|
||||
this.writeConfigFile(tempConfigPath, dbDir);
|
||||
|
||||
this.logger.info(
|
||||
`Starting server: ${this.serverPath} (port ${this._port})`
|
||||
);
|
||||
|
||||
// Release the port reservation right before spawning to minimize
|
||||
// the TOCTOU window between port discovery and server binding.
|
||||
reservation.release();
|
||||
|
||||
this.process = spawn(this.serverPath, [tempConfigPath], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
detached: false
|
||||
});
|
||||
|
||||
this.process.stdout?.on("data", (data: Buffer) => {
|
||||
this.logger.info(`[SERVER] ${data.toString().trim()}`);
|
||||
});
|
||||
|
||||
this.process.stderr?.on("data", (data: Buffer) => {
|
||||
this.logger.info(`[SERVER] ${data.toString().trim()}`);
|
||||
});
|
||||
|
||||
this.process.on("error", (err) => {
|
||||
this.logger.error(`[SERVER] Process error: ${err.message}`);
|
||||
});
|
||||
|
||||
const currentProcess = this.process;
|
||||
currentProcess.on("exit", (code, signal) => {
|
||||
this.logger.info(
|
||||
`Server exited with code ${code}, signal ${signal}`
|
||||
);
|
||||
// Only clear state if this handler is for the current process.
|
||||
// A fast stop→start cycle could create a new process before this
|
||||
// handler fires — clearing state here would corrupt the new one.
|
||||
if (this.process === currentProcess) {
|
||||
this.process = null;
|
||||
this._isPaused = false;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await this.waitForReady();
|
||||
} catch (error) {
|
||||
// Kill the spawned process if it failed to become ready,
|
||||
// preventing a zombie process from lingering.
|
||||
try {
|
||||
await this.stop();
|
||||
} catch {
|
||||
// Best-effort cleanup
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public async waitForReady(
|
||||
maxAttempts: number = SERVER_READY_MAX_ATTEMPTS
|
||||
): Promise<void> {
|
||||
const pingUrl = `${this.remoteUri}/vaults/test/ping`;
|
||||
for (let i = 0; i < maxAttempts; i++) {
|
||||
if (this.process?.exitCode !== null) {
|
||||
throw new Error(
|
||||
"Server process died while waiting for it to become ready"
|
||||
);
|
||||
}
|
||||
try {
|
||||
const response = await fetch(pingUrl);
|
||||
if (response.ok) {
|
||||
this.logger.info("[SERVER] Ready");
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// Server not ready yet, continue polling
|
||||
}
|
||||
await sleep(SERVER_READY_POLL_INTERVAL_MS);
|
||||
}
|
||||
throw new Error("Server failed to start within timeout");
|
||||
}
|
||||
|
||||
public pause(): void {
|
||||
if (this.process?.pid === undefined) {
|
||||
throw new Error("Server is not running");
|
||||
}
|
||||
if (this._isPaused) {
|
||||
this.logger.warn("Server is already paused, skipping double-pause");
|
||||
return;
|
||||
}
|
||||
this.logger.info("Server pausing...");
|
||||
try {
|
||||
process.kill(this.process.pid, "SIGSTOP");
|
||||
this._isPaused = true;
|
||||
this.logger.info("Server paused (SIGSTOP sent)");
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Failed to pause server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public resume(): void {
|
||||
if (this.process?.pid === undefined) {
|
||||
throw new Error("Server is not running");
|
||||
}
|
||||
if (!this._isPaused) {
|
||||
return;
|
||||
}
|
||||
this.logger.info("Server resuming...");
|
||||
try {
|
||||
process.kill(this.process.pid, "SIGCONT");
|
||||
this._isPaused = false;
|
||||
this.logger.info("Server resumed (SIGCONT sent)");
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Failed to resume server (pid ${this.process.pid}): ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async stop(): Promise<void> {
|
||||
const proc = this.process;
|
||||
if (proc?.pid === undefined) {
|
||||
this.cleanupTempDir();
|
||||
return;
|
||||
}
|
||||
|
||||
// Resume if paused — a SIGSTOP'd process ignores SIGKILL
|
||||
if (this._isPaused) {
|
||||
try {
|
||||
process.kill(proc.pid, "SIGCONT");
|
||||
} catch {
|
||||
// Process may already be gone
|
||||
}
|
||||
this._isPaused = false;
|
||||
}
|
||||
|
||||
this.logger.info("Server stopping...");
|
||||
|
||||
// Set up a promise that resolves when the process actually exits.
|
||||
const exitPromise = new Promise<void>((resolve) => {
|
||||
if (proc.exitCode !== null) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
proc.on("exit", () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
process.kill(proc.pid, "SIGKILL");
|
||||
} catch {
|
||||
// Process already gone
|
||||
}
|
||||
|
||||
// Wait for the process to actually exit before cleaning up,
|
||||
// with a 5s safety timeout to avoid hanging forever.
|
||||
await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]);
|
||||
|
||||
this.process = null;
|
||||
this._isPaused = false;
|
||||
this.cleanupTempDir();
|
||||
}
|
||||
|
||||
public isRunning(): boolean {
|
||||
const proc = this.process;
|
||||
return (
|
||||
proc !== null &&
|
||||
proc.pid !== undefined &&
|
||||
proc.exitCode === null &&
|
||||
proc.signalCode === null
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronously SIGCONT-then-SIGKILL the child process. Safe to call
|
||||
* from a `process.on("exit", ...)` handler, where async work cannot
|
||||
* run. Used as a last-resort cleanup so a SIGSTOP'd server doesn't
|
||||
* outlive the test runner and wedge the next CI invocation.
|
||||
*/
|
||||
public forceKillSync(): void {
|
||||
const proc = this.process;
|
||||
if (proc?.pid === undefined) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
process.kill(proc.pid, "SIGCONT");
|
||||
} catch {
|
||||
// Process may already be gone or never paused.
|
||||
}
|
||||
try {
|
||||
process.kill(proc.pid, "SIGKILL");
|
||||
} catch {
|
||||
// Process already gone.
|
||||
}
|
||||
}
|
||||
|
||||
private writeConfigFile(destPath: string, dbDir: string): void {
|
||||
// Assumes config-e2e.yml has exactly one 2-space-indented `port:` and
|
||||
// one `databases_directory_path:` (under `server:` and `database:`
|
||||
// respectively)
|
||||
const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8");
|
||||
const config = baseConfig
|
||||
.replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`)
|
||||
.replace(
|
||||
/^\s*databases_directory_path:\s*.+/m,
|
||||
` databases_directory_path: ${dbDir}`
|
||||
);
|
||||
fs.writeFileSync(destPath, config);
|
||||
}
|
||||
|
||||
private cleanupTempDir(): void {
|
||||
if (this.tempDir !== undefined) {
|
||||
try {
|
||||
fs.rmSync(this.tempDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup
|
||||
}
|
||||
this.tempDir = undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
71
frontend/deterministic-tests/src/server-manager.ts
Normal file
71
frontend/deterministic-tests/src/server-manager.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import type { ServerControl } from "./server-control";
|
||||
import type { Logger } from "sync-client";
|
||||
|
||||
export class ServerManager {
|
||||
private readonly activeServers = new Set<ServerControl>();
|
||||
private readonly logger: Logger;
|
||||
private isShuttingDown = false;
|
||||
|
||||
public constructor(logger: Logger) {
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
public track(server: ServerControl): void {
|
||||
this.activeServers.add(server);
|
||||
}
|
||||
|
||||
public untrack(server: ServerControl): void {
|
||||
this.activeServers.delete(server);
|
||||
}
|
||||
|
||||
public async stopAll(): Promise<void> {
|
||||
if (this.isShuttingDown) {
|
||||
return;
|
||||
}
|
||||
this.isShuttingDown = true;
|
||||
|
||||
const servers = Array.from(this.activeServers);
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
await Promise.all(
|
||||
servers.map(async (server) => {
|
||||
try {
|
||||
await server.stop();
|
||||
} catch {
|
||||
// Best-effort cleanup during shutdown
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
public installSignalHandlers(): void {
|
||||
process.on("SIGINT", () => {
|
||||
this.logger.info("Received SIGINT, shutting down...");
|
||||
void this.stopAll()
|
||||
.catch(() => {
|
||||
/* no-op */
|
||||
})
|
||||
.then(() => process.exit(130));
|
||||
});
|
||||
|
||||
process.on("SIGTERM", () => {
|
||||
this.logger.info("Received SIGTERM, shutting down...");
|
||||
void this.stopAll()
|
||||
.catch(() => {
|
||||
/* no-op */
|
||||
})
|
||||
.then(() => process.exit(143));
|
||||
});
|
||||
|
||||
// Last-resort synchronous cleanup. Runs even when the process is
|
||||
// exiting via process.exit() from unhandledRejection /
|
||||
// uncaughtException — paths where async stopAll() cannot complete.
|
||||
// SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the
|
||||
// kernel keeps them as zombies holding the test's tmpdir, and the
|
||||
// next CI run can't reuse the port.
|
||||
process.on("exit", () => {
|
||||
for (const server of this.activeServers) {
|
||||
server.forceKillSync();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
49
frontend/deterministic-tests/src/test-definition.ts
Normal file
49
frontend/deterministic-tests/src/test-definition.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import type { AssertableState } from "./utils/assertable-state";
|
||||
|
||||
export interface ClientState {
|
||||
files: Map<string, string>;
|
||||
clientFiles: Map<string, string>[];
|
||||
}
|
||||
|
||||
export type TestStep =
|
||||
| { type: "create"; client: number; path: string; content: string }
|
||||
| { type: "update"; client: number; path: string; content: string }
|
||||
| { type: "rename"; client: number; oldPath: string; newPath: string }
|
||||
| {
|
||||
type: "rename-next-write";
|
||||
client: number;
|
||||
oldPath: string;
|
||||
newPath: string;
|
||||
}
|
||||
| { type: "delete"; client: number; path: string }
|
||||
| { type: "sync"; client?: number }
|
||||
| { type: "disable-sync"; client: number }
|
||||
| { type: "enable-sync"; client: number }
|
||||
| { type: "pause-server" }
|
||||
| { type: "resume-server" }
|
||||
| {
|
||||
type: "resume-server-until-history-then-pause";
|
||||
client: number;
|
||||
syncType: "CREATE" | "UPDATE" | "DELETE";
|
||||
path: string;
|
||||
}
|
||||
| { type: "barrier" }
|
||||
| { type: "assert-consistent"; verify?: (state: AssertableState) => void }
|
||||
| { type: "pause-websocket"; client: number }
|
||||
| { type: "resume-websocket"; client: number }
|
||||
| { type: "drop-next-create-response"; client: number }
|
||||
| { type: "wait-for-dropped-create-response"; client: number }
|
||||
| { type: "sleep"; ms: number }
|
||||
| { type: "reset"; client: number };
|
||||
|
||||
export interface TestDefinition {
|
||||
description?: string;
|
||||
clients: number;
|
||||
steps: TestStep[];
|
||||
}
|
||||
|
||||
export interface TestResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
duration?: number;
|
||||
}
|
||||
245
frontend/deterministic-tests/src/test-registry.ts
Normal file
245
frontend/deterministic-tests/src/test-registry.ts
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
import type { TestDefinition } from "./test-definition";
|
||||
import { renameCreateConflictTest } from "./tests/rename-create-conflict.test";
|
||||
import { renameChainTest } from "./tests/rename-chain.test";
|
||||
import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test";
|
||||
import { deleteRenameConflictTest } from "./tests/delete-rename-conflict.test";
|
||||
import { multiFileOperationsTest } from "./tests/multi-file-operations.test";
|
||||
import { deleteRecreateSamePathTest } from "./tests/delete-recreate-same-path.test";
|
||||
import { offlineRenameAndEditTest } from "./tests/offline-rename-and-edit.test";
|
||||
import { simultaneousCreateDeleteSamePathTest } from "./tests/simultaneous-create-delete-same-path.test";
|
||||
import { idempotencyAfterServerPauseTest } from "./tests/idempotency-after-server-pause.test";
|
||||
import { sequentialCreateDuplicateContentTest } from "./tests/sequential-create-duplicate-content.test";
|
||||
import { mcThreeClientRenameOfflineUpdateTest } from "./tests/mc-three-client-rename-offline-update.test";
|
||||
import { mcMultiDeleteOfflineRenameTest } from "./tests/mc-multi-delete-offline-rename.test";
|
||||
import { mcCrossCreateRenameSameTargetTest } from "./tests/mc-cross-create-rename-same-target.test";
|
||||
import { mcDeleteThenOfflineRenameTest } from "./tests/mc-delete-then-offline-rename.test";
|
||||
import { offlineMixedOperationsTest } from "./tests/offline-mixed-operations.test";
|
||||
import { offlineConcurrentRenamesTest } from "./tests/offline-concurrent-renames.test";
|
||||
import { offlineMultipleEditsTest } from "./tests/offline-multiple-edits.test";
|
||||
import { serverPauseBothClientsCreateTest } from "./tests/server-pause-both-clients-create.test";
|
||||
import { serverPauseUpdateAndCreateTest } from "./tests/server-pause-update-and-create.test";
|
||||
import { renameSwapTest } from "./tests/rename-swap.test";
|
||||
import { renameCircularTest } from "./tests/rename-circular.test";
|
||||
import { renameRoundtripTest } from "./tests/rename-roundtrip.test";
|
||||
import { offlineRenameRemoteCreateOldPathTest } from "./tests/offline-rename-remote-create-old-path.test";
|
||||
import { offlineEditRemoteRenameTest } from "./tests/offline-edit-remote-rename.test";
|
||||
import { renameChainThenDeleteTest } from "./tests/rename-chain-then-delete.test";
|
||||
import { offlineDeleteRemoteRenameTest } from "./tests/offline-delete-remote-rename.test";
|
||||
import { overlappingEditsSameSectionTest } from "./tests/overlapping-edits-same-section.test";
|
||||
import { rapidUpdatesAfterMergeTest } from "./tests/rapid-updates-after-merge.test";
|
||||
import { deleteRecreateConcurrentUpdateTest } from "./tests/delete-recreate-concurrent-update.test";
|
||||
import { moveAndConcurrentRemoteUpdateTest } from "./tests/move-and-concurrent-remote-update.test";
|
||||
import { offlineDeleteVsRemoteUpdateTest } from "./tests/offline-delete-vs-remote-update.test";
|
||||
import { doubleOfflineCycleTest } from "./tests/double-offline-cycle.test";
|
||||
import { serverPauseRenameEditResumeTest } from "./tests/server-pause-rename-edit-resume.test";
|
||||
import { offlineUpdateBothThenDeleteOneTest } from "./tests/offline-update-both-then-delete-one.test";
|
||||
import { offlineCreateSamePathMergeableTest } from "./tests/offline-create-same-path-mergeable.test";
|
||||
import { deleteDuringPendingCreateTest } from "./tests/delete-during-pending-create.test";
|
||||
import { threeClientRenameCreateDeleteTest } from "./tests/three-client-rename-create-delete.test";
|
||||
import { renameToPathOfUnconfirmedDeleteTest } from "./tests/rename-to-path-of-unconfirmed-delete.test";
|
||||
import { offlineEditThenMoveSameContentTest } from "./tests/offline-edit-then-move-same-content.test";
|
||||
import { rapidCreateUpdateDeleteCycleTest } from "./tests/rapid-create-update-delete-cycle.test";
|
||||
import { serverPauseBothEditSameFileTest } from "./tests/server-pause-both-edit-same-file.test";
|
||||
import { deleteRecreateDifferentContentTest } from "./tests/delete-recreate-different-content.test";
|
||||
import { updateDuringCreateProcessingTest } from "./tests/update-during-create-processing.test";
|
||||
import { offlineMoveThenRemoteDeleteTest } from "./tests/offline-move-then-remote-delete.test";
|
||||
import { resetClearsRecentlyDeletedResurrectionTest } from "./tests/reset-clears-recently-deleted-resurrection.test";
|
||||
import { moveThenDeleteStalePathTest } from "./tests/move-then-delete-stale-path.test";
|
||||
import { interruptedDeleteRetryTest } from "./tests/interrupted-delete-retry.test";
|
||||
import { updateDoesNotSurviveRemoteDeleteTest } from "./tests/update-does-not-survive-remote-delete.test";
|
||||
import { movePreservesRemoteUpdateTest } from "./tests/move-preserves-remote-update.test";
|
||||
import { recentlyDeletedClearedOnReconnectTest } from "./tests/recently-deleted-cleared-on-reconnect.test";
|
||||
import { watermarkAdvancesOnSkipTest } from "./tests/watermark-advances-on-skip.test";
|
||||
import { watermarkGapRemoteUpdateNotRecordedTest } from "./tests/watermark-gap-remote-update-not-recorded.test";
|
||||
import { queueResetLosesCoalescedLocalEditTest } from "./tests/queue-reset-loses-coalesced-local-edit.test";
|
||||
import { renameToPendingPathFallbackTest } from "./tests/rename-to-pending-path-fallback.test";
|
||||
import { moveRemoteUpdateRevertsRenameTest } from "./tests/move-remote-update-reverts-rename.test";
|
||||
import { localEditLostDuringCreateMergeTest } from "./tests/local-edit-lost-during-create-merge.test";
|
||||
import { renamePendingCreateBeforeResponseTest } from "./tests/rename-pending-create-before-response.test";
|
||||
import { createRenameResponseSkipsFileTest } from "./tests/create-rename-response-skips-file.test";
|
||||
import { onlineCreateRenameConcurrentCreateOrphanTest } from "./tests/online-create-rename-concurrent-create-orphan.test";
|
||||
import { concurrentRenameFirstWinsTest } from "./tests/concurrent-rename-first-wins.test";
|
||||
import { binaryToTextTransitionTest } from "./tests/binary-to-text-transition.test";
|
||||
import { textPendingCreateNotDisplacedTest } from "./tests/text-pending-create-not-displaced.test";
|
||||
import { binaryPendingCreateNotDisplacedTest } from "./tests/binary-pending-create-not-displaced.test";
|
||||
import { coalesceUpdateRemoteUpdateDataLossTest } from "./tests/coalesce-update-remote-update-data-loss.test";
|
||||
import { coalescedRemoteUpdateWatermarkLossTest } from "./tests/coalesced-remote-update-watermark-loss.test";
|
||||
import { concurrentDeleteDuringRemoteUpdateTest } from "./tests/concurrent-delete-during-remote-update.test";
|
||||
import { concurrentEditExactSamePositionTest } from "./tests/concurrent-edit-exact-same-position.test";
|
||||
import { concurrentRenameAndCreateAtTargetRenameFirstTest } from "./tests/concurrent-rename-and-create-at-target-rename-first.test";
|
||||
import { concurrentRenameAndCreateAtTargetCreateFirstTest } from "./tests/concurrent-rename-and-create-at-target-create-first.test";
|
||||
import { concurrentRenameSameTargetTest } from "./tests/concurrent-rename-same-target.test";
|
||||
import { concurrentUpdateDiffConsistencyTest } from "./tests/concurrent-update-diff-consistency.test";
|
||||
import { userParenthesizedFileNotDeletedTest } from "./tests/user-parenthesized-file-not-deleted.test";
|
||||
import { createDeleteNoopTest } from "./tests/create-delete-noop.test";
|
||||
import { createMergeDeleteTest } from "./tests/create-merge-delete.test";
|
||||
import { moveIdenticalContentAmbiguityTest } from "./tests/move-identical-content-ambiguity.test";
|
||||
import { createUpdateCoalesceServerPauseTest } from "./tests/create-update-coalesce-server-pause.test";
|
||||
import { createDuringReconciliationTest } from "./tests/create-during-reconciliation.test";
|
||||
import { createMergePreservesRenamedUpdateTest } from "./tests/create-merge-preserves-renamed-update.test";
|
||||
import { createRenameCreateSamePathTest } from "./tests/create-rename-create-same-path.test";
|
||||
import { moveChainThreeFilesTest } from "./tests/move-chain-three-files.test";
|
||||
import { deleteByOtherClientThenRecreateTest } from "./tests/delete-by-other-client-then-recreate.test";
|
||||
import { onlineDeleteRecreateRapidCycleTest } from "./tests/online-delete-recreate-rapid-cycle.test";
|
||||
import { onlineEditVsDeleteConvergenceTest } from "./tests/online-edit-vs-delete-convergence.test";
|
||||
import { rapidEditDeleteOnlineConvergenceTest } from "./tests/rapid-edit-delete-online-convergence.test";
|
||||
import { serverPauseDeleteRecreateTest } from "./tests/server-pause-delete-recreate.test";
|
||||
import { onlineBothCreateSamePathDeconflictTest } from "./tests/online-both-create-same-path-deconflict.test";
|
||||
import { onlineCreateUpdateWhileOtherCreatesSamePathTest } from "./tests/online-create-update-while-other-creates-same-path.test";
|
||||
import { displacedFileNotMarkedDeletedTest } from "./tests/displaced-file-not-marked-deleted.test";
|
||||
import { remoteUpdateResurrectsDeletedDocTest } from "./tests/remote-update-resurrects-deleted-doc.test";
|
||||
import { localUpdateSurvivesRemoteRenameTest } from "./tests/local-update-survives-remote-rename.test";
|
||||
import { mergingUpdateResponseSurvivesUserRenameTest } from "./tests/merging-update-response-survives-user-rename.test";
|
||||
import { catchupCreateAndUpdateNotSkippedTest } from "./tests/catchup-create-and-update-not-skipped.test";
|
||||
import { localRenameSurvivesRemoteRenameTest } from "./tests/local-rename-survives-remote-rename.test";
|
||||
import { renameChainDuringPendingCreateTest } from "./tests/rename-chain-during-pending-create.test";
|
||||
import { remoteRenameCollidesWithPendingLocalCreateTest } from "./tests/remote-rename-collides-with-pending-local-create.test";
|
||||
import { remoteUpdateSurvivesUserRenameTest } from "./tests/remote-update-survives-user-rename.test";
|
||||
import { sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest } from "./tests/same-doc-id-collapse-on-local-create-after-remote-create.test";
|
||||
import { sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest } from "./tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test";
|
||||
import { renameOverwritesPendingCreateThenDeleteTest } from "./tests/rename-overwrites-pending-create-then-delete.test";
|
||||
import { deleteRecreatedPendingCreateWithStaleDeletingRecordTest } from "./tests/delete-recreated-pending-create-with-stale-deleting-record.test";
|
||||
import { queuedCreateDeleteDoesNotHijackReusedPathTest } from "./tests/queued-create-delete-does-not-hijack-reused-path.test";
|
||||
import { renamedPendingCreateReusedPathThenDeleteTest } from "./tests/renamed-pending-create-reused-path-then-delete.test";
|
||||
import { renamePendingCreateOntoPendingDeletePathTest } from "./tests/rename-pending-create-onto-pending-delete-path.test";
|
||||
import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-write-rename-before-record.test";
|
||||
import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test";
|
||||
|
||||
export const TESTS: Partial<Record<string, TestDefinition>> = {
|
||||
"rename-create-conflict": renameCreateConflictTest,
|
||||
"rename-chain": renameChainTest,
|
||||
"rename-update-conflict": renameUpdateConflictTest,
|
||||
"delete-rename-conflict": deleteRenameConflictTest,
|
||||
"multi-file-operations": multiFileOperationsTest,
|
||||
"delete-recreate-same-path": deleteRecreateSamePathTest,
|
||||
"offline-rename-and-edit": offlineRenameAndEditTest,
|
||||
"simultaneous-create-delete-same-path":
|
||||
simultaneousCreateDeleteSamePathTest,
|
||||
"idempotency-after-server-pause": idempotencyAfterServerPauseTest,
|
||||
"sequential-create-duplicate-content": sequentialCreateDuplicateContentTest,
|
||||
"mc-three-client-rename-offline-update":
|
||||
mcThreeClientRenameOfflineUpdateTest,
|
||||
"mc-multi-delete-offline-rename": mcMultiDeleteOfflineRenameTest,
|
||||
"mc-cross-create-rename-same-target": mcCrossCreateRenameSameTargetTest,
|
||||
"mc-delete-then-offline-rename": mcDeleteThenOfflineRenameTest,
|
||||
"offline-mixed-operations": offlineMixedOperationsTest,
|
||||
"offline-concurrent-renames": offlineConcurrentRenamesTest,
|
||||
"offline-multiple-edits": offlineMultipleEditsTest,
|
||||
"server-pause-both-clients-create": serverPauseBothClientsCreateTest,
|
||||
"server-pause-update-and-create": serverPauseUpdateAndCreateTest,
|
||||
"rename-swap": renameSwapTest,
|
||||
"rename-circular": renameCircularTest,
|
||||
"rename-roundtrip": renameRoundtripTest,
|
||||
"offline-rename-remote-create-old-path":
|
||||
offlineRenameRemoteCreateOldPathTest,
|
||||
"offline-edit-remote-rename": offlineEditRemoteRenameTest,
|
||||
"rename-chain-then-delete": renameChainThenDeleteTest,
|
||||
"offline-delete-remote-rename": offlineDeleteRemoteRenameTest,
|
||||
"overlapping-edits-same-section": overlappingEditsSameSectionTest,
|
||||
"rapid-updates-after-merge": rapidUpdatesAfterMergeTest,
|
||||
"delete-recreate-concurrent-update": deleteRecreateConcurrentUpdateTest,
|
||||
"move-and-concurrent-remote-update": moveAndConcurrentRemoteUpdateTest,
|
||||
"double-offline-cycle": doubleOfflineCycleTest,
|
||||
"server-pause-rename-edit-resume": serverPauseRenameEditResumeTest,
|
||||
"offline-update-both-then-delete-one": offlineUpdateBothThenDeleteOneTest,
|
||||
"offline-create-same-path-mergeable": offlineCreateSamePathMergeableTest,
|
||||
"delete-during-pending-create": deleteDuringPendingCreateTest,
|
||||
"three-client-rename-create-delete": threeClientRenameCreateDeleteTest,
|
||||
"rename-to-path-of-unconfirmed-delete": renameToPathOfUnconfirmedDeleteTest,
|
||||
"offline-edit-then-move-same-content": offlineEditThenMoveSameContentTest,
|
||||
"rapid-create-update-delete-cycle": rapidCreateUpdateDeleteCycleTest,
|
||||
"server-pause-both-edit-same-file": serverPauseBothEditSameFileTest,
|
||||
"delete-recreate-different-content": deleteRecreateDifferentContentTest,
|
||||
"update-during-create-processing": updateDuringCreateProcessingTest,
|
||||
"offline-move-then-remote-delete": offlineMoveThenRemoteDeleteTest,
|
||||
"reset-clears-recently-deleted-resurrection":
|
||||
resetClearsRecentlyDeletedResurrectionTest,
|
||||
"move-then-delete-stale-path": moveThenDeleteStalePathTest,
|
||||
"offline-delete-vs-remote-update": offlineDeleteVsRemoteUpdateTest,
|
||||
"interrupted-delete-retry": interruptedDeleteRetryTest,
|
||||
"update-does-not-survive-remote-delete": updateDoesNotSurviveRemoteDeleteTest,
|
||||
"move-preserves-remote-update": movePreservesRemoteUpdateTest,
|
||||
"recently-deleted-cleared-on-reconnect":
|
||||
recentlyDeletedClearedOnReconnectTest,
|
||||
"watermark-advances-on-skip": watermarkAdvancesOnSkipTest,
|
||||
"watermark-gap-remote-update-not-recorded":
|
||||
watermarkGapRemoteUpdateNotRecordedTest,
|
||||
"queue-reset-loses-coalesced-local-edit":
|
||||
queueResetLosesCoalescedLocalEditTest,
|
||||
"rename-to-pending-path-fallback": renameToPendingPathFallbackTest,
|
||||
"move-remote-update-reverts-rename": moveRemoteUpdateRevertsRenameTest,
|
||||
"local-edit-lost-during-create-merge": localEditLostDuringCreateMergeTest,
|
||||
"rename-pending-create-before-response":
|
||||
renamePendingCreateBeforeResponseTest,
|
||||
"create-rename-response-skips-file": createRenameResponseSkipsFileTest,
|
||||
"online-create-rename-concurrent-create-orphan":
|
||||
onlineCreateRenameConcurrentCreateOrphanTest,
|
||||
"concurrent-rename-first-wins": concurrentRenameFirstWinsTest,
|
||||
"binary-to-text-transition": binaryToTextTransitionTest,
|
||||
"text-pending-create-not-displaced": textPendingCreateNotDisplacedTest,
|
||||
"binary-pending-create-not-displaced": binaryPendingCreateNotDisplacedTest,
|
||||
"coalesce-update-remote-update-data-loss":
|
||||
coalesceUpdateRemoteUpdateDataLossTest,
|
||||
"coalesced-remote-update-watermark-loss":
|
||||
coalescedRemoteUpdateWatermarkLossTest,
|
||||
"concurrent-delete-during-remote-update":
|
||||
concurrentDeleteDuringRemoteUpdateTest,
|
||||
"concurrent-edit-exact-same-position": concurrentEditExactSamePositionTest,
|
||||
"concurrent-rename-and-create-at-target-rename-first":
|
||||
concurrentRenameAndCreateAtTargetRenameFirstTest,
|
||||
"concurrent-rename-and-create-at-target-create-first":
|
||||
concurrentRenameAndCreateAtTargetCreateFirstTest,
|
||||
"concurrent-rename-same-target": concurrentRenameSameTargetTest,
|
||||
"concurrent-update-diff-consistency": concurrentUpdateDiffConsistencyTest,
|
||||
"user-parenthesized-file-not-deleted": userParenthesizedFileNotDeletedTest,
|
||||
"create-delete-noop": createDeleteNoopTest,
|
||||
"create-merge-delete": createMergeDeleteTest,
|
||||
"move-identical-content-ambiguity": moveIdenticalContentAmbiguityTest,
|
||||
"create-update-coalesce-server-pause": createUpdateCoalesceServerPauseTest,
|
||||
"create-during-reconciliation": createDuringReconciliationTest,
|
||||
"create-merge-preserves-renamed-update":
|
||||
createMergePreservesRenamedUpdateTest,
|
||||
"create-rename-create-same-path": createRenameCreateSamePathTest,
|
||||
"move-chain-three-files": moveChainThreeFilesTest,
|
||||
"delete-by-other-client-then-recreate": deleteByOtherClientThenRecreateTest,
|
||||
"online-delete-recreate-rapid-cycle": onlineDeleteRecreateRapidCycleTest,
|
||||
"online-edit-vs-delete-convergence": onlineEditVsDeleteConvergenceTest,
|
||||
"rapid-edit-delete-online-convergence":
|
||||
rapidEditDeleteOnlineConvergenceTest,
|
||||
"server-pause-delete-recreate": serverPauseDeleteRecreateTest,
|
||||
"online-both-create-same-path-deconflict":
|
||||
onlineBothCreateSamePathDeconflictTest,
|
||||
"online-create-update-while-other-creates-same-path":
|
||||
onlineCreateUpdateWhileOtherCreatesSamePathTest,
|
||||
"displaced-file-not-marked-deleted": displacedFileNotMarkedDeletedTest,
|
||||
"remote-update-resurrects-deleted-doc":
|
||||
remoteUpdateResurrectsDeletedDocTest,
|
||||
"local-update-survives-remote-rename": localUpdateSurvivesRemoteRenameTest,
|
||||
"merging-update-response-survives-user-rename":
|
||||
mergingUpdateResponseSurvivesUserRenameTest,
|
||||
"catchup-create-and-update-not-skipped":
|
||||
catchupCreateAndUpdateNotSkippedTest,
|
||||
"local-rename-survives-remote-rename": localRenameSurvivesRemoteRenameTest,
|
||||
"rename-chain-during-pending-create": renameChainDuringPendingCreateTest,
|
||||
"remote-rename-collides-with-pending-local-create":
|
||||
remoteRenameCollidesWithPendingLocalCreateTest,
|
||||
"remote-update-survives-user-rename": remoteUpdateSurvivesUserRenameTest,
|
||||
"same-doc-id-collapse-on-local-create-after-remote-create":
|
||||
sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest,
|
||||
"renamed-pending-create-reused-path-then-delete":
|
||||
renamedPendingCreateReusedPathThenDeleteTest,
|
||||
"rename-pending-create-onto-pending-delete-path":
|
||||
renamePendingCreateOntoPendingDeletePathTest,
|
||||
"rename-overwrites-pending-create-then-delete":
|
||||
renameOverwritesPendingCreateThenDeleteTest,
|
||||
"same-doc-id-collapse-after-remote-quick-write-and-pending-rename":
|
||||
sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest,
|
||||
"delete-recreated-pending-create-with-stale-deleting-record":
|
||||
deleteRecreatedPendingCreateWithStaleDeletingRecordTest,
|
||||
"queued-create-delete-does-not-hijack-reused-path":
|
||||
queuedCreateDeleteDoesNotHijackReusedPathTest,
|
||||
"remote-quick-write-rename-before-record":
|
||||
remoteQuickWriteRenameBeforeRecordTest,
|
||||
"self-merge-pending-rename-aliases-second-create":
|
||||
selfMergePendingRenameAliasesSecondCreateTest
|
||||
};
|
||||
399
frontend/deterministic-tests/src/test-runner.ts
Normal file
399
frontend/deterministic-tests/src/test-runner.ts
Normal file
|
|
@ -0,0 +1,399 @@
|
|||
import type { TestDefinition, TestResult, TestStep } from "./test-definition";
|
||||
import { DeterministicAgent } from "./deterministic-agent";
|
||||
import type { ServerControl } from "./server-control";
|
||||
import type { SyncSettings, Logger } from "sync-client";
|
||||
import { assert } from "./utils/assert";
|
||||
import { AssertableState } from "./utils/assertable-state";
|
||||
import { sleep } from "./utils/sleep";
|
||||
import { withTimeout } from "./utils/with-timeout";
|
||||
import {
|
||||
CONVERGENCE_TIMEOUT_MS,
|
||||
CONVERGENCE_RETRY_DELAY_MS,
|
||||
AGENT_INIT_TIMEOUT_MS,
|
||||
IS_SYNC_ENABLED_BY_DEFAULT
|
||||
} from "./consts";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
export class TestRunner {
|
||||
private agents: DeterministicAgent[] = [];
|
||||
private readonly serverControl: ServerControl;
|
||||
private readonly token: string;
|
||||
private readonly remoteUri: string;
|
||||
private readonly logger: Logger;
|
||||
|
||||
public constructor(
|
||||
serverControl: ServerControl,
|
||||
logger: Logger,
|
||||
token: string,
|
||||
remoteUri: string
|
||||
) {
|
||||
this.serverControl = serverControl;
|
||||
this.logger = logger;
|
||||
this.token = token;
|
||||
this.remoteUri = remoteUri;
|
||||
}
|
||||
|
||||
public async runTest(
|
||||
name: string,
|
||||
test: TestDefinition
|
||||
): Promise<TestResult> {
|
||||
const startTime = Date.now();
|
||||
this.logger.info(`Running test: ${name}`);
|
||||
if (test.description !== undefined && test.description !== "") {
|
||||
this.logger.info(`Description: ${test.description}`);
|
||||
}
|
||||
this.logger.info(`Clients: ${test.clients}`);
|
||||
this.logger.info(`Steps: ${test.steps.length}`);
|
||||
|
||||
try {
|
||||
assert(
|
||||
this.serverControl.isRunning(),
|
||||
"Server is not running before test start"
|
||||
);
|
||||
|
||||
await this.initializeAgents(test.clients);
|
||||
|
||||
for (let i = 0; i < test.steps.length; i++) {
|
||||
const step = test.steps[i];
|
||||
this.logger.info(
|
||||
`Step ${i + 1}/${test.steps.length}: ${JSON.stringify(step)}`
|
||||
);
|
||||
await this.executeStep(step);
|
||||
}
|
||||
|
||||
await this.cleanup();
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
this.logger.info(`\n✓ Test passed: ${name} (${duration}ms)`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
duration
|
||||
};
|
||||
} catch (error) {
|
||||
const duration = Date.now() - startTime;
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
this.logger.info(`\n✗ Test failed: ${name}`);
|
||||
this.logger.info(`Error: ${errorMessage}`);
|
||||
|
||||
await this.cleanup();
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
duration
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private async initializeAgents(count: number): Promise<void> {
|
||||
assert(count > 0, `Client count must be positive, got ${count}`);
|
||||
const vaultName = `test-${randomUUID()}`;
|
||||
this.logger.info(
|
||||
`Initializing ${count} agents with vault: ${vaultName}`
|
||||
);
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const settings: Partial<SyncSettings> = {
|
||||
isSyncEnabled: IS_SYNC_ENABLED_BY_DEFAULT,
|
||||
token: this.token,
|
||||
vaultName,
|
||||
remoteUri: this.remoteUri
|
||||
};
|
||||
|
||||
const agent = new DeterministicAgent(i, settings, (msg) => {
|
||||
this.logger.info(msg);
|
||||
});
|
||||
|
||||
// Push before init so cleanup() handles this agent if init fails
|
||||
this.agents.push(agent);
|
||||
await withTimeout(
|
||||
agent.init(fetch),
|
||||
AGENT_INIT_TIMEOUT_MS,
|
||||
`Client ${i} init timed out after ${AGENT_INIT_TIMEOUT_MS}ms`
|
||||
);
|
||||
this.logger.info(`Initialized client ${i}`);
|
||||
}
|
||||
|
||||
this.logger.info("All agents initialized");
|
||||
}
|
||||
|
||||
private getAgent(index: number): DeterministicAgent {
|
||||
assert(
|
||||
index >= 0 && index < this.agents.length,
|
||||
`Client index ${index} out of bounds (have ${this.agents.length} agents)`
|
||||
);
|
||||
return this.agents[index];
|
||||
}
|
||||
|
||||
private async executeStep(step: TestStep): Promise<void> {
|
||||
switch (step.type) {
|
||||
case "create":
|
||||
case "update":
|
||||
await this.getAgent(step.client).write(
|
||||
step.path,
|
||||
new TextEncoder().encode(step.content)
|
||||
);
|
||||
break;
|
||||
|
||||
case "rename":
|
||||
await this.getAgent(step.client).rename(
|
||||
step.oldPath,
|
||||
step.newPath
|
||||
);
|
||||
break;
|
||||
|
||||
case "rename-next-write":
|
||||
this.getAgent(step.client).renameNextWrite(
|
||||
step.oldPath,
|
||||
step.newPath
|
||||
);
|
||||
break;
|
||||
|
||||
case "delete":
|
||||
await this.getAgent(step.client).delete(step.path);
|
||||
break;
|
||||
|
||||
case "sync":
|
||||
if (step.client !== undefined) {
|
||||
await this.getAgent(step.client).waitForSync();
|
||||
} else {
|
||||
for (const agent of this.agents) {
|
||||
await agent.waitForSync();
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "disable-sync":
|
||||
await this.getAgent(step.client).disableSync();
|
||||
break;
|
||||
|
||||
case "enable-sync":
|
||||
await this.getAgent(step.client).enableSync();
|
||||
break;
|
||||
|
||||
case "pause-server":
|
||||
this.serverControl.pause();
|
||||
break;
|
||||
|
||||
case "resume-server":
|
||||
this.serverControl.resume();
|
||||
// Verify the server is actually responsive before proceeding.
|
||||
// This replaces relying solely on hardcoded waits.
|
||||
await this.serverControl.waitForReady();
|
||||
break;
|
||||
|
||||
case "resume-server-until-history-then-pause": {
|
||||
const agent = this.getAgent(step.client);
|
||||
const historySeen = agent.waitForHistoryEntry(
|
||||
(entry) =>
|
||||
entry.details.type === step.syncType &&
|
||||
entry.details.relativePath === step.path,
|
||||
() => this.serverControl.pause()
|
||||
);
|
||||
this.serverControl.resume();
|
||||
await historySeen;
|
||||
break;
|
||||
}
|
||||
|
||||
case "barrier":
|
||||
await this.waitForConvergence();
|
||||
break;
|
||||
|
||||
case "assert-consistent":
|
||||
await this.assertConsistent(step.verify);
|
||||
break;
|
||||
|
||||
case "pause-websocket":
|
||||
this.getAgent(step.client).pauseWebSocket();
|
||||
break;
|
||||
|
||||
case "resume-websocket":
|
||||
this.getAgent(step.client).resumeWebSocket();
|
||||
break;
|
||||
|
||||
case "drop-next-create-response":
|
||||
this.getAgent(step.client).dropNextCreateResponse();
|
||||
break;
|
||||
|
||||
case "wait-for-dropped-create-response":
|
||||
await this.getAgent(step.client).waitForDroppedCreateResponse();
|
||||
break;
|
||||
|
||||
case "sleep":
|
||||
await sleep(step.ms);
|
||||
break;
|
||||
|
||||
case "reset":
|
||||
await this.getAgent(step.client).reset();
|
||||
break;
|
||||
|
||||
default: {
|
||||
const unknownStep = step as { type: string };
|
||||
throw new Error(`Unknown step type: ${unknownStep.type}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for all agents to reach a consistent state.
|
||||
*
|
||||
* Waiting for agents is done in two full rounds: the first round
|
||||
* drains in-flight operations, but completing those operations can
|
||||
* trigger new work on OTHER agents via server broadcasts. The second
|
||||
* round waits for that cascading work to settle. Deeper cascades
|
||||
* are handled by the outer retry loop.
|
||||
*/
|
||||
private async waitForConvergence(): Promise<void> {
|
||||
this.logger.info("Barrier: waiting for convergence...");
|
||||
|
||||
const deadline = Date.now() + CONVERGENCE_TIMEOUT_MS;
|
||||
let lastError: Error | undefined = undefined;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
await this.waitAllAgentsSettled();
|
||||
|
||||
try {
|
||||
await this.assertConsistent();
|
||||
this.logger.info("Barrier complete: all clients converged");
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
this.logger.info("Barrier: not yet converged, retrying...");
|
||||
await sleep(CONVERGENCE_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Convergence timed out after ${CONVERGENCE_TIMEOUT_MS}ms: ${lastError?.message ?? "no consistency check ran"}`,
|
||||
{ cause: lastError }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for all agents to be simultaneously idle.
|
||||
*
|
||||
* Completing work on agent A can trigger a server broadcast that
|
||||
* enqueues new work on agent B, which can cascade further. With N
|
||||
* agents the worst-case cascade depth is N (a chain A→B→C→…→A),
|
||||
* so we run N+1 sequential passes to drain it. Extra passes are
|
||||
* essentially free when there is no outstanding work.
|
||||
*
|
||||
* The outer {@link waitForConvergence} loop with consistency checks
|
||||
* remains the ultimate guarantee — this method just minimizes how
|
||||
* many slow retry iterations are needed.
|
||||
*/
|
||||
private async waitAllAgentsSettled(): Promise<void> {
|
||||
const rounds = this.agents.length + 1;
|
||||
for (let round = 0; round < rounds; round++) {
|
||||
for (const agent of this.agents) {
|
||||
await agent.waitForSync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async assertConsistent(
|
||||
verify?: (state: AssertableState) => void
|
||||
): Promise<void> {
|
||||
this.logger.info("Asserting all clients are consistent...");
|
||||
assert(
|
||||
this.agents.length >= 2,
|
||||
"Need at least 2 agents for consistency check"
|
||||
);
|
||||
|
||||
// Snapshot all agents' file states upfront to minimize the window
|
||||
// where background sync could mutate state between reads.
|
||||
const clientFiles: Map<string, string>[] = [];
|
||||
for (const agent of this.agents) {
|
||||
const sortedFiles = (await agent.listFilesRecursively()).sort();
|
||||
const fileMap = new Map<string, string>();
|
||||
for (const file of sortedFiles) {
|
||||
const content = await agent.getFileContent(file);
|
||||
fileMap.set(file, content);
|
||||
}
|
||||
clientFiles.push(fileMap);
|
||||
}
|
||||
|
||||
const referenceFiles = Array.from(clientFiles[0].keys());
|
||||
|
||||
this.logger.info(
|
||||
`Reference client has ${referenceFiles.length} files: ${referenceFiles.join(", ")}`
|
||||
);
|
||||
|
||||
for (let i = 1; i < clientFiles.length; i++) {
|
||||
const agentFileKeys = Array.from(clientFiles[i].keys());
|
||||
|
||||
this.logger.info(
|
||||
`Client ${i} has ${agentFileKeys.length} files: ${agentFileKeys.join(", ")}`
|
||||
);
|
||||
|
||||
assert(
|
||||
agentFileKeys.length === referenceFiles.length,
|
||||
`File count mismatch: client 0 has ${referenceFiles.length} files, client ${i} has ${agentFileKeys.length} files`
|
||||
);
|
||||
|
||||
for (let j = 0; j < agentFileKeys.length; j++) {
|
||||
assert(
|
||||
agentFileKeys[j] === referenceFiles[j],
|
||||
`File list mismatch at index ${j}: client 0 has "${referenceFiles[j]}", client ${i} has "${agentFileKeys[j]}"`
|
||||
);
|
||||
}
|
||||
|
||||
for (const file of referenceFiles) {
|
||||
const referenceContent = clientFiles[0].get(file);
|
||||
const agentContent = clientFiles[i].get(file);
|
||||
|
||||
assert(
|
||||
referenceContent === agentContent,
|
||||
`Content mismatch for ${file}:\nClient 0: "${referenceContent}"\nClient ${i}: "${agentContent}"`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info("✓ All clients are consistent");
|
||||
|
||||
if (verify) {
|
||||
this.logger.info("Running custom verification...");
|
||||
try {
|
||||
verify(
|
||||
new AssertableState({
|
||||
files: clientFiles[0],
|
||||
clientFiles
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
const msg =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`Custom verification failed: ${msg}`);
|
||||
}
|
||||
this.logger.info("✓ Custom verification passed");
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanup(): Promise<void> {
|
||||
// Always resume the server in case a test paused it and then
|
||||
// failed before reaching the resume step. Without this, all
|
||||
// subsequent tests would hang because the server process is
|
||||
// frozen (SIGSTOP) and can't respond to HTTP or WebSocket.
|
||||
try {
|
||||
this.serverControl.resume();
|
||||
} catch {
|
||||
// Server wasn't paused or isn't running — safe to ignore
|
||||
}
|
||||
|
||||
this.logger.info("\nCleaning up agents...");
|
||||
for (const agent of this.agents) {
|
||||
try {
|
||||
await agent.cleanup();
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Agent cleanup error: ${error instanceof Error ? error.message : String(error)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
this.agents = [];
|
||||
this.logger.info("Cleanup complete");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import type { AssertableState } from "../utils/assertable-state";
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const binaryPendingCreateNotDisplacedTest: TestDefinition = {
|
||||
description:
|
||||
"Two clients each create a binary file at the same path while offline. " +
|
||||
"After syncing, both files should exist on both clients at separate paths.",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{
|
||||
type: "create",
|
||||
client: 0,
|
||||
path: "data.bin",
|
||||
content: "binary data from client 0"
|
||||
},
|
||||
{
|
||||
type: "create",
|
||||
client: 1,
|
||||
path: "data.bin",
|
||||
content: "binary data from client 1"
|
||||
},
|
||||
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertFileCount(2)
|
||||
.assertFileExists("data.bin")
|
||||
.assertFileExists("data (1).bin")
|
||||
.assertAnyFileContains(
|
||||
"binary data from client 0",
|
||||
"binary data from client 1"
|
||||
);
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
import type { AssertableState } from "../utils/assertable-state";
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const binaryToTextTransitionTest: TestDefinition = {
|
||||
description:
|
||||
"A .bin file is created and synced. Both clients edit it offline " +
|
||||
"(binary last-write-wins), then client 0 renames it to .md and " +
|
||||
"writes a clean text baseline. Both clients edit different sections " +
|
||||
"offline. The text merge should preserve both edits.",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{
|
||||
type: "create",
|
||||
client: 0,
|
||||
path: "data.bin",
|
||||
content: "original content"
|
||||
},
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertContent("data.bin", "original content");
|
||||
}
|
||||
},
|
||||
|
||||
{ type: "disable-sync", client: 0 },
|
||||
{ type: "disable-sync", client: 1 },
|
||||
|
||||
{ type: "update", client: 0, path: "data.bin", content: "version A" },
|
||||
{ type: "update", client: 1, path: "data.bin", content: "version B" },
|
||||
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertFileCount(1).assertContainsAny(
|
||||
"data.bin",
|
||||
"version A",
|
||||
"version B"
|
||||
);
|
||||
}
|
||||
},
|
||||
|
||||
{ type: "disable-sync", client: 1 },
|
||||
{ type: "rename", client: 0, oldPath: "data.bin", newPath: "data.md" },
|
||||
{
|
||||
type: "update",
|
||||
client: 0,
|
||||
path: "data.md",
|
||||
content: "top line\nmiddle line\nbottom line"
|
||||
},
|
||||
{ type: "sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertContent(
|
||||
"data.md",
|
||||
"top line\nmiddle line\nbottom line"
|
||||
);
|
||||
}
|
||||
},
|
||||
|
||||
{ type: "disable-sync", client: 0 },
|
||||
{ type: "disable-sync", client: 1 },
|
||||
|
||||
{
|
||||
type: "update",
|
||||
client: 0,
|
||||
path: "data.md",
|
||||
content: "alpha\nmiddle line\nbottom line"
|
||||
},
|
||||
{
|
||||
type: "update",
|
||||
client: 1,
|
||||
path: "data.md",
|
||||
content: "top line\nmiddle line\nbeta"
|
||||
},
|
||||
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertFileCount(1).assertContains("data.md", "alpha", "beta");
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import type { AssertableState } from "../utils/assertable-state";
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const catchupCreateAndUpdateNotSkippedTest: TestDefinition = {
|
||||
description:
|
||||
"Client 1 disconnects (sync disabled). Client 0 creates a doc and " +
|
||||
"then updates it. When Client 1 reconnects, the server's catch-up " +
|
||||
"stream sends only the doc's *latest* version (the update), not the " +
|
||||
"full history. Pre-fix the wire's `is_new_file` was set to " +
|
||||
"`creation == latest_version`, so the catch-up flagged the doc as " +
|
||||
"non-new even though Client 1 had never seen its creation. Client " +
|
||||
"1's `processRemoteChange` then dropped it as a 'stale RemoteChange " +
|
||||
"for untracked, non-new document' and the doc was silently lost. " +
|
||||
"Post-fix `is_new_file` in the catch-up stream means 'new relative " +
|
||||
"to the recipient's watermark' (`creation > last_seen_vault_update_id`).",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
// Establish a baseline so Client 1's last_seen is non-zero before
|
||||
// we take it offline. This makes the bug genuinely about catch-up
|
||||
// missing the create rather than just an empty-vault first sync.
|
||||
{ type: "create", client: 0, path: "warmup.md", content: "w\n" },
|
||||
{ type: "barrier" },
|
||||
|
||||
// Client 1 goes offline.
|
||||
{ type: "disable-sync", client: 1 },
|
||||
|
||||
// Client 0 creates the doc (vault_update_id v_C, after Client 1's
|
||||
// watermark). Client 1 doesn't see this because it's offline.
|
||||
{ type: "create", client: 0, path: "doc.md", content: "v1\n" },
|
||||
// Wait for the create's HTTP to land before the update; otherwise
|
||||
// both writes are coalesced into a single POST and the server
|
||||
// never sees the doc as "create followed by update".
|
||||
{ type: "sync", client: 0 },
|
||||
|
||||
// Client 0 updates the doc (vault_update_id v_X > v_C). The
|
||||
// server's `latest_document_versions` view now returns the
|
||||
// *update* row — its `creation_vault_update_id != vault_update_id`.
|
||||
{
|
||||
type: "update",
|
||||
client: 0,
|
||||
path: "doc.md",
|
||||
content: "v1\nupdate\n"
|
||||
},
|
||||
{ type: "sync", client: 0 },
|
||||
|
||||
// Client 1 reconnects. Server's catch-up replays docs with
|
||||
// `vault_update_id > last_seen`. For doc.md it sends v_X with
|
||||
// `is_new_file` derived from `creation_vault_update_id >
|
||||
// last_seen_vault_update_id` (post-fix) — so Client 1 treats it
|
||||
// as a fresh create and downloads the latest content.
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (state: AssertableState): void => {
|
||||
state.assertFileCount(2);
|
||||
state.assertFileExists("doc.md");
|
||||
state.assertContent("doc.md", "v1\nupdate\n");
|
||||
state.assertContent("warmup.md", "w\n");
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import type { AssertableState } from "../utils/assertable-state";
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const coalesceUpdateRemoteUpdateDataLossTest: TestDefinition = {
|
||||
description:
|
||||
"Divergent offline edits with text-merge expectation. Client 0's " +
|
||||
"remote update fully lands before Client 1 reconnects (`sync`-after " +
|
||||
"the c0 update enforces this), so Client 1's offline edit merges " +
|
||||
"against a server-known version, not a coalesced batch. Both " +
|
||||
"additions must survive in the final merged content. (Filename's " +
|
||||
"'coalesce' framing is aspirational — a true update-coalesce test " +
|
||||
"would skip the c0 sync and queue overlapping local + remote " +
|
||||
"updates against the same parent version.)",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{
|
||||
type: "create",
|
||||
client: 0,
|
||||
path: "doc.md",
|
||||
content: "line 1\nline 2\nline 3"
|
||||
},
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{ type: "disable-sync", client: 1 },
|
||||
|
||||
{
|
||||
type: "update",
|
||||
client: 0,
|
||||
path: "doc.md",
|
||||
content: "line 1\nline 2\nline 3\nclient 0 addition"
|
||||
},
|
||||
{ type: "sync", client: 0 },
|
||||
|
||||
{
|
||||
type: "update",
|
||||
client: 1,
|
||||
path: "doc.md",
|
||||
content: "client 1 addition\nline 1\nline 2\nline 3"
|
||||
},
|
||||
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (state: AssertableState): void => {
|
||||
state
|
||||
.assertFileCount(1)
|
||||
.assertContains(
|
||||
"doc.md",
|
||||
"client 0 addition",
|
||||
"client 1 addition"
|
||||
);
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import type { AssertableState } from "../utils/assertable-state";
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const coalescedRemoteUpdateWatermarkLossTest: TestDefinition = {
|
||||
description:
|
||||
"Client 0 sends three rapid updates. After syncing, both clients " +
|
||||
"disconnect and reconnect twice. Content should remain correct " +
|
||||
"after each reconnect.",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{ type: "create", client: 0, path: "doc.md", content: "original" },
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{ type: "update", client: 0, path: "doc.md", content: "update 1" },
|
||||
{ type: "update", client: 0, path: "doc.md", content: "update 2" },
|
||||
{ type: "update", client: 0, path: "doc.md", content: "final update" },
|
||||
|
||||
{ type: "barrier" },
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertFileCount(1).assertContent("doc.md", "final update");
|
||||
}
|
||||
},
|
||||
|
||||
{ type: "disable-sync", client: 0 },
|
||||
{ type: "disable-sync", client: 1 },
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertFileCount(1).assertContent("doc.md", "final update");
|
||||
}
|
||||
},
|
||||
|
||||
{ type: "disable-sync", client: 0 },
|
||||
{ type: "disable-sync", client: 1 },
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (s: AssertableState): void => {
|
||||
s.assertFileCount(1).assertContent("doc.md", "final update");
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import type { AssertableState } from "../utils/assertable-state";
|
||||
import type { TestDefinition } from "../test-definition";
|
||||
|
||||
export const concurrentDeleteDuringRemoteUpdateTest: TestDefinition = {
|
||||
description:
|
||||
"One client updates a file while the other deletes it at the same " +
|
||||
"time. Both clients should converge without errors.",
|
||||
clients: 2,
|
||||
steps: [
|
||||
{ type: "create", client: 0, path: "doc.md", content: "original" },
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{ type: "disable-sync", client: 0 },
|
||||
{ type: "disable-sync", client: 1 },
|
||||
|
||||
{ type: "update", client: 0, path: "doc.md", content: "updated by 0" },
|
||||
{ type: "delete", client: 1, path: "doc.md" },
|
||||
|
||||
{ type: "enable-sync", client: 0 },
|
||||
{ type: "enable-sync", client: 1 },
|
||||
{ type: "barrier" },
|
||||
|
||||
{
|
||||
type: "assert-consistent",
|
||||
verify: (state: AssertableState): void => {
|
||||
state.assertFileCount(0);
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue