power loss

This commit is contained in:
Andras Schmelczer 2026-09-21 21:29:52 +01:00
commit d56084e4bc
10 changed files with 18667 additions and 3902 deletions

View file

@ -0,0 +1,36 @@
# Seeded sync workloads
The old random mock filesystem/agent have been replaced by the same v4 adapter,
client wrapper, and canonical-state assertions as the scripted suite.
Periodic polling is explicitly disabled so it cannot repair missed notifications
and hide failures in these workloads. Polling has separate fake-clock tests.
From `frontend`:
```sh
npm run build --workspace sync-client --workspace test-client
node test-client/dist/cli.js --seed 42 --iterations 25
node test-client/dist/cli.js --replay /path/to/trace-42.json
```
The server binary must already be built. Each invocation starts and cleans up its
own server/database, and prints the replay command before executing any actions.
`--artifacts` chooses the trace/result directory. Failures include database,
visible/durable disk images, and CAS request records. The default 25 iterations
are scenarios selected in shuffled cycles of nine operations, using two or three
clients. Existing documents are repeatedly edited, moved, swapped, deleted and
recreated. Concurrent edits share an established base; delayed editor events stay
queued through reconnection and a subsequent remote edit. Create, content-update
and manifest requests receive before/after network faults.
Each checkpoint serializes a complete expected file set with exact bytes, paths
and persistent identity keys. Independent creates must remain distinct, and a
recreated file must obtain a new UUID. Later checkpoints reassert earlier edits
and moves, rather than retaining only their markers. Old traces without a client
count still replay with two clients.
All preservation assertions run even with delayed editor events. Explicit deletes
and overwrites exempt only their specific markers; unrelated content still must
appear exactly once. UUID continuity and canonical server state are checked too.
Seeds reproduce choices, not OS scheduling. See the
[full harness documentation](../deterministic-tests/README.md).

View file

@ -1,368 +0,0 @@
import { choose } from "../utils/choose";
import { v4 as uuidv4 } from "uuid";
import { assert } from "../utils/assert";
import type { RelativePath, SyncSettings } from "sync-client";
import { debugging, Logger, LogLevel, utils } from "sync-client";
import { MockClient } from "./mock-client";
import { sleep } from "../utils/sleep";
import type { LogLine } from "sync-client";
import { withTimeout } from "../utils/with-timeout";
const TIMEOUT_MS = 10 * 60 * 1000;
export class MockAgent extends MockClient {
private readonly writtenContents: string[] = [];
private readonly pendingActions: Promise<unknown>[] = [];
// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
private readonly doNotTouchWhileOffline: string[] = [];
public constructor(
initialSettings: Partial<SyncSettings>,
public readonly name: string,
private readonly doDeletes: boolean,
private readonly doResets: boolean,
useSlowFileEvents: boolean,
private readonly jitterScaleInSeconds: number
) {
super(initialSettings, useSlowFileEvents);
}
public async init(): Promise<void> {
await super.init(
debugging.slowFetchFactory(this.jitterScaleInSeconds),
debugging.slowWebSocketFactory(
this.jitterScaleInSeconds,
new Logger() // this logger isn't wired anywhere, so messages to it will be ignored
)
);
assert(
(await this.client.checkConnection()).isSuccessful,
"Connection check failed"
);
this.client.logger.onLogEmitted.add((logLine: LogLine) => {
const state = this.client.getSettings().isSyncEnabled
? "(online) "
: "(offline)";
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
const historyEntry = /.*History entry: (.*.md).*/.exec(
logLine.message
);
if (historyEntry) {
utils.removeFromArray(
this.doNotTouchWhileOffline,
historyEntry[1]
);
}
switch (logLine.level) {
case LogLevel.ERROR:
console.error(formatted);
if (!this.useSlowFileEvents) {
// Let's wait for the error to be caught if there was one
// eslint-disable-next-line @typescript-eslint/no-floating-promises
sleep(100).then(() => process.exit(1));
}
break;
case LogLevel.WARNING:
console.warn(formatted);
break;
case LogLevel.INFO:
console.info(formatted);
break;
case LogLevel.DEBUG:
console.debug(formatted);
break;
}
});
this.client.logger.info("Agent initialized");
}
public async act(): Promise<void> {
const options: (() => Promise<unknown>)[] = [
this.createFileAction.bind(this)
];
if (
this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.length === 0
) {
options.push(this.disableSyncAction.bind(this));
} else {
options.push(this.enableSyncAction.bind(this));
}
const files = await this.listFilesRecursively();
if (files.length > 0) {
options.push(
this.renameFileAction.bind(this, files),
this.updateFileAction.bind(this, files)
);
if (this.doDeletes) {
options.push(this.deleteFileAction.bind(this, files));
}
}
if (Math.random() < 0.015 && this.doResets) {
// we can't just queue this up as once it's destroyed, no more method calls can go to SyncClient
await this.resetClient();
} else {
this.pendingActions.push(
(async (): Promise<unknown> => {
try {
return await choose(options)();
} catch (error) {
this.client.logger.error(
`Failed to perform an action: ${error}`
);
this.client.logger.info(
JSON.stringify(this.data, null, 2)
);
this.client.logger.info(
JSON.stringify(this.localFiles, null, 2)
);
throw error;
}
})()
);
}
}
public async finish(): Promise<void> {
await withTimeout(
(async (): Promise<void> => {
await this.client.setSetting("isSyncEnabled", true);
await utils.awaitAll(this.pendingActions);
await this.client.waitUntilFinished();
})(),
TIMEOUT_MS,
"finish()"
);
}
public async destroy(): Promise<void> {
await withTimeout(
(async (): Promise<void> => {
await this.client.waitUntilFinished();
await this.client.destroy();
})(),
TIMEOUT_MS,
"destroy()"
);
}
public assertFileSystemsAreConsistent(otherAgent: MockAgent): void {
const globalFiles = Array.from(otherAgent.localFiles.keys());
const localFiles = Array.from(this.localFiles.keys());
const missingInOther = localFiles.filter(
(file) => !otherAgent.localFiles.has(file)
);
const missingInLocal = globalFiles.filter(
(file) => !this.localFiles.has(file)
);
try {
assert(
missingInOther.length === 0,
`Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}`
);
assert(
missingInLocal.length === 0,
`Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}`
);
for (const file of globalFiles) {
const localContent = new TextDecoder().decode(
this.localFiles.get(file)
);
const otherContent = new TextDecoder().decode(
otherAgent.localFiles.get(file)
);
assert(
localContent === otherContent,
`Content mismatch for file ${file}:\n${localContent}\n${otherContent}`
);
}
} catch (e) {
this.client.logger.info(
"Local data: " + JSON.stringify(this.data, null, 2)
);
this.client.logger.info(
"Local files: " +
Array.from(otherAgent.localFiles.keys()).join(", ")
);
otherAgent.client.logger.info(
"Local data: " + JSON.stringify(otherAgent.data, null, 2)
);
otherAgent.client.logger.info(
"Local files: " +
Array.from(otherAgent.localFiles.keys()).join(", ")
);
throw e;
}
}
public assertAllContentIsPresentOnce(): void {
if (this.useSlowFileEvents) {
this.client.logger.info(
// We can't ensure that we have seen every single update
`Skipping content check for ${this.name} because slow file events are enabled`
);
return;
}
for (const content of this.writtenContents) {
const found = Array.from(this.localFiles.keys()).filter((key) => {
return new TextDecoder()
.decode(this.localFiles.get(key))
.includes(content);
});
if (this.doDeletes) {
assert(
found.length <= 1,
`[${this.name}] Content ${content} found in ${found.join(", ")}`
);
} else {
assert(
found.length >= 1,
`[${this.name}] Content ${content} not found in any files`
);
assert(
found.length <= 1,
`[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}`
);
const [file] = found;
const fileContent = new TextDecoder().decode(
this.localFiles.get(file)
);
assert(
fileContent.split(content).length == 2,
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
);
}
}
}
private async resetClient(): Promise<void> {
this.client.logger.info(`Resetting client ${this.name}`);
await this.client.destroy();
await this.init();
}
private async createFileAction(): Promise<void> {
const file = this.getFileName();
if (
(!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(file)) ||
(await this.exists(file))
) {
return;
}
const content = this.getContent();
this.client.logger.info(
`Decided to create file ${file} with content ${content}`
);
return this.create(file, new TextEncoder().encode(` ${content} `));
}
private async disableSyncAction(): Promise<void> {
this.client.logger.info(`Decided to disable sync`);
await this.client.setSetting("isSyncEnabled", false);
}
private async enableSyncAction(): Promise<void> {
this.client.logger.info(`Decided to enable sync`);
await this.client.setSetting("isSyncEnabled", true);
}
private async renameFileAction(files: RelativePath[]): Promise<void> {
const file = choose(files);
// We can't edit files offline that have been updated while offline.
// Otherwise, the resolution logic couldn't handle it.
if (
!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(file)
) {
this.client.logger.info(
`Skipping file ${file} because it has been updated while offline`
);
return;
}
const newName = this.getFileName();
if (
(!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(newName)) ||
(await this.exists(newName))
) {
return;
}
this.client.logger.info(`Decided to rename file ${file} to ${newName}`);
this.doNotTouchWhileOffline.push(file, newName);
return this.rename(file, newName);
}
private async updateFileAction(files: RelativePath[]): Promise<void> {
const file = choose(files);
// We can't edit files offline that have been updated while offline.
// Otherwise, the resolution logic couldn't handle it.
if (
!this.client.getSettings().isSyncEnabled &&
this.doNotTouchWhileOffline.includes(file)
) {
this.client.logger.info(
`Skipping file ${file} because it has been updated while offline`
);
return;
}
const content = this.getContent();
this.client.logger.info(
`Decided to update file ${file} with ${content}`
);
this.doNotTouchWhileOffline.push(file);
await this.atomicUpdateText(file, (old) => ({
text: old.text + ` ${content} `,
cursors: []
}));
}
private async deleteFileAction(files: RelativePath[]): Promise<void> {
const file = choose(files);
this.client.logger.info(`Decided to delete file ${file}`);
return this.delete(file);
}
private getContent(): string {
const uuid = uuidv4();
this.writtenContents.push(uuid);
return uuid;
}
private getFileName(): string {
// Simulate name collisions between the clients
return `file-${Math.floor(Math.random() * 64)}.md`;
}
}

View file

@ -1,197 +0,0 @@
import type { StoredDatabase, TextWithCursors } from "sync-client";
import { assert } from "../utils/assert";
import {
type RelativePath,
type FileSystemOperations,
type SyncSettings,
SyncClient
} from "sync-client";
export class MockClient implements FileSystemOperations {
protected readonly localFiles = new Map<string, Uint8Array>();
protected client!: SyncClient;
protected data: Partial<{
settings: Partial<SyncSettings>;
database: Partial<StoredDatabase>;
}> = {
database: {
// Assume all clients start at the same time so there's no need to fetch
// any shared state.
hasInitialSyncCompleted: true
}
};
public constructor(
initialSettings: Partial<SyncSettings>,
protected readonly useSlowFileEvents: boolean
) {
this.data.settings = initialSettings;
}
public async init(
fetchImplementation: typeof globalThis.fetch,
webSocketImplementation: typeof globalThis.WebSocket
): Promise<void> {
this.client = await SyncClient.create({
fs: this,
persistence: {
load: async () => this.data,
save: async (data) => void (this.data = data)
},
fetch: fetchImplementation,
webSocket: webSocketImplementation
});
await this.client.start();
}
public async listFilesRecursively(
_root: RelativePath | undefined = undefined // we don't use multi-level paths during tests
): Promise<RelativePath[]> {
return Array.from(this.localFiles.keys());
}
public async read(path: RelativePath): Promise<Uint8Array> {
const file = this.localFiles.get(path);
if (!file) {
throw new Error(`File ${path} does not exist`);
}
return file;
}
public async getFileSize(path: RelativePath): Promise<number> {
return (await this.read(path)).length;
}
public async exists(path: RelativePath): Promise<boolean> {
return this.localFiles.has(path);
}
public async create(
path: RelativePath,
newContent: Uint8Array
): Promise<void> {
if (this.localFiles.has(path)) {
throw new Error(`File ${path} already exists`);
}
this.client.logger.info(
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
);
this.localFiles.set(path, newContent);
this.executeFileOperation(async () =>
this.client.syncLocallyCreatedFile(path)
);
}
public async createDirectory(_path: RelativePath): Promise<void> {
// This doesn't mean anything in our virtual FS representation
}
public async atomicUpdateText(
path: RelativePath,
updater: (currentContent: TextWithCursors) => TextWithCursors
): Promise<string> {
const file = this.localFiles.get(path);
if (!file) {
throw new Error(`File ${path} does not exist`);
}
const currentContent = new TextDecoder().decode(file);
const newContent = updater({ text: currentContent, cursors: [] }).text;
const newContentUint8Array = new TextEncoder().encode(newContent);
this.localFiles.set(path, newContentUint8Array);
if (!this.useSlowFileEvents) {
const existingParts = currentContent
.split(" ")
.map((part) => part.trim());
const newParts = newContent.split(" ").map((part) => part.trim());
existingParts.forEach((part) =>
// all changes should be additive
{
assert(
newParts.includes(part),
`Part ${part} not found in new content: ${newContent}`
);
}
);
}
this.client.logger.info(
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
);
this.executeFileOperation(async () =>
this.client.syncLocallyUpdatedFile({
relativePath: path
})
);
return newContent;
}
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
const hasExisted = this.localFiles.has(path);
this.localFiles.set(path, content);
this.client.logger.info(
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
);
this.executeFileOperation(async () => {
if (hasExisted) {
return this.client.syncLocallyUpdatedFile({
relativePath: path
});
} else {
return this.client.syncLocallyCreatedFile(path);
}
});
}
public async delete(path: RelativePath): Promise<void> {
this.client.logger.info(
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
);
this.localFiles.delete(path);
this.executeFileOperation(async () =>
this.client.syncLocallyDeletedFile(path)
);
}
public async rename(
oldPath: RelativePath,
newPath: RelativePath
): Promise<void> {
const file = this.localFiles.get(oldPath);
if (!file) {
throw new Error(`File ${oldPath} does not exist`);
}
this.localFiles.set(newPath, file);
if (oldPath !== newPath) {
this.localFiles.delete(oldPath);
}
this.client.logger.info(
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
);
this.executeFileOperation(async () =>
this.client.syncLocallyUpdatedFile({
oldPath,
relativePath: newPath
})
);
}
private executeFileOperation(callback: () => unknown): void {
if (this.useSlowFileEvents) {
// we aren't the best client and it takes some time to notice changes
setTimeout(callback, Math.random() * 100);
} else {
callback();
}
}
}

View file

@ -1,205 +1,90 @@
import type { SyncSettings } from "sync-client";
import { utils } from "sync-client";
import { MockAgent } from "./agent/mock-agent";
import { sleep } from "./utils/sleep";
import { v4 as uuidv4 } from "uuid";
import { randomCasing } from "./utils/random-casing";
import { parseArgs } from "node:util";
import * as fs from "node:fs";
import * as path from "node:path";
import * as os from "node:os";
import { Logger } from "sync-client";
import { ServerControl } from "../../deterministic-tests/src/server-control";
import { ServerManager } from "../../deterministic-tests/src/server-manager";
import { TestRunner } from "../../deterministic-tests/src/test-runner";
import { generateTrace, traceTest, type Trace } from "./workload";
const TEST_ITERATIONS = 5;
// Simulate async file access by injecting waiting time before returning from file operations.
let slowFileEvents = false;
// Whether to do resets in the test runs
let doResets = false;
async function runTest({
agentCount,
concurrency,
iterations,
doDeletes,
useResets,
useSlowFileEvents,
jitterScaleInSeconds
}: {
agentCount: number;
concurrency: number;
iterations: number;
doDeletes: boolean;
useResets: boolean;
useSlowFileEvents: boolean;
jitterScaleInSeconds: number;
}): Promise<void> {
slowFileEvents = useSlowFileEvents;
doResets = useResets;
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
console.info(`Running test ${settings}`);
const vaultName = uuidv4();
console.info(`Using vault name: ${vaultName}`);
const initialSettings: Partial<SyncSettings> = {
isSyncEnabled: true,
token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces
vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter
syncConcurrency: concurrency,
remoteUri: "http://localhost:3000"
};
const clients: MockAgent[] = [];
for (let i = 0; i < agentCount; i++) {
clients.push(
new MockAgent(
initialSettings,
`agent-${i}`,
doDeletes,
useResets,
useSlowFileEvents,
jitterScaleInSeconds
)
);
}
try {
await utils.awaitAll(clients.map(async (client) => client.init()));
for (let i = 0; i < iterations; i++) {
console.info(`Iteration ${i + 1}/${iterations}`);
await utils.awaitAll(clients.map(async (client) => client.act()));
await sleep(Math.random() * 200);
}
console.info("Stopping agents");
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
for (const client of clients) {
try {
console.info(`Finishing up ${client.name}`);
await client.finish();
} catch (err) {
if (!slowFileEvents) {
throw err;
}
}
}
// then we need a second pass to ensure that all agents pull the same state.
for (const client of clients) {
try {
console.info(`Destroying ${client.name}`);
await client.destroy();
} catch (err) {
if (!slowFileEvents) {
throw err;
}
}
}
console.info("Agents finished successfully");
clients.slice(0, -1).forEach((client, i) => {
console.info(
`Checking consistency between ${client.name} and ${clients[i + 1].name}`
);
client.assertFileSystemsAreConsistent(clients[i]);
console.info(`Consistency check for ${client.name} passed`);
});
console.info("File systems found to be consistent");
clients.forEach((client) => {
console.info(`Checking content for ${client.name}`);
client.assertAllContentIsPresentOnce();
console.info(`Content check for ${client.name} passed`);
});
console.info(`Test passed ${settings}`);
} catch (err) {
console.error(`Test failed ${settings}`);
throw err;
}
function integer(value: string, zero = false): number {
const number = Number(value);
if (
!/^\d+$/.test(value) ||
!Number.isSafeInteger(number) ||
number < (zero ? 0 : 1)
)
throw new Error(`Invalid integer: ${value}`);
return number;
}
async function runTests(): Promise<void> {
for (let i = 0; i < TEST_ITERATIONS; i++) {
await runTest({
agentCount: 2,
concurrency: 16,
iterations: 100,
doDeletes: true,
useResets: true,
useSlowFileEvents: true,
jitterScaleInSeconds: 0.75
});
for (const useSlowFileEvents of [true, false]) {
for (const concurrency of [
16,
1 // test with concurrency 1 to check for deadlocks
]) {
for (const doDeletes of [false, true]) {
await runTest({
agentCount: 2,
concurrency,
iterations: 100,
doDeletes,
useResets: false,
useSlowFileEvents,
jitterScaleInSeconds: 0.75
});
}
}
async function main(): Promise<void> {
const { values } = parseArgs({
options: {
seed: { type: "string", default: "1" },
iterations: { type: "string", default: "25" },
replay: { type: "string" },
artifacts: { type: "string" },
"server-binary": { type: "string" },
config: { type: "string" }
}
}
}
process.on("uncaughtException", (error) => {
if (
error instanceof Error &&
error.message.includes(
"WebSocket was closed before the connection was established"
)
) {
return;
}
console.error("Uncaught exception:", error);
process.exit(1);
});
process.on("unhandledRejection", (error, _promise) => {
if (error instanceof Error && error.message === "Sync was reset") {
return;
}
if (
slowFileEvents &&
error instanceof Error &&
(error.message.includes("Document not found") ||
error.message.includes("Document already exists at new location"))
) {
return;
}
if (
doResets &&
error instanceof Error &&
error.message.includes(
"SyncClient has been destroyed and can no longer be used"
)
) {
return;
}
console.error("Unhandled rejection:", error);
process.exit(1);
});
runTests()
.then(() => {
process.exit(0);
})
.catch((err: unknown) => {
console.error(err);
process.exit(1);
});
const seed = integer(values.seed!, true),
iterations = integer(values.iterations!);
const trace: Trace = values.replay
? (JSON.parse(fs.readFileSync(values.replay, "utf8")) as Trace)
: generateTrace(seed, iterations);
const artifacts = values.artifacts
? path.resolve(values.artifacts)
: fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-fuzz-"));
fs.mkdirSync(artifacts, { recursive: true });
const tracePath = path.join(artifacts, `trace-${trace.seed}.json`);
fs.writeFileSync(tracePath, JSON.stringify(trace, null, 2));
console.log(`Replay: node ${__filename} --replay ${tracePath}`);
const logger = new Logger();
logger.onLogEmitted.add((line) => console.log(line.message));
const root = path.resolve(__dirname, "../../..");
const server = new ServerControl(
values["server-binary"] ??
path.join(root, "sync-server/target/release/sync_server"),
values.config ?? path.join(root, "sync-server/config-e2e.yml"),
logger
);
const manager = new ServerManager(logger);
manager.track(server);
manager.installSignalHandlers();
try {
await server.start();
const result = await new TestRunner(
server,
logger,
"test-token-change-me",
server.remoteUri
).runTest(`seed-${trace.seed}`, traceTest(trace));
fs.writeFileSync(
path.join(artifacts, `result-${trace.seed}.json`),
JSON.stringify(result, null, 2)
);
if (!result.success) throw new Error(result.error);
} finally {
await server.stop();
manager.untrack(server);
}
}
process.on("unhandledRejection", (error) => {
console.error(error);
process.exit(1);
});
process.on("uncaughtException", (error) => {
console.error(error);
process.exit(1);
});
main().then(
() => process.exit(0),
(error: unknown) => {
console.error(error);
process.exit(1);
}
);

View file

@ -6,7 +6,7 @@ describe("randomCasing", () => {
it("simple test", () => {
const input =
"hello, this is a really long string with a lot of characters";
const result = randomCasing(input);
const result = randomCasing(input, () => 0);
assert.strictEqual(result.toLowerCase(), input.toLowerCase());
assert.notStrictEqual(result, input);
});

View file

@ -1,7 +1,7 @@
export function randomCasing(str: string): string {
export function randomCasing(str: string, random: () => number = Math.random): string {
const chars = str.split("");
const randomCasedChars = chars.map((char) => {
if (Math.random() < 0.5) {
if (random() < 0.5) {
return char.toUpperCase();
}
return char.toLowerCase();

View file

@ -0,0 +1,71 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { generateTrace, traceTest } from "./workload";
test("concrete traces replay all choices and retain preservation assertions with delayed events", () => {
const trace = generateTrace(42, 12);
assert.deepEqual(trace, generateTrace(42, 12));
assert.notDeepEqual(trace, generateTrace(43, 12));
assert.deepEqual(
traceTest(trace),
traceTest(JSON.parse(JSON.stringify(trace)))
);
assert.equal(
trace.steps.filter((step) => step.type === "assert-markers").length,
12
);
assert(trace.steps.some((step) => step.type === "delay-notifications"));
assert(trace.steps.some((step) => step.type === "delete"));
assert(trace.steps.some((step) => step.type === "assert-documents"));
const assertions = trace.steps.filter(
(step) => step.type === "assert-documents"
);
assert(
assertions.at(-1)!.expected.some((doc) => doc.conflict),
"Independent creates need two expected documents"
);
assert(
assertions
.at(-1)!
.expected.some((doc) =>
assertions[0].expected.some(
(initial) => initial.key === doc.key
)
),
"Later assertions must retain earlier identities"
);
for (const kind of ["create", "manifest", "content"]) {
assert(
Array.from({ length: 10 }, (_, seed) =>
generateTrace(seed, 9)
).some((generated) =>
generated.steps.some(
(step) =>
step.type === "drop-response" && step.kind === kind
)
)
);
}
const delayed = trace.steps.findIndex(
(step) => step.type === "delay-notifications"
);
const flushed = trace.steps.findIndex(
(step, i) => i > delayed && step.type === "flush-notifications"
);
assert(
trace.steps
.slice(delayed, flushed)
.some((step) => step.type === "enable-sync")
);
assert(
trace.steps
.slice(delayed, flushed)
.some((step) => step.type === "assert-documents")
);
assert.equal(traceTest({ ...trace, clients: undefined }).clients, 2);
assert(
Array.from({ length: 10 }, (_, seed) => generateTrace(seed, 1)).some(
(generated) => generated.clients === 3
)
);
});

View file

@ -0,0 +1,310 @@
import type {
ExpectedDocument,
TestDefinition,
TestStep
} from "../../deterministic-tests/src/test-definition";
import { Random } from "../../test-support/random";
export interface Trace {
version: 1;
seed: number;
iterations: number;
/** Old traces omit this and retain their original two-client behavior. */
clients?: number;
steps: TestStep[];
}
type Document = ExpectedDocument & { content: string };
/** Concrete actions and independent expected states are serialized together.
* Every round checks the whole history, including earlier bytes and UUIDs.
* A shuffled cycle covers nine actions; most revisit existing documents.
*/
export function generateTrace(seed: number, iterations: number): Trace {
const random = new Random(seed);
const clients = 2 + random.int(2);
const steps: TestStep[] = [];
const documents: Document[] = [];
const markers: string[] = [];
const removed: string[] = [];
let serial = 0;
const newDocument = (path: string): Document => {
const key = `seed-${seed}-document-${serial++}`;
const marker = `[${key}]`;
markers.push(marker);
return {
key,
path,
content: `${marker}\n\nfirst paragraph\n\nlast paragraph\n`
};
};
const all = (type: "enable-sync" | "disable-sync") => {
for (let client = 0; client < clients; client++)
steps.push({ type, client });
};
const expected = () =>
steps.push({
type: "assert-documents",
expected: structuredClone(documents)
});
const barrier = () => {
steps.push({ type: "barrier" });
expected();
};
const forget = (doc: Document) => {
for (const marker of markers)
if (doc.content.includes(marker) && !removed.includes(marker))
removed.push(marker);
documents.splice(documents.indexOf(doc), 1);
};
all("enable-sync");
steps.push({ type: "barrier" });
// A lasting pool makes later actions interact with earlier versions/moves.
for (const path of ["shared/a.md", "日本語/b.md", "café/c.md"]) {
const doc = newDocument(path);
documents.push(doc);
steps.push({ type: "create", client: 0, path, content: doc.content });
}
barrier();
let choices: number[] = [];
for (let round = 0; round < iterations; round++) {
if (!choices.length) {
choices = Array.from({ length: 9 }, (_, i) => i);
for (let i = choices.length - 1; i > 0; i--) {
const j = random.int(i + 1);
[choices[i], choices[j]] = [choices[j], choices[i]];
}
}
const kind = choices.pop()!;
const writer = random.int(clients);
const other = (writer + 1 + random.int(clients - 1)) % clients;
const doc = random.pick(documents.filter((entry) => !entry.conflict));
const directory = `round-${round}/${random.pick(["nested", "日本語", "café", "space folder"])}`;
if (kind === 0) {
all("disable-sync");
const first = newDocument(`${directory}/same.md`);
const second = { ...newDocument(first.path), conflict: true };
steps.push(
{
type: "create",
client: writer,
path: first.path,
content: first.content
},
{
type: "create",
client: other,
path: second.path,
content: second.content
},
{
type: "drop-response",
client: writer,
kind: random.pick(["create", "manifest"] as const),
point: random.pick(["before", "after"] as const)
},
{ type: "enable-sync", client: writer },
{ type: "sync", client: writer },
{ type: "wait-for-response-drop", client: writer }
);
all("enable-sync");
documents.push(first, second);
} else if (kind === 1) {
// Both clients edit the same established version, at opposite ends.
all("disable-sync");
const prefix = `remote prefix ${round}\n\n`;
const suffix = `\n\nlocal suffix ${round}\n`;
steps.push(
{
type: "update",
client: writer,
path: doc.path,
content: doc.content + suffix
},
{
type: "update",
client: other,
path: doc.path,
content: prefix + doc.content
},
{ type: "enable-sync", client: writer }
);
all("enable-sync");
doc.content = prefix + doc.content + suffix;
} else if (kind === 2) {
all("disable-sync");
const target = `${directory}/moved.md`;
doc.content += `\nremote addition ${round}\n`;
steps.push(
{
type: "rename",
client: writer,
oldPath: doc.path,
newPath: target
},
{
type: "update",
client: other,
path: doc.path,
content: doc.content
},
{ type: "enable-sync", client: writer }
);
all("enable-sync");
doc.path = target;
} else if (kind === 3) {
const second = random.pick(
documents.filter((entry) => !entry.conflict && entry !== doc)
);
steps.push(
{ type: "disable-sync", client: writer },
{
type: "rename",
client: writer,
oldPath: doc.path,
newPath: `${directory}/temporary.md`
},
{
type: "rename",
client: writer,
oldPath: second.path,
newPath: doc.path
},
{
type: "rename",
client: writer,
oldPath: `${directory}/temporary.md`,
newPath: second.path
},
{ type: "enable-sync", client: writer }
);
[doc.path, second.path] = [second.path, doc.path];
} else if (kind === 4) {
// Reuse the same path but require a new UUID, even in later rounds.
steps.push({ type: "delete", client: writer, path: doc.path });
forget(doc);
barrier();
const replacement = newDocument(doc.path);
steps.push({
type: "create",
client: other,
path: replacement.path,
content: replacement.content
});
documents.push(replacement);
} else if (kind === 5) {
const target = `${directory}/rapid.md`;
steps.push(
{
type: "rename",
client: writer,
oldPath: doc.path,
newPath: target
},
{
type: "rename",
client: writer,
oldPath: target,
newPath: doc.path
},
{
type: "rename",
client: writer,
oldPath: doc.path,
newPath: target
}
);
doc.path = target;
doc.content += `\nrapid edit ${round}\n`;
steps.push({
type: "update",
client: writer,
path: doc.path,
content: doc.content
});
} else if (kind === 6) {
steps.push({
type: "drop-response",
client: writer,
kind: "content",
point: random.pick(["before", "after"] as const)
});
doc.content += `\nretry edit ${round}\n`;
steps.push(
{
type: "update",
client: writer,
path: doc.path,
content: doc.content
},
{ type: "wait-for-response-drop", client: writer }
);
} else if (kind === 7) {
// Keep an editor event delayed across reconnection and another edit.
steps.push(
{ type: "disable-sync", client: writer },
{ type: "delay-notifications", client: writer }
);
doc.content += `\ndelayed local edit ${round}\n`;
steps.push(
{
type: "update",
client: writer,
path: doc.path,
content: doc.content
},
{ type: "enable-sync", client: writer }
);
barrier();
doc.content += `\nlater remote edit ${round}\n`;
steps.push({
type: "update",
client: other,
path: doc.path,
content: doc.content
});
barrier();
steps.push({ type: "flush-notifications", client: writer });
} else {
// Explicit replacement retains the UUID. Only markers in the
// overwritten bytes are exempted from preservation.
for (const marker of markers)
if (doc.content.includes(marker) && !removed.includes(marker))
removed.push(marker);
doc.content = newDocument(doc.path).content;
steps.push({
type: "update",
client: writer,
path: doc.path,
content: doc.content
});
}
barrier();
steps.push({
type: "assert-markers",
markers: [...markers],
removed: [...removed]
});
if (random.int(4) === 0) {
steps.push({ type: "reset", client: random.int(clients) });
barrier();
}
}
return { version: 1, seed, iterations, clients, steps };
}
export function traceTest(trace: Trace): TestDefinition {
if (
trace.version !== 1 ||
!Array.isArray(trace.steps) ||
!trace.steps.length ||
(trace.clients !== undefined &&
(!Number.isInteger(trace.clients) || trace.clients < 2))
)
throw new Error("Invalid/empty replay trace");
return {
clients: trace.clients ?? 2,
description: `Seed ${trace.seed}; ${trace.iterations} generated conflict scenarios`,
steps: trace.steps
};
}

View file

@ -2,6 +2,7 @@
"compilerOptions": {
"baseUrl": ".",
"strict": true,
"skipLibCheck": true,
"target": "ES2022",
"module": "CommonJS",
"esModuleInterop": true,

View file

@ -4,6 +4,7 @@ const webpack = require("webpack");
module.exports = {
entry: "./src/cli.ts",
target: "node",
externals: { "sync-client": "commonjs sync-client" },
mode: "production",
optimization: {
minimize: false