power loss
This commit is contained in:
parent
0f6b618c0b
commit
d56084e4bc
7 changed files with 18667 additions and 3902 deletions
|
|
@ -1,368 +0,0 @@
|
|||
import { choose } from "../utils/choose";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { assert } from "../utils/assert";
|
||||
import type { RelativePath, SyncSettings } from "sync-client";
|
||||
import { debugging, Logger, LogLevel, utils } from "sync-client";
|
||||
import { MockClient } from "./mock-client";
|
||||
import { sleep } from "../utils/sleep";
|
||||
import type { LogLine } from "sync-client";
|
||||
import { withTimeout } from "../utils/with-timeout";
|
||||
|
||||
const TIMEOUT_MS = 10 * 60 * 1000;
|
||||
|
||||
export class MockAgent extends MockClient {
|
||||
private readonly writtenContents: string[] = [];
|
||||
private readonly pendingActions: Promise<unknown>[] = [];
|
||||
|
||||
// The renamed file finding algorithm isn't too smart so we can't both update and rename the same file
|
||||
private readonly doNotTouchWhileOffline: string[] = [];
|
||||
|
||||
public constructor(
|
||||
initialSettings: Partial<SyncSettings>,
|
||||
public readonly name: string,
|
||||
private readonly doDeletes: boolean,
|
||||
private readonly doResets: boolean,
|
||||
useSlowFileEvents: boolean,
|
||||
private readonly jitterScaleInSeconds: number
|
||||
) {
|
||||
super(initialSettings, useSlowFileEvents);
|
||||
}
|
||||
|
||||
public async init(): Promise<void> {
|
||||
await super.init(
|
||||
debugging.slowFetchFactory(this.jitterScaleInSeconds),
|
||||
debugging.slowWebSocketFactory(
|
||||
this.jitterScaleInSeconds,
|
||||
new Logger() // this logger isn't wired anywhere, so messages to it will be ignored
|
||||
)
|
||||
);
|
||||
|
||||
assert(
|
||||
(await this.client.checkConnection()).isSuccessful,
|
||||
"Connection check failed"
|
||||
);
|
||||
|
||||
this.client.logger.onLogEmitted.add((logLine: LogLine) => {
|
||||
const state = this.client.getSettings().isSyncEnabled
|
||||
? "(online) "
|
||||
: "(offline)";
|
||||
const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`;
|
||||
|
||||
// HACK: we have to ensure the file has been synced if we want to change it offline without data loss
|
||||
const historyEntry = /.*History entry: (.*.md).*/.exec(
|
||||
logLine.message
|
||||
);
|
||||
|
||||
if (historyEntry) {
|
||||
utils.removeFromArray(
|
||||
this.doNotTouchWhileOffline,
|
||||
historyEntry[1]
|
||||
);
|
||||
}
|
||||
switch (logLine.level) {
|
||||
case LogLevel.ERROR:
|
||||
console.error(formatted);
|
||||
|
||||
if (!this.useSlowFileEvents) {
|
||||
// Let's wait for the error to be caught if there was one
|
||||
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
||||
sleep(100).then(() => process.exit(1));
|
||||
}
|
||||
|
||||
break;
|
||||
case LogLevel.WARNING:
|
||||
console.warn(formatted);
|
||||
break;
|
||||
case LogLevel.INFO:
|
||||
console.info(formatted);
|
||||
break;
|
||||
case LogLevel.DEBUG:
|
||||
console.debug(formatted);
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
this.client.logger.info("Agent initialized");
|
||||
}
|
||||
|
||||
public async act(): Promise<void> {
|
||||
const options: (() => Promise<unknown>)[] = [
|
||||
this.createFileAction.bind(this)
|
||||
];
|
||||
|
||||
if (
|
||||
this.client.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.length === 0
|
||||
) {
|
||||
options.push(this.disableSyncAction.bind(this));
|
||||
} else {
|
||||
options.push(this.enableSyncAction.bind(this));
|
||||
}
|
||||
|
||||
const files = await this.listFilesRecursively();
|
||||
|
||||
if (files.length > 0) {
|
||||
options.push(
|
||||
this.renameFileAction.bind(this, files),
|
||||
this.updateFileAction.bind(this, files)
|
||||
);
|
||||
|
||||
if (this.doDeletes) {
|
||||
options.push(this.deleteFileAction.bind(this, files));
|
||||
}
|
||||
}
|
||||
|
||||
if (Math.random() < 0.015 && this.doResets) {
|
||||
// we can't just queue this up as once it's destroyed, no more method calls can go to SyncClient
|
||||
await this.resetClient();
|
||||
} else {
|
||||
this.pendingActions.push(
|
||||
(async (): Promise<unknown> => {
|
||||
try {
|
||||
return await choose(options)();
|
||||
} catch (error) {
|
||||
this.client.logger.error(
|
||||
`Failed to perform an action: ${error}`
|
||||
);
|
||||
this.client.logger.info(
|
||||
JSON.stringify(this.data, null, 2)
|
||||
);
|
||||
this.client.logger.info(
|
||||
JSON.stringify(this.localFiles, null, 2)
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
})()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public async finish(): Promise<void> {
|
||||
await withTimeout(
|
||||
(async (): Promise<void> => {
|
||||
await this.client.setSetting("isSyncEnabled", true);
|
||||
await utils.awaitAll(this.pendingActions);
|
||||
await this.client.waitUntilFinished();
|
||||
})(),
|
||||
TIMEOUT_MS,
|
||||
"finish()"
|
||||
);
|
||||
}
|
||||
|
||||
public async destroy(): Promise<void> {
|
||||
await withTimeout(
|
||||
(async (): Promise<void> => {
|
||||
await this.client.waitUntilFinished();
|
||||
await this.client.destroy();
|
||||
})(),
|
||||
TIMEOUT_MS,
|
||||
"destroy()"
|
||||
);
|
||||
}
|
||||
|
||||
public assertFileSystemsAreConsistent(otherAgent: MockAgent): void {
|
||||
const globalFiles = Array.from(otherAgent.localFiles.keys());
|
||||
const localFiles = Array.from(this.localFiles.keys());
|
||||
|
||||
const missingInOther = localFiles.filter(
|
||||
(file) => !otherAgent.localFiles.has(file)
|
||||
);
|
||||
const missingInLocal = globalFiles.filter(
|
||||
(file) => !this.localFiles.has(file)
|
||||
);
|
||||
|
||||
try {
|
||||
assert(
|
||||
missingInOther.length === 0,
|
||||
`Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}`
|
||||
);
|
||||
assert(
|
||||
missingInLocal.length === 0,
|
||||
`Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}`
|
||||
);
|
||||
|
||||
for (const file of globalFiles) {
|
||||
const localContent = new TextDecoder().decode(
|
||||
this.localFiles.get(file)
|
||||
);
|
||||
const otherContent = new TextDecoder().decode(
|
||||
otherAgent.localFiles.get(file)
|
||||
);
|
||||
assert(
|
||||
localContent === otherContent,
|
||||
`Content mismatch for file ${file}:\n${localContent}\n${otherContent}`
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
this.client.logger.info(
|
||||
"Local data: " + JSON.stringify(this.data, null, 2)
|
||||
);
|
||||
this.client.logger.info(
|
||||
"Local files: " +
|
||||
Array.from(otherAgent.localFiles.keys()).join(", ")
|
||||
);
|
||||
otherAgent.client.logger.info(
|
||||
"Local data: " + JSON.stringify(otherAgent.data, null, 2)
|
||||
);
|
||||
otherAgent.client.logger.info(
|
||||
"Local files: " +
|
||||
Array.from(otherAgent.localFiles.keys()).join(", ")
|
||||
);
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public assertAllContentIsPresentOnce(): void {
|
||||
if (this.useSlowFileEvents) {
|
||||
this.client.logger.info(
|
||||
// We can't ensure that we have seen every single update
|
||||
`Skipping content check for ${this.name} because slow file events are enabled`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const content of this.writtenContents) {
|
||||
const found = Array.from(this.localFiles.keys()).filter((key) => {
|
||||
return new TextDecoder()
|
||||
.decode(this.localFiles.get(key))
|
||||
.includes(content);
|
||||
});
|
||||
|
||||
if (this.doDeletes) {
|
||||
assert(
|
||||
found.length <= 1,
|
||||
`[${this.name}] Content ${content} found in ${found.join(", ")}`
|
||||
);
|
||||
} else {
|
||||
assert(
|
||||
found.length >= 1,
|
||||
`[${this.name}] Content ${content} not found in any files`
|
||||
);
|
||||
|
||||
assert(
|
||||
found.length <= 1,
|
||||
`[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}`
|
||||
);
|
||||
|
||||
const [file] = found;
|
||||
const fileContent = new TextDecoder().decode(
|
||||
this.localFiles.get(file)
|
||||
);
|
||||
assert(
|
||||
fileContent.split(content).length == 2,
|
||||
`Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async resetClient(): Promise<void> {
|
||||
this.client.logger.info(`Resetting client ${this.name}`);
|
||||
await this.client.destroy();
|
||||
await this.init();
|
||||
}
|
||||
|
||||
private async createFileAction(): Promise<void> {
|
||||
const file = this.getFileName();
|
||||
|
||||
if (
|
||||
(!this.client.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.includes(file)) ||
|
||||
(await this.exists(file))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const content = this.getContent();
|
||||
this.client.logger.info(
|
||||
`Decided to create file ${file} with content ${content}`
|
||||
);
|
||||
|
||||
return this.create(file, new TextEncoder().encode(` ${content} `));
|
||||
}
|
||||
|
||||
private async disableSyncAction(): Promise<void> {
|
||||
this.client.logger.info(`Decided to disable sync`);
|
||||
await this.client.setSetting("isSyncEnabled", false);
|
||||
}
|
||||
|
||||
private async enableSyncAction(): Promise<void> {
|
||||
this.client.logger.info(`Decided to enable sync`);
|
||||
await this.client.setSetting("isSyncEnabled", true);
|
||||
}
|
||||
|
||||
private async renameFileAction(files: RelativePath[]): Promise<void> {
|
||||
const file = choose(files);
|
||||
|
||||
// We can't edit files offline that have been updated while offline.
|
||||
// Otherwise, the resolution logic couldn't handle it.
|
||||
if (
|
||||
!this.client.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.includes(file)
|
||||
) {
|
||||
this.client.logger.info(
|
||||
`Skipping file ${file} because it has been updated while offline`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const newName = this.getFileName();
|
||||
|
||||
if (
|
||||
(!this.client.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.includes(newName)) ||
|
||||
(await this.exists(newName))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.client.logger.info(`Decided to rename file ${file} to ${newName}`);
|
||||
this.doNotTouchWhileOffline.push(file, newName);
|
||||
|
||||
return this.rename(file, newName);
|
||||
}
|
||||
|
||||
private async updateFileAction(files: RelativePath[]): Promise<void> {
|
||||
const file = choose(files);
|
||||
|
||||
// We can't edit files offline that have been updated while offline.
|
||||
// Otherwise, the resolution logic couldn't handle it.
|
||||
if (
|
||||
!this.client.getSettings().isSyncEnabled &&
|
||||
this.doNotTouchWhileOffline.includes(file)
|
||||
) {
|
||||
this.client.logger.info(
|
||||
`Skipping file ${file} because it has been updated while offline`
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const content = this.getContent();
|
||||
this.client.logger.info(
|
||||
`Decided to update file ${file} with ${content}`
|
||||
);
|
||||
this.doNotTouchWhileOffline.push(file);
|
||||
await this.atomicUpdateText(file, (old) => ({
|
||||
text: old.text + ` ${content} `,
|
||||
cursors: []
|
||||
}));
|
||||
}
|
||||
|
||||
private async deleteFileAction(files: RelativePath[]): Promise<void> {
|
||||
const file = choose(files);
|
||||
this.client.logger.info(`Decided to delete file ${file}`);
|
||||
return this.delete(file);
|
||||
}
|
||||
|
||||
private getContent(): string {
|
||||
const uuid = uuidv4();
|
||||
this.writtenContents.push(uuid);
|
||||
return uuid;
|
||||
}
|
||||
|
||||
private getFileName(): string {
|
||||
// Simulate name collisions between the clients
|
||||
return `file-${Math.floor(Math.random() * 64)}.md`;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
import type { StoredDatabase, TextWithCursors } from "sync-client";
|
||||
import { assert } from "../utils/assert";
|
||||
import {
|
||||
type RelativePath,
|
||||
type FileSystemOperations,
|
||||
type SyncSettings,
|
||||
SyncClient
|
||||
} from "sync-client";
|
||||
|
||||
export class MockClient implements FileSystemOperations {
|
||||
protected readonly localFiles = new Map<string, Uint8Array>();
|
||||
protected client!: SyncClient;
|
||||
|
||||
protected data: Partial<{
|
||||
settings: Partial<SyncSettings>;
|
||||
database: Partial<StoredDatabase>;
|
||||
}> = {
|
||||
database: {
|
||||
// Assume all clients start at the same time so there's no need to fetch
|
||||
// any shared state.
|
||||
hasInitialSyncCompleted: true
|
||||
}
|
||||
};
|
||||
|
||||
public constructor(
|
||||
initialSettings: Partial<SyncSettings>,
|
||||
protected readonly useSlowFileEvents: boolean
|
||||
) {
|
||||
this.data.settings = initialSettings;
|
||||
}
|
||||
|
||||
public async init(
|
||||
fetchImplementation: typeof globalThis.fetch,
|
||||
webSocketImplementation: typeof globalThis.WebSocket
|
||||
): Promise<void> {
|
||||
this.client = await SyncClient.create({
|
||||
fs: this,
|
||||
persistence: {
|
||||
load: async () => this.data,
|
||||
save: async (data) => void (this.data = data)
|
||||
},
|
||||
fetch: fetchImplementation,
|
||||
webSocket: webSocketImplementation
|
||||
});
|
||||
|
||||
await this.client.start();
|
||||
}
|
||||
|
||||
public async listFilesRecursively(
|
||||
_root: RelativePath | undefined = undefined // we don't use multi-level paths during tests
|
||||
): Promise<RelativePath[]> {
|
||||
return Array.from(this.localFiles.keys());
|
||||
}
|
||||
|
||||
public async read(path: RelativePath): Promise<Uint8Array> {
|
||||
const file = this.localFiles.get(path);
|
||||
if (!file) {
|
||||
throw new Error(`File ${path} does not exist`);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
public async getFileSize(path: RelativePath): Promise<number> {
|
||||
return (await this.read(path)).length;
|
||||
}
|
||||
|
||||
public async exists(path: RelativePath): Promise<boolean> {
|
||||
return this.localFiles.has(path);
|
||||
}
|
||||
|
||||
public async create(
|
||||
path: RelativePath,
|
||||
newContent: Uint8Array
|
||||
): Promise<void> {
|
||||
if (this.localFiles.has(path)) {
|
||||
throw new Error(`File ${path} already exists`);
|
||||
}
|
||||
this.client.logger.info(
|
||||
`Creating file ${path} with content ${new TextDecoder().decode(newContent)}`
|
||||
);
|
||||
this.localFiles.set(path, newContent);
|
||||
|
||||
this.executeFileOperation(async () =>
|
||||
this.client.syncLocallyCreatedFile(path)
|
||||
);
|
||||
}
|
||||
|
||||
public async createDirectory(_path: RelativePath): Promise<void> {
|
||||
// This doesn't mean anything in our virtual FS representation
|
||||
}
|
||||
|
||||
public async atomicUpdateText(
|
||||
path: RelativePath,
|
||||
updater: (currentContent: TextWithCursors) => TextWithCursors
|
||||
): Promise<string> {
|
||||
const file = this.localFiles.get(path);
|
||||
if (!file) {
|
||||
throw new Error(`File ${path} does not exist`);
|
||||
}
|
||||
const currentContent = new TextDecoder().decode(file);
|
||||
const newContent = updater({ text: currentContent, cursors: [] }).text;
|
||||
const newContentUint8Array = new TextEncoder().encode(newContent);
|
||||
this.localFiles.set(path, newContentUint8Array);
|
||||
|
||||
if (!this.useSlowFileEvents) {
|
||||
const existingParts = currentContent
|
||||
.split(" ")
|
||||
.map((part) => part.trim());
|
||||
const newParts = newContent.split(" ").map((part) => part.trim());
|
||||
existingParts.forEach((part) =>
|
||||
// all changes should be additive
|
||||
{
|
||||
assert(
|
||||
newParts.includes(part),
|
||||
`Part ${part} not found in new content: ${newContent}`
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
this.client.logger.info(
|
||||
`Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}`
|
||||
);
|
||||
|
||||
this.executeFileOperation(async () =>
|
||||
this.client.syncLocallyUpdatedFile({
|
||||
relativePath: path
|
||||
})
|
||||
);
|
||||
|
||||
return newContent;
|
||||
}
|
||||
|
||||
public async write(path: RelativePath, content: Uint8Array): Promise<void> {
|
||||
const hasExisted = this.localFiles.has(path);
|
||||
this.localFiles.set(path, content);
|
||||
|
||||
this.client.logger.info(
|
||||
`Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}`
|
||||
);
|
||||
|
||||
this.executeFileOperation(async () => {
|
||||
if (hasExisted) {
|
||||
return this.client.syncLocallyUpdatedFile({
|
||||
relativePath: path
|
||||
});
|
||||
} else {
|
||||
return this.client.syncLocallyCreatedFile(path);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async delete(path: RelativePath): Promise<void> {
|
||||
this.client.logger.info(
|
||||
`Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}`
|
||||
);
|
||||
this.localFiles.delete(path);
|
||||
|
||||
this.executeFileOperation(async () =>
|
||||
this.client.syncLocallyDeletedFile(path)
|
||||
);
|
||||
}
|
||||
|
||||
public async rename(
|
||||
oldPath: RelativePath,
|
||||
newPath: RelativePath
|
||||
): Promise<void> {
|
||||
const file = this.localFiles.get(oldPath);
|
||||
if (!file) {
|
||||
throw new Error(`File ${oldPath} does not exist`);
|
||||
}
|
||||
this.localFiles.set(newPath, file);
|
||||
if (oldPath !== newPath) {
|
||||
this.localFiles.delete(oldPath);
|
||||
}
|
||||
|
||||
this.client.logger.info(
|
||||
`Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}`
|
||||
);
|
||||
|
||||
this.executeFileOperation(async () =>
|
||||
this.client.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath: newPath
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
private executeFileOperation(callback: () => unknown): void {
|
||||
if (this.useSlowFileEvents) {
|
||||
// we aren't the best client and it takes some time to notice changes
|
||||
setTimeout(callback, Math.random() * 100);
|
||||
} else {
|
||||
callback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,205 +1,90 @@
|
|||
import type { SyncSettings } from "sync-client";
|
||||
import { utils } from "sync-client";
|
||||
import { MockAgent } from "./agent/mock-agent";
|
||||
import { sleep } from "./utils/sleep";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomCasing } from "./utils/random-casing";
|
||||
import { parseArgs } from "node:util";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import * as os from "node:os";
|
||||
import { Logger } from "sync-client";
|
||||
import { ServerControl } from "../../deterministic-tests/src/server-control";
|
||||
import { ServerManager } from "../../deterministic-tests/src/server-manager";
|
||||
import { TestRunner } from "../../deterministic-tests/src/test-runner";
|
||||
import { generateTrace, traceTest, type Trace } from "./workload";
|
||||
|
||||
const TEST_ITERATIONS = 5;
|
||||
|
||||
// Simulate async file access by injecting waiting time before returning from file operations.
|
||||
let slowFileEvents = false;
|
||||
|
||||
// Whether to do resets in the test runs
|
||||
let doResets = false;
|
||||
|
||||
async function runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations,
|
||||
doDeletes,
|
||||
useResets,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
}: {
|
||||
agentCount: number;
|
||||
concurrency: number;
|
||||
iterations: number;
|
||||
doDeletes: boolean;
|
||||
useResets: boolean;
|
||||
useSlowFileEvents: boolean;
|
||||
jitterScaleInSeconds: number;
|
||||
}): Promise<void> {
|
||||
slowFileEvents = useSlowFileEvents;
|
||||
doResets = useResets;
|
||||
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
|
||||
console.info(`Running test ${settings}`);
|
||||
|
||||
const vaultName = uuidv4();
|
||||
console.info(`Using vault name: ${vaultName}`);
|
||||
const initialSettings: Partial<SyncSettings> = {
|
||||
isSyncEnabled: true,
|
||||
token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces
|
||||
vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter
|
||||
syncConcurrency: concurrency,
|
||||
remoteUri: "http://localhost:3000"
|
||||
};
|
||||
|
||||
const clients: MockAgent[] = [];
|
||||
for (let i = 0; i < agentCount; i++) {
|
||||
clients.push(
|
||||
new MockAgent(
|
||||
initialSettings,
|
||||
`agent-${i}`,
|
||||
doDeletes,
|
||||
useResets,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await utils.awaitAll(clients.map(async (client) => client.init()));
|
||||
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
console.info(`Iteration ${i + 1}/${iterations}`);
|
||||
await utils.awaitAll(clients.map(async (client) => client.act()));
|
||||
await sleep(Math.random() * 200);
|
||||
}
|
||||
|
||||
console.info("Stopping agents");
|
||||
|
||||
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
|
||||
for (const client of clients) {
|
||||
try {
|
||||
console.info(`Finishing up ${client.name}`);
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// then we need a second pass to ensure that all agents pull the same state.
|
||||
for (const client of clients) {
|
||||
try {
|
||||
console.info(`Destroying ${client.name}`);
|
||||
await client.destroy();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.info("Agents finished successfully");
|
||||
|
||||
clients.slice(0, -1).forEach((client, i) => {
|
||||
console.info(
|
||||
`Checking consistency between ${client.name} and ${clients[i + 1].name}`
|
||||
);
|
||||
client.assertFileSystemsAreConsistent(clients[i]);
|
||||
console.info(`Consistency check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info("File systems found to be consistent");
|
||||
|
||||
clients.forEach((client) => {
|
||||
console.info(`Checking content for ${client.name}`);
|
||||
client.assertAllContentIsPresentOnce();
|
||||
console.info(`Content check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info(`Test passed ${settings}`);
|
||||
} catch (err) {
|
||||
console.error(`Test failed ${settings}`);
|
||||
throw err;
|
||||
}
|
||||
function integer(value: string, zero = false): number {
|
||||
const number = Number(value);
|
||||
if (
|
||||
!/^\d+$/.test(value) ||
|
||||
!Number.isSafeInteger(number) ||
|
||||
number < (zero ? 0 : 1)
|
||||
)
|
||||
throw new Error(`Invalid integer: ${value}`);
|
||||
return number;
|
||||
}
|
||||
|
||||
async function runTests(): Promise<void> {
|
||||
for (let i = 0; i < TEST_ITERATIONS; i++) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
concurrency: 16,
|
||||
iterations: 100,
|
||||
doDeletes: true,
|
||||
useResets: true,
|
||||
useSlowFileEvents: true,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
|
||||
for (const useSlowFileEvents of [true, false]) {
|
||||
for (const concurrency of [
|
||||
16,
|
||||
1 // test with concurrency 1 to check for deadlocks
|
||||
]) {
|
||||
for (const doDeletes of [false, true]) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
concurrency,
|
||||
iterations: 100,
|
||||
doDeletes,
|
||||
useResets: false,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
}
|
||||
}
|
||||
async function main(): Promise<void> {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
seed: { type: "string", default: "1" },
|
||||
iterations: { type: "string", default: "25" },
|
||||
replay: { type: "string" },
|
||||
artifacts: { type: "string" },
|
||||
"server-binary": { type: "string" },
|
||||
config: { type: "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process.on("uncaughtException", (error) => {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
"WebSocket was closed before the connection was established"
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("Uncaught exception:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (error, _promise) => {
|
||||
if (error instanceof Error && error.message === "Sync was reset") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
slowFileEvents &&
|
||||
error instanceof Error &&
|
||||
(error.message.includes("Document not found") ||
|
||||
error.message.includes("Document already exists at new location"))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
doResets &&
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
"SyncClient has been destroyed and can no longer be used"
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("Unhandled rejection:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runTests()
|
||||
.then(() => {
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
const seed = integer(values.seed!, true),
|
||||
iterations = integer(values.iterations!);
|
||||
const trace: Trace = values.replay
|
||||
? (JSON.parse(fs.readFileSync(values.replay, "utf8")) as Trace)
|
||||
: generateTrace(seed, iterations);
|
||||
const artifacts = values.artifacts
|
||||
? path.resolve(values.artifacts)
|
||||
: fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-fuzz-"));
|
||||
fs.mkdirSync(artifacts, { recursive: true });
|
||||
const tracePath = path.join(artifacts, `trace-${trace.seed}.json`);
|
||||
fs.writeFileSync(tracePath, JSON.stringify(trace, null, 2));
|
||||
console.log(`Replay: node ${__filename} --replay ${tracePath}`);
|
||||
const logger = new Logger();
|
||||
logger.onLogEmitted.add((line) => console.log(line.message));
|
||||
const root = path.resolve(__dirname, "../../..");
|
||||
const server = new ServerControl(
|
||||
values["server-binary"] ??
|
||||
path.join(root, "sync-server/target/release/sync_server"),
|
||||
values.config ?? path.join(root, "sync-server/config-e2e.yml"),
|
||||
logger
|
||||
);
|
||||
const manager = new ServerManager(logger);
|
||||
manager.track(server);
|
||||
manager.installSignalHandlers();
|
||||
try {
|
||||
await server.start();
|
||||
const result = await new TestRunner(
|
||||
server,
|
||||
logger,
|
||||
"test-token-change-me",
|
||||
server.remoteUri
|
||||
).runTest(`seed-${trace.seed}`, traceTest(trace));
|
||||
fs.writeFileSync(
|
||||
path.join(artifacts, `result-${trace.seed}.json`),
|
||||
JSON.stringify(result, null, 2)
|
||||
);
|
||||
if (!result.success) throw new Error(result.error);
|
||||
} finally {
|
||||
await server.stop();
|
||||
manager.untrack(server);
|
||||
}
|
||||
}
|
||||
|
||||
process.on("unhandledRejection", (error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
process.on("uncaughtException", (error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
main().then(
|
||||
() => process.exit(0),
|
||||
(error: unknown) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ describe("randomCasing", () => {
|
|||
it("simple test", () => {
|
||||
const input =
|
||||
"hello, this is a really long string with a lot of characters";
|
||||
const result = randomCasing(input);
|
||||
const result = randomCasing(input, () => 0);
|
||||
assert.strictEqual(result.toLowerCase(), input.toLowerCase());
|
||||
assert.notStrictEqual(result, input);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
export function randomCasing(str: string): string {
|
||||
export function randomCasing(str: string, random: () => number = Math.random): string {
|
||||
const chars = str.split("");
|
||||
const randomCasedChars = chars.map((char) => {
|
||||
if (Math.random() < 0.5) {
|
||||
if (random() < 0.5) {
|
||||
return char.toUpperCase();
|
||||
}
|
||||
return char.toLowerCase();
|
||||
|
|
|
|||
71
frontend/test-client/src/workload.test.ts
Normal file
71
frontend/test-client/src/workload.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { generateTrace, traceTest } from "./workload";
|
||||
|
||||
test("concrete traces replay all choices and retain preservation assertions with delayed events", () => {
|
||||
const trace = generateTrace(42, 12);
|
||||
assert.deepEqual(trace, generateTrace(42, 12));
|
||||
assert.notDeepEqual(trace, generateTrace(43, 12));
|
||||
assert.deepEqual(
|
||||
traceTest(trace),
|
||||
traceTest(JSON.parse(JSON.stringify(trace)))
|
||||
);
|
||||
assert.equal(
|
||||
trace.steps.filter((step) => step.type === "assert-markers").length,
|
||||
12
|
||||
);
|
||||
assert(trace.steps.some((step) => step.type === "delay-notifications"));
|
||||
assert(trace.steps.some((step) => step.type === "delete"));
|
||||
assert(trace.steps.some((step) => step.type === "assert-documents"));
|
||||
const assertions = trace.steps.filter(
|
||||
(step) => step.type === "assert-documents"
|
||||
);
|
||||
assert(
|
||||
assertions.at(-1)!.expected.some((doc) => doc.conflict),
|
||||
"Independent creates need two expected documents"
|
||||
);
|
||||
assert(
|
||||
assertions
|
||||
.at(-1)!
|
||||
.expected.some((doc) =>
|
||||
assertions[0].expected.some(
|
||||
(initial) => initial.key === doc.key
|
||||
)
|
||||
),
|
||||
"Later assertions must retain earlier identities"
|
||||
);
|
||||
for (const kind of ["create", "manifest", "content"]) {
|
||||
assert(
|
||||
Array.from({ length: 10 }, (_, seed) =>
|
||||
generateTrace(seed, 9)
|
||||
).some((generated) =>
|
||||
generated.steps.some(
|
||||
(step) =>
|
||||
step.type === "drop-response" && step.kind === kind
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
const delayed = trace.steps.findIndex(
|
||||
(step) => step.type === "delay-notifications"
|
||||
);
|
||||
const flushed = trace.steps.findIndex(
|
||||
(step, i) => i > delayed && step.type === "flush-notifications"
|
||||
);
|
||||
assert(
|
||||
trace.steps
|
||||
.slice(delayed, flushed)
|
||||
.some((step) => step.type === "enable-sync")
|
||||
);
|
||||
assert(
|
||||
trace.steps
|
||||
.slice(delayed, flushed)
|
||||
.some((step) => step.type === "assert-documents")
|
||||
);
|
||||
assert.equal(traceTest({ ...trace, clients: undefined }).clients, 2);
|
||||
assert(
|
||||
Array.from({ length: 10 }, (_, seed) => generateTrace(seed, 1)).some(
|
||||
(generated) => generated.clients === 3
|
||||
)
|
||||
);
|
||||
});
|
||||
310
frontend/test-client/src/workload.ts
Normal file
310
frontend/test-client/src/workload.ts
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
import type {
|
||||
ExpectedDocument,
|
||||
TestDefinition,
|
||||
TestStep
|
||||
} from "../../deterministic-tests/src/test-definition";
|
||||
import { Random } from "../../test-support/random";
|
||||
|
||||
export interface Trace {
|
||||
version: 1;
|
||||
seed: number;
|
||||
iterations: number;
|
||||
/** Old traces omit this and retain their original two-client behavior. */
|
||||
clients?: number;
|
||||
steps: TestStep[];
|
||||
}
|
||||
|
||||
type Document = ExpectedDocument & { content: string };
|
||||
|
||||
/** Concrete actions and independent expected states are serialized together.
|
||||
* Every round checks the whole history, including earlier bytes and UUIDs.
|
||||
* A shuffled cycle covers nine actions; most revisit existing documents.
|
||||
*/
|
||||
export function generateTrace(seed: number, iterations: number): Trace {
|
||||
const random = new Random(seed);
|
||||
const clients = 2 + random.int(2);
|
||||
const steps: TestStep[] = [];
|
||||
const documents: Document[] = [];
|
||||
const markers: string[] = [];
|
||||
const removed: string[] = [];
|
||||
let serial = 0;
|
||||
const newDocument = (path: string): Document => {
|
||||
const key = `seed-${seed}-document-${serial++}`;
|
||||
const marker = `[${key}]`;
|
||||
markers.push(marker);
|
||||
return {
|
||||
key,
|
||||
path,
|
||||
content: `${marker}\n\nfirst paragraph\n\nlast paragraph\n`
|
||||
};
|
||||
};
|
||||
const all = (type: "enable-sync" | "disable-sync") => {
|
||||
for (let client = 0; client < clients; client++)
|
||||
steps.push({ type, client });
|
||||
};
|
||||
const expected = () =>
|
||||
steps.push({
|
||||
type: "assert-documents",
|
||||
expected: structuredClone(documents)
|
||||
});
|
||||
const barrier = () => {
|
||||
steps.push({ type: "barrier" });
|
||||
expected();
|
||||
};
|
||||
const forget = (doc: Document) => {
|
||||
for (const marker of markers)
|
||||
if (doc.content.includes(marker) && !removed.includes(marker))
|
||||
removed.push(marker);
|
||||
documents.splice(documents.indexOf(doc), 1);
|
||||
};
|
||||
all("enable-sync");
|
||||
steps.push({ type: "barrier" });
|
||||
// A lasting pool makes later actions interact with earlier versions/moves.
|
||||
for (const path of ["shared/a.md", "日本語/b.md", "café/c.md"]) {
|
||||
const doc = newDocument(path);
|
||||
documents.push(doc);
|
||||
steps.push({ type: "create", client: 0, path, content: doc.content });
|
||||
}
|
||||
barrier();
|
||||
let choices: number[] = [];
|
||||
for (let round = 0; round < iterations; round++) {
|
||||
if (!choices.length) {
|
||||
choices = Array.from({ length: 9 }, (_, i) => i);
|
||||
for (let i = choices.length - 1; i > 0; i--) {
|
||||
const j = random.int(i + 1);
|
||||
[choices[i], choices[j]] = [choices[j], choices[i]];
|
||||
}
|
||||
}
|
||||
const kind = choices.pop()!;
|
||||
const writer = random.int(clients);
|
||||
const other = (writer + 1 + random.int(clients - 1)) % clients;
|
||||
const doc = random.pick(documents.filter((entry) => !entry.conflict));
|
||||
const directory = `round-${round}/${random.pick(["nested", "日本語", "café", "space folder"])}`;
|
||||
if (kind === 0) {
|
||||
all("disable-sync");
|
||||
const first = newDocument(`${directory}/same.md`);
|
||||
const second = { ...newDocument(first.path), conflict: true };
|
||||
steps.push(
|
||||
{
|
||||
type: "create",
|
||||
client: writer,
|
||||
path: first.path,
|
||||
content: first.content
|
||||
},
|
||||
{
|
||||
type: "create",
|
||||
client: other,
|
||||
path: second.path,
|
||||
content: second.content
|
||||
},
|
||||
{
|
||||
type: "drop-response",
|
||||
client: writer,
|
||||
kind: random.pick(["create", "manifest"] as const),
|
||||
point: random.pick(["before", "after"] as const)
|
||||
},
|
||||
{ type: "enable-sync", client: writer },
|
||||
{ type: "sync", client: writer },
|
||||
{ type: "wait-for-response-drop", client: writer }
|
||||
);
|
||||
all("enable-sync");
|
||||
documents.push(first, second);
|
||||
} else if (kind === 1) {
|
||||
// Both clients edit the same established version, at opposite ends.
|
||||
all("disable-sync");
|
||||
const prefix = `remote prefix ${round}\n\n`;
|
||||
const suffix = `\n\nlocal suffix ${round}\n`;
|
||||
steps.push(
|
||||
{
|
||||
type: "update",
|
||||
client: writer,
|
||||
path: doc.path,
|
||||
content: doc.content + suffix
|
||||
},
|
||||
{
|
||||
type: "update",
|
||||
client: other,
|
||||
path: doc.path,
|
||||
content: prefix + doc.content
|
||||
},
|
||||
{ type: "enable-sync", client: writer }
|
||||
);
|
||||
all("enable-sync");
|
||||
doc.content = prefix + doc.content + suffix;
|
||||
} else if (kind === 2) {
|
||||
all("disable-sync");
|
||||
const target = `${directory}/moved.md`;
|
||||
doc.content += `\nremote addition ${round}\n`;
|
||||
steps.push(
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: doc.path,
|
||||
newPath: target
|
||||
},
|
||||
{
|
||||
type: "update",
|
||||
client: other,
|
||||
path: doc.path,
|
||||
content: doc.content
|
||||
},
|
||||
{ type: "enable-sync", client: writer }
|
||||
);
|
||||
all("enable-sync");
|
||||
doc.path = target;
|
||||
} else if (kind === 3) {
|
||||
const second = random.pick(
|
||||
documents.filter((entry) => !entry.conflict && entry !== doc)
|
||||
);
|
||||
steps.push(
|
||||
{ type: "disable-sync", client: writer },
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: doc.path,
|
||||
newPath: `${directory}/temporary.md`
|
||||
},
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: second.path,
|
||||
newPath: doc.path
|
||||
},
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: `${directory}/temporary.md`,
|
||||
newPath: second.path
|
||||
},
|
||||
{ type: "enable-sync", client: writer }
|
||||
);
|
||||
[doc.path, second.path] = [second.path, doc.path];
|
||||
} else if (kind === 4) {
|
||||
// Reuse the same path but require a new UUID, even in later rounds.
|
||||
steps.push({ type: "delete", client: writer, path: doc.path });
|
||||
forget(doc);
|
||||
barrier();
|
||||
const replacement = newDocument(doc.path);
|
||||
steps.push({
|
||||
type: "create",
|
||||
client: other,
|
||||
path: replacement.path,
|
||||
content: replacement.content
|
||||
});
|
||||
documents.push(replacement);
|
||||
} else if (kind === 5) {
|
||||
const target = `${directory}/rapid.md`;
|
||||
steps.push(
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: doc.path,
|
||||
newPath: target
|
||||
},
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: target,
|
||||
newPath: doc.path
|
||||
},
|
||||
{
|
||||
type: "rename",
|
||||
client: writer,
|
||||
oldPath: doc.path,
|
||||
newPath: target
|
||||
}
|
||||
);
|
||||
doc.path = target;
|
||||
doc.content += `\nrapid edit ${round}\n`;
|
||||
steps.push({
|
||||
type: "update",
|
||||
client: writer,
|
||||
path: doc.path,
|
||||
content: doc.content
|
||||
});
|
||||
} else if (kind === 6) {
|
||||
steps.push({
|
||||
type: "drop-response",
|
||||
client: writer,
|
||||
kind: "content",
|
||||
point: random.pick(["before", "after"] as const)
|
||||
});
|
||||
doc.content += `\nretry edit ${round}\n`;
|
||||
steps.push(
|
||||
{
|
||||
type: "update",
|
||||
client: writer,
|
||||
path: doc.path,
|
||||
content: doc.content
|
||||
},
|
||||
{ type: "wait-for-response-drop", client: writer }
|
||||
);
|
||||
} else if (kind === 7) {
|
||||
// Keep an editor event delayed across reconnection and another edit.
|
||||
steps.push(
|
||||
{ type: "disable-sync", client: writer },
|
||||
{ type: "delay-notifications", client: writer }
|
||||
);
|
||||
doc.content += `\ndelayed local edit ${round}\n`;
|
||||
steps.push(
|
||||
{
|
||||
type: "update",
|
||||
client: writer,
|
||||
path: doc.path,
|
||||
content: doc.content
|
||||
},
|
||||
{ type: "enable-sync", client: writer }
|
||||
);
|
||||
barrier();
|
||||
doc.content += `\nlater remote edit ${round}\n`;
|
||||
steps.push({
|
||||
type: "update",
|
||||
client: other,
|
||||
path: doc.path,
|
||||
content: doc.content
|
||||
});
|
||||
barrier();
|
||||
steps.push({ type: "flush-notifications", client: writer });
|
||||
} else {
|
||||
// Explicit replacement retains the UUID. Only markers in the
|
||||
// overwritten bytes are exempted from preservation.
|
||||
for (const marker of markers)
|
||||
if (doc.content.includes(marker) && !removed.includes(marker))
|
||||
removed.push(marker);
|
||||
doc.content = newDocument(doc.path).content;
|
||||
steps.push({
|
||||
type: "update",
|
||||
client: writer,
|
||||
path: doc.path,
|
||||
content: doc.content
|
||||
});
|
||||
}
|
||||
barrier();
|
||||
steps.push({
|
||||
type: "assert-markers",
|
||||
markers: [...markers],
|
||||
removed: [...removed]
|
||||
});
|
||||
if (random.int(4) === 0) {
|
||||
steps.push({ type: "reset", client: random.int(clients) });
|
||||
barrier();
|
||||
}
|
||||
}
|
||||
return { version: 1, seed, iterations, clients, steps };
|
||||
}
|
||||
|
||||
export function traceTest(trace: Trace): TestDefinition {
|
||||
if (
|
||||
trace.version !== 1 ||
|
||||
!Array.isArray(trace.steps) ||
|
||||
!trace.steps.length ||
|
||||
(trace.clients !== undefined &&
|
||||
(!Number.isInteger(trace.clients) || trace.clients < 2))
|
||||
)
|
||||
throw new Error("Invalid/empty replay trace");
|
||||
return {
|
||||
clients: trace.clients ?? 2,
|
||||
description: `Seed ${trace.seed}; ${trace.iterations} generated conflict scenarios`,
|
||||
steps: trace.steps
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue