power loss
This commit is contained in:
parent
0f6b618c0b
commit
d56084e4bc
1 changed files with 18667 additions and 3902 deletions
|
|
@ -1,205 +1,90 @@
|
|||
import type { SyncSettings } from "sync-client";
|
||||
import { utils } from "sync-client";
|
||||
import { MockAgent } from "./agent/mock-agent";
|
||||
import { sleep } from "./utils/sleep";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { randomCasing } from "./utils/random-casing";
|
||||
import { parseArgs } from "node:util";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import * as os from "node:os";
|
||||
import { Logger } from "sync-client";
|
||||
import { ServerControl } from "../../deterministic-tests/src/server-control";
|
||||
import { ServerManager } from "../../deterministic-tests/src/server-manager";
|
||||
import { TestRunner } from "../../deterministic-tests/src/test-runner";
|
||||
import { generateTrace, traceTest, type Trace } from "./workload";
|
||||
|
||||
const TEST_ITERATIONS = 5;
|
||||
|
||||
// Simulate async file access by injecting waiting time before returning from file operations.
|
||||
let slowFileEvents = false;
|
||||
|
||||
// Whether to do resets in the test runs
|
||||
let doResets = false;
|
||||
|
||||
async function runTest({
|
||||
agentCount,
|
||||
concurrency,
|
||||
iterations,
|
||||
doDeletes,
|
||||
useResets,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
}: {
|
||||
agentCount: number;
|
||||
concurrency: number;
|
||||
iterations: number;
|
||||
doDeletes: boolean;
|
||||
useResets: boolean;
|
||||
useSlowFileEvents: boolean;
|
||||
jitterScaleInSeconds: number;
|
||||
}): Promise<void> {
|
||||
slowFileEvents = useSlowFileEvents;
|
||||
doResets = useResets;
|
||||
|
||||
const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`;
|
||||
console.info(`Running test ${settings}`);
|
||||
|
||||
const vaultName = uuidv4();
|
||||
console.info(`Using vault name: ${vaultName}`);
|
||||
const initialSettings: Partial<SyncSettings> = {
|
||||
isSyncEnabled: true,
|
||||
token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces
|
||||
vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter
|
||||
syncConcurrency: concurrency,
|
||||
remoteUri: "http://localhost:3000"
|
||||
};
|
||||
|
||||
const clients: MockAgent[] = [];
|
||||
for (let i = 0; i < agentCount; i++) {
|
||||
clients.push(
|
||||
new MockAgent(
|
||||
initialSettings,
|
||||
`agent-${i}`,
|
||||
doDeletes,
|
||||
useResets,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await utils.awaitAll(clients.map(async (client) => client.init()));
|
||||
|
||||
for (let i = 0; i < iterations; i++) {
|
||||
console.info(`Iteration ${i + 1}/${iterations}`);
|
||||
await utils.awaitAll(clients.map(async (client) => client.act()));
|
||||
await sleep(Math.random() * 200);
|
||||
}
|
||||
|
||||
console.info("Stopping agents");
|
||||
|
||||
// Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and
|
||||
for (const client of clients) {
|
||||
try {
|
||||
console.info(`Finishing up ${client.name}`);
|
||||
await client.finish();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// then we need a second pass to ensure that all agents pull the same state.
|
||||
for (const client of clients) {
|
||||
try {
|
||||
console.info(`Destroying ${client.name}`);
|
||||
await client.destroy();
|
||||
} catch (err) {
|
||||
if (!slowFileEvents) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.info("Agents finished successfully");
|
||||
|
||||
clients.slice(0, -1).forEach((client, i) => {
|
||||
console.info(
|
||||
`Checking consistency between ${client.name} and ${clients[i + 1].name}`
|
||||
);
|
||||
client.assertFileSystemsAreConsistent(clients[i]);
|
||||
console.info(`Consistency check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info("File systems found to be consistent");
|
||||
|
||||
clients.forEach((client) => {
|
||||
console.info(`Checking content for ${client.name}`);
|
||||
client.assertAllContentIsPresentOnce();
|
||||
console.info(`Content check for ${client.name} passed`);
|
||||
});
|
||||
|
||||
console.info(`Test passed ${settings}`);
|
||||
} catch (err) {
|
||||
console.error(`Test failed ${settings}`);
|
||||
throw err;
|
||||
}
|
||||
function integer(value: string, zero = false): number {
|
||||
const number = Number(value);
|
||||
if (
|
||||
!/^\d+$/.test(value) ||
|
||||
!Number.isSafeInteger(number) ||
|
||||
number < (zero ? 0 : 1)
|
||||
)
|
||||
throw new Error(`Invalid integer: ${value}`);
|
||||
return number;
|
||||
}
|
||||
|
||||
async function runTests(): Promise<void> {
|
||||
for (let i = 0; i < TEST_ITERATIONS; i++) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
concurrency: 16,
|
||||
iterations: 100,
|
||||
doDeletes: true,
|
||||
useResets: true,
|
||||
useSlowFileEvents: true,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
|
||||
for (const useSlowFileEvents of [true, false]) {
|
||||
for (const concurrency of [
|
||||
16,
|
||||
1 // test with concurrency 1 to check for deadlocks
|
||||
]) {
|
||||
for (const doDeletes of [false, true]) {
|
||||
await runTest({
|
||||
agentCount: 2,
|
||||
concurrency,
|
||||
iterations: 100,
|
||||
doDeletes,
|
||||
useResets: false,
|
||||
useSlowFileEvents,
|
||||
jitterScaleInSeconds: 0.75
|
||||
});
|
||||
}
|
||||
}
|
||||
async function main(): Promise<void> {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
seed: { type: "string", default: "1" },
|
||||
iterations: { type: "string", default: "25" },
|
||||
replay: { type: "string" },
|
||||
artifacts: { type: "string" },
|
||||
"server-binary": { type: "string" },
|
||||
config: { type: "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
process.on("uncaughtException", (error) => {
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
"WebSocket was closed before the connection was established"
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("Uncaught exception:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", (error, _promise) => {
|
||||
if (error instanceof Error && error.message === "Sync was reset") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
slowFileEvents &&
|
||||
error instanceof Error &&
|
||||
(error.message.includes("Document not found") ||
|
||||
error.message.includes("Document already exists at new location"))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
doResets &&
|
||||
error instanceof Error &&
|
||||
error.message.includes(
|
||||
"SyncClient has been destroyed and can no longer be used"
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.error("Unhandled rejection:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
runTests()
|
||||
.then(() => {
|
||||
process.exit(0);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
const seed = integer(values.seed!, true),
|
||||
iterations = integer(values.iterations!);
|
||||
const trace: Trace = values.replay
|
||||
? (JSON.parse(fs.readFileSync(values.replay, "utf8")) as Trace)
|
||||
: generateTrace(seed, iterations);
|
||||
const artifacts = values.artifacts
|
||||
? path.resolve(values.artifacts)
|
||||
: fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-fuzz-"));
|
||||
fs.mkdirSync(artifacts, { recursive: true });
|
||||
const tracePath = path.join(artifacts, `trace-${trace.seed}.json`);
|
||||
fs.writeFileSync(tracePath, JSON.stringify(trace, null, 2));
|
||||
console.log(`Replay: node ${__filename} --replay ${tracePath}`);
|
||||
const logger = new Logger();
|
||||
logger.onLogEmitted.add((line) => console.log(line.message));
|
||||
const root = path.resolve(__dirname, "../../..");
|
||||
const server = new ServerControl(
|
||||
values["server-binary"] ??
|
||||
path.join(root, "sync-server/target/release/sync_server"),
|
||||
values.config ?? path.join(root, "sync-server/config-e2e.yml"),
|
||||
logger
|
||||
);
|
||||
const manager = new ServerManager(logger);
|
||||
manager.track(server);
|
||||
manager.installSignalHandlers();
|
||||
try {
|
||||
await server.start();
|
||||
const result = await new TestRunner(
|
||||
server,
|
||||
logger,
|
||||
"test-token-change-me",
|
||||
server.remoteUri
|
||||
).runTest(`seed-${trace.seed}`, traceTest(trace));
|
||||
fs.writeFileSync(
|
||||
path.join(artifacts, `result-${trace.seed}.json`),
|
||||
JSON.stringify(result, null, 2)
|
||||
);
|
||||
if (!result.success) throw new Error(result.error);
|
||||
} finally {
|
||||
await server.stop();
|
||||
manager.untrack(server);
|
||||
}
|
||||
}
|
||||
|
||||
process.on("unhandledRejection", (error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
process.on("uncaughtException", (error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
main().then(
|
||||
() => process.exit(0),
|
||||
(error: unknown) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
}
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in a new issue