Move e2e harness into mjs
This commit is contained in:
parent
935387c769
commit
2e4e3535e4
3 changed files with 438 additions and 162 deletions
266
scripts/e2e-runner.mjs
Normal file
266
scripts/e2e-runner.mjs
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
import { spawn } from "node:child_process";
|
||||
import {
|
||||
createWriteStream,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export function parseOptions(env = process.env) {
|
||||
const workers = positive(env.E2E_WORKERS ?? "1", "E2E_WORKERS");
|
||||
|
||||
const seed = Number(env.E2E_SEED ?? "1");
|
||||
|
||||
if (
|
||||
!/^\d+$/.test(env.E2E_SEED ?? "1") ||
|
||||
!Number.isInteger(seed) ||
|
||||
seed < 0 ||
|
||||
seed + workers - 1 > 0xffffffff
|
||||
) {
|
||||
throw new Error("E2E_SEED range must fit uint32");
|
||||
}
|
||||
|
||||
return {
|
||||
workers,
|
||||
seed,
|
||||
iterations: positive(env.E2E_ITERATIONS ?? "25", "E2E_ITERATIONS"),
|
||||
timeout:
|
||||
positive(env.E2E_TIMEOUT_SECONDS ?? "900", "E2E_TIMEOUT_SECONDS") *
|
||||
1000,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runCommand(
|
||||
command,
|
||||
args,
|
||||
{ cwd, logPath, timeout, active = new Set(), env = process.env },
|
||||
) {
|
||||
const log = createWriteStream(logPath, { flags: "w" });
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
env: { ...env, NO_COLOR: "1", FORCE_COLOR: "0" },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
detached: process.platform !== "win32",
|
||||
});
|
||||
|
||||
let logError;
|
||||
log.on("error", (error) => {
|
||||
logError = error;
|
||||
killOwned(child, "SIGTERM");
|
||||
});
|
||||
|
||||
active.add(child);
|
||||
child.stdout.pipe(log, { end: false });
|
||||
child.stderr.pipe(log, { end: false });
|
||||
let timedOut = false;
|
||||
let hardKill;
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
killOwned(child, "SIGTERM");
|
||||
hardKill = setTimeout(() => killOwned(child, "SIGKILL"), 5_000);
|
||||
}, timeout);
|
||||
|
||||
let spawnError;
|
||||
child.once("error", (error) => {
|
||||
spawnError = error;
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await new Promise((resolveResult) =>
|
||||
child.once("close", (code, signal) =>
|
||||
resolveResult({ code, signal }),
|
||||
),
|
||||
);
|
||||
|
||||
if (!logError) {
|
||||
await new Promise((resolveLog) => {
|
||||
log.once("error", resolveLog);
|
||||
log.end(resolveLog);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
command,
|
||||
args,
|
||||
...result,
|
||||
timedOut,
|
||||
error: spawnError?.message ?? logError?.message,
|
||||
success: result.code === 0 && !timedOut && !spawnError && !logError,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
clearTimeout(hardKill);
|
||||
active.delete(child);
|
||||
}
|
||||
}
|
||||
|
||||
function killOwned(child, signal) {
|
||||
if (!child.pid) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.platform === "win32") {
|
||||
child.kill(signal);
|
||||
} else {
|
||||
process.kill(-child.pid, signal); // the group created by this spawn
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== "ESRCH") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function positive(value, label) {
|
||||
const result = Number(value);
|
||||
if (
|
||||
!/^\d+$/.test(String(value)) ||
|
||||
!Number.isSafeInteger(result) ||
|
||||
result < 1
|
||||
)
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
return result;
|
||||
};
|
||||
|
||||
async function main() {
|
||||
const options = parseOptions();
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const artifacts = process.env.E2E_ARTIFACTS
|
||||
? resolve(process.env.E2E_ARTIFACTS)
|
||||
: mkdtempSync(join(tmpdir(), "vault-link-e2e-"));
|
||||
|
||||
mkdirSync(artifacts, { recursive: true });
|
||||
console.log(`Artifacts: ${artifacts}`);
|
||||
console.log(
|
||||
`Seeds: ${options.seed}..${options.seed + options.workers - 1}; iterations: ${options.iterations}`,
|
||||
);
|
||||
|
||||
const active = new Set();
|
||||
let interrupted = false;
|
||||
const stop = () => {
|
||||
interrupted = true;
|
||||
for (const child of active) killOwned(child, "SIGTERM");
|
||||
};
|
||||
process.on("SIGINT", stop);
|
||||
process.on("SIGTERM", stop);
|
||||
process.on("exit", () => {
|
||||
for (const child of active) killOwned(child, "SIGKILL");
|
||||
});
|
||||
|
||||
const results = [];
|
||||
const run = async (name, command, args, cwd = root) => {
|
||||
if (interrupted) {
|
||||
throw new Error("Interrupted");
|
||||
}
|
||||
|
||||
const logPath = join(artifacts, `${name}.log`);
|
||||
const result = await runCommand(command, args, {
|
||||
cwd,
|
||||
logPath,
|
||||
timeout: options.timeout,
|
||||
active,
|
||||
env: { ...process.env, E2E_ARTIFACTS: artifacts },
|
||||
});
|
||||
results.push({ name, ...result });
|
||||
writeFileSync(
|
||||
join(artifacts, "results.json"),
|
||||
JSON.stringify({ options, results }, null, 2),
|
||||
);
|
||||
console.log(`${result.success ? "PASS" : "FAIL"} ${name}: ${logPath}`);
|
||||
return result.success;
|
||||
};
|
||||
const frontend = join(root, "frontend");
|
||||
|
||||
// Build only packages in scope and always rebuild the protocol peers.
|
||||
if (
|
||||
!(await run(
|
||||
"build-server",
|
||||
"cargo",
|
||||
["build", "--release"],
|
||||
join(root, "sync-server"),
|
||||
))
|
||||
) {
|
||||
throw new Error("Server build failed");
|
||||
}
|
||||
|
||||
if (
|
||||
!(await run(
|
||||
"build-clients",
|
||||
"npm",
|
||||
[
|
||||
"run",
|
||||
"build",
|
||||
"--workspace",
|
||||
"sync-client",
|
||||
"--workspace",
|
||||
"deterministic-tests",
|
||||
"--workspace",
|
||||
"test-client",
|
||||
],
|
||||
frontend,
|
||||
))
|
||||
) {
|
||||
throw new Error(
|
||||
"Client build failed (run npm ci in frontend if dependencies are missing)",
|
||||
);
|
||||
}
|
||||
|
||||
await run(
|
||||
"server-unit-tests",
|
||||
"cargo",
|
||||
["test"],
|
||||
join(root, "sync-server"),
|
||||
);
|
||||
await run(
|
||||
"client-unit-tests",
|
||||
"npm",
|
||||
["run", "test", "--workspace", "sync-client"],
|
||||
frontend,
|
||||
);
|
||||
await run("harness-self-tests", "npm", ["run", "test:harness"], frontend);
|
||||
await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend);
|
||||
|
||||
await run("deterministic", process.execPath, [
|
||||
join(frontend, "deterministic-tests/dist/cli.js"),
|
||||
"--concurrency",
|
||||
String(options.workers),
|
||||
]);
|
||||
|
||||
// Execute all suites even if one reports a regression; failure is never
|
||||
// inferred from log text and successful workers cannot mask failed ones.
|
||||
await Promise.all(
|
||||
Array.from({ length: options.workers }, (_, i) =>
|
||||
run(`fuzz-${options.seed + i}`, process.execPath, [
|
||||
join(frontend, "test-client/dist/cli.js"),
|
||||
"--seed",
|
||||
String(options.seed + i),
|
||||
"--iterations",
|
||||
String(options.iterations),
|
||||
"--artifacts",
|
||||
artifacts,
|
||||
]),
|
||||
),
|
||||
);
|
||||
if (interrupted || results.some((result) => !result.success)) {
|
||||
throw new Error(
|
||||
`Tests failed; inspect ${join(artifacts, "results.json")}`,
|
||||
);
|
||||
}
|
||||
|
||||
console.log("All harness suites passed");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
main().catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
160
scripts/e2e-runner.test.mjs
Normal file
160
scripts/e2e-runner.test.mjs
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { parseOptions, runCommand } from "./e2e-runner.mjs";
|
||||
|
||||
test("the clean-install lockfile includes every workspace and its declared dependencies", () => {
|
||||
const frontend = new URL("../frontend/", import.meta.url);
|
||||
const readJson = (path) =>
|
||||
JSON.parse(readFileSync(new URL(path, frontend), "utf8"));
|
||||
const root = readJson("package.json");
|
||||
const lock = readJson("package-lock.json");
|
||||
assert.deepEqual(lock.packages[""].workspaces, root.workspaces);
|
||||
for (const workspace of ["", ...root.workspaces]) {
|
||||
const manifest = workspace
|
||||
? readJson(`${workspace}/package.json`)
|
||||
: root;
|
||||
const locked = lock.packages[workspace];
|
||||
assert(locked, `Missing workspace from lockfile: ${workspace}`);
|
||||
for (const field of [
|
||||
"dependencies",
|
||||
"devDependencies",
|
||||
"optionalDependencies",
|
||||
"peerDependencies",
|
||||
])
|
||||
assert.deepEqual(
|
||||
locked[field] ?? {},
|
||||
manifest[field] ?? {},
|
||||
`Stale ${workspace || "root"} ${field}; regenerate the lockfile before CI`,
|
||||
);
|
||||
if (workspace) {
|
||||
assert.equal(locked.version, manifest.version);
|
||||
assert.deepEqual(lock.packages[`node_modules/${manifest.name}`], {
|
||||
resolved: workspace,
|
||||
link: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("CI installs dependencies before E2E and uploads the runner's artifacts on failure", () => {
|
||||
const workflow = readFileSync(
|
||||
new URL("../.forgejo/workflows/e2e.yml", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
// Guard the workflow/runner contract without requiring a CI service or
|
||||
// installing dependencies just to run the runner's own tests.
|
||||
const steps = workflow.split(/^ - /m);
|
||||
const install = steps.findIndex((step) => /run: npm ci\s*$/m.test(step));
|
||||
const e2e = steps.findIndex((step) =>
|
||||
/run: scripts\/e2e\.sh\s*$/m.test(step),
|
||||
);
|
||||
assert(install > 0 && install < e2e, "npm ci must precede the E2E build");
|
||||
assert.match(steps[install], /working-directory: frontend\s*$/m);
|
||||
assert.match(workflow, /^ E2E_ARTIFACTS: logs\/e2e\s*$/m);
|
||||
assert.match(workflow, /^ E2E_WORKERS: 8\s*$/m);
|
||||
const upload = steps.find((step) =>
|
||||
step.includes("uses: actions/upload-artifact@"),
|
||||
);
|
||||
assert(upload, "CI must collect diagnostics");
|
||||
assert.match(upload, /if: always\(\)/);
|
||||
assert.match(upload, /path: \$\{\{ env\.E2E_ARTIFACTS \}\}/);
|
||||
assert.doesNotMatch(
|
||||
workflow,
|
||||
/cargo run|SERVER_PID|scripts\/clean-up\.sh/,
|
||||
"The runner owns server lifecycle and isolated temporary databases",
|
||||
);
|
||||
});
|
||||
|
||||
test("e2e rejects zero/fractional/garbage workers and seed overflow", () => {
|
||||
for (const value of ["0", "1.5", "2x", "-1", "33"])
|
||||
assert.throws(() => parseOptions({ E2E_WORKERS: value }));
|
||||
assert.throws(() =>
|
||||
parseOptions({ E2E_WORKERS: "2", E2E_SEED: "4294967295" }),
|
||||
);
|
||||
assert.equal(parseOptions({ E2E_SEED: "0" }).seed, 0);
|
||||
});
|
||||
|
||||
test("e2e observes exit codes, spawn errors, logs and bounded timeout", async () => {
|
||||
const directory = mkdtempSync(join(tmpdir(), "vault-link-runner-test-"));
|
||||
try {
|
||||
const options = {
|
||||
cwd: directory,
|
||||
logPath: join(directory, "run.log"),
|
||||
timeout: 5_000,
|
||||
};
|
||||
const failed = await runCommand(
|
||||
process.execPath,
|
||||
["-e", "console.log('last line'); process.exitCode = 7"],
|
||||
options,
|
||||
);
|
||||
assert.equal(failed.code, 7);
|
||||
assert.equal(failed.success, false);
|
||||
assert.equal(readFileSync(options.logPath, "utf8").trim(), "last line");
|
||||
const missing = await runCommand(
|
||||
join(directory, "missing-command"),
|
||||
[],
|
||||
options,
|
||||
);
|
||||
assert.equal(missing.success, false);
|
||||
assert(missing.error);
|
||||
const badLog = await runCommand(
|
||||
process.execPath,
|
||||
["-e", "setInterval(() => {}, 1000)"],
|
||||
{ ...options, logPath: directory },
|
||||
);
|
||||
assert(!badLog.success);
|
||||
assert(badLog.error);
|
||||
const timedOut = await runCommand(
|
||||
process.execPath,
|
||||
["-e", "setInterval(() => {}, 1000)"],
|
||||
{ ...options, timeout: 40 },
|
||||
);
|
||||
assert(timedOut.timedOut);
|
||||
assert(!timedOut.success);
|
||||
assert(
|
||||
(
|
||||
await runCommand(
|
||||
process.execPath,
|
||||
["-e", "process.exit(0)"],
|
||||
options,
|
||||
)
|
||||
).success,
|
||||
);
|
||||
} finally {
|
||||
rmSync(directory, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("E2E includes both peers' unit suites and the permanent audit regressions", () => {
|
||||
const runner = readFileSync(
|
||||
new URL("./e2e-runner.mjs", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(
|
||||
runner.replace(/\s+/g, ""),
|
||||
/run\("server-unit-tests","cargo",\["test"\]/,
|
||||
);
|
||||
assert.match(
|
||||
runner.replace(/\s+/g, ""),
|
||||
/run\("client-unit-tests","npm",\["run","test","--workspace","sync-client"\]/,
|
||||
);
|
||||
const scripts = JSON.parse(
|
||||
readFileSync(
|
||||
new URL("../frontend/package.json", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
).scripts;
|
||||
for (const file of ["engine-regressions", "sync-audit"])
|
||||
assert(
|
||||
scripts["test:harness"].includes(`'test-support/${file}.test.ts'`),
|
||||
`${file} must be an individually quoted test argument`,
|
||||
);
|
||||
assert(
|
||||
scripts["test:protocol"].includes(
|
||||
"'test-support/protocol-audit.test.ts'",
|
||||
),
|
||||
);
|
||||
});
|
||||
148
scripts/e2e.sh
148
scripts/e2e.sh
|
|
@ -1,145 +1,5 @@
|
|||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
NO_COLOR=1
|
||||
FORCE_COLOR=0
|
||||
|
||||
./scripts/utils/check-node.sh
|
||||
|
||||
# Check if the argument is provided
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <number_of_processes>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get the number of processes from the first argument
|
||||
process_count=$1
|
||||
|
||||
mkdir -p logs
|
||||
|
||||
# Build and restart the server
|
||||
echo "Building server..."
|
||||
(
|
||||
cd sync-server
|
||||
cargo build --release
|
||||
)
|
||||
|
||||
# Kill any existing server process
|
||||
echo "Stopping existing server..."
|
||||
pkill -f "sync_server" 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Clean databases (uses tmpfs via /dev/shm for zero disk I/O)
|
||||
echo "Cleaning databases..."
|
||||
rm -rf /host/tmp/vaultlink-e2e-databases
|
||||
|
||||
# Start the server in the background
|
||||
echo "Starting server..."
|
||||
(
|
||||
cd sync-server
|
||||
./target/release/sync_server config-e2e.yml &
|
||||
echo $! > ../logs/server.pid
|
||||
)
|
||||
server_pid=$(cat logs/server.pid)
|
||||
echo "Server started with PID: $server_pid"
|
||||
|
||||
# Ensure server is killed on script exit
|
||||
cleanup_server() {
|
||||
if [ -n "$server_pid" ]; then
|
||||
echo "Stopping server (PID: $server_pid)..."
|
||||
kill $server_pid 2>/dev/null || true
|
||||
wait $server_pid 2>/dev/null || true
|
||||
server_pid=""
|
||||
fi
|
||||
}
|
||||
trap cleanup_server EXIT
|
||||
|
||||
cd ..
|
||||
|
||||
(
|
||||
cd frontend
|
||||
npm ci
|
||||
npm run build
|
||||
)
|
||||
|
||||
./scripts/utils/wait-for-server.sh
|
||||
|
||||
pids=()
|
||||
for i in $(seq 1 $process_count); do
|
||||
node frontend/test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 &
|
||||
pid=$!
|
||||
pids+=($pid)
|
||||
echo "Started process $i with PID: $pid (log: logs/log_${i}.log)"
|
||||
done
|
||||
|
||||
print_failed_log() {
|
||||
for i in $(seq 1 $process_count); do
|
||||
if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then
|
||||
# Get the exit code of the process
|
||||
wait ${pids[$i-1]}
|
||||
exit_code=$?
|
||||
|
||||
# Only consider non-zero exit codes as failures
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "----- Log for process ${pids[$i-1]} (log_${i}.log) -----"
|
||||
cat "$(pwd)/logs/log_${i}.log"
|
||||
echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log"
|
||||
return 0
|
||||
else
|
||||
echo "Process ${pids[$i-1]} completed successfully with exit code 0"
|
||||
# Mark this PID as processed by setting it to empty
|
||||
pids[$i-1]=""
|
||||
fi
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
E2E_TIMEOUT=${2:-3600}
|
||||
start_time=$(date +%s)
|
||||
echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)"
|
||||
|
||||
# Monitor processes
|
||||
while true; do
|
||||
# Script-level timeout to prevent indefinite hangs
|
||||
current_time=$(date +%s)
|
||||
elapsed=$((current_time - start_time))
|
||||
if [ $elapsed -ge $E2E_TIMEOUT ]; then
|
||||
echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes."
|
||||
for pid in "${pids[@]}"; do
|
||||
if [ -n "$pid" ]; then
|
||||
kill $pid 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if print_failed_log; then
|
||||
# Kill remaining processes
|
||||
for pid in "${pids[@]}"; do
|
||||
if [ -n "$pid" ]; then
|
||||
kill $pid 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if all processes have completed
|
||||
all_done=true
|
||||
for pid in "${pids[@]}"; do
|
||||
if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then
|
||||
all_done=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if $all_done; then
|
||||
cleanup_server
|
||||
echo "All processes completed successfully"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
sleep 0.2
|
||||
done
|
||||
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec node "$script_dir/e2e-runner.mjs"
|
||||
|
|
|
|||
Loading…
Reference in a new issue