diff --git a/.forgejo/workflows/e2e.yml b/.forgejo/workflows/e2e.yml index eb8d1e54..ce30c205 100644 --- a/.forgejo/workflows/e2e.yml +++ b/.forgejo/workflows/e2e.yml @@ -15,6 +15,8 @@ concurrency: env: RUSTFLAGS: "-Dwarnings" + E2E_ARTIFACTS: logs/e2e + E2E_WORKERS: 8 jobs: build: @@ -30,6 +32,10 @@ jobs: with: node-version: "25.x" + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -44,28 +50,12 @@ jobs: sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 - name: E2E tests - run: | - cd sync-server - cargo run config-e2e.yml --color never & - SERVER_PID=$! - cd .. - - scripts/e2e.sh 8 - EXIT_CODE=$? - - kill $SERVER_PID 2>/dev/null || true - wait $SERVER_PID 2>/dev/null || true - - exit $EXIT_CODE + run: scripts/e2e.sh - name: Upload e2e logs if: always() uses: actions/upload-artifact@v4 with: name: e2e-logs - path: logs/ + path: ${{ env.E2E_ARTIFACTS }} retention-days: 30 - - - name: Cleanup - if: always() - run: scripts/clean-up.sh diff --git a/scripts/e2e-runner.mjs b/scripts/e2e-runner.mjs new file mode 100644 index 00000000..d38fe17b --- /dev/null +++ b/scripts/e2e-runner.mjs @@ -0,0 +1,266 @@ +import { spawn } from "node:child_process"; +import { + createWriteStream, + mkdirSync, + mkdtempSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function parseOptions(env = process.env) { + const workers = positive(env.E2E_WORKERS ?? "1", "E2E_WORKERS"); + + const seed = Number(env.E2E_SEED ?? "1"); + + if ( + !/^\d+$/.test(env.E2E_SEED ?? "1") || + !Number.isInteger(seed) || + seed < 0 || + seed + workers - 1 > 0xffffffff + ) { + throw new Error("E2E_SEED range must fit uint32"); + } + + return { + workers, + seed, + iterations: positive(env.E2E_ITERATIONS ?? "25", "E2E_ITERATIONS"), + timeout: + positive(env.E2E_TIMEOUT_SECONDS ?? "900", "E2E_TIMEOUT_SECONDS") * + 1000, + }; +} + +export async function runCommand( + command, + args, + { cwd, logPath, timeout, active = new Set(), env = process.env }, +) { + const log = createWriteStream(logPath, { flags: "w" }); + const child = spawn(command, args, { + cwd, + env: { ...env, NO_COLOR: "1", FORCE_COLOR: "0" }, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + }); + + let logError; + log.on("error", (error) => { + logError = error; + killOwned(child, "SIGTERM"); + }); + + active.add(child); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + let timedOut = false; + let hardKill; + + const timer = setTimeout(() => { + timedOut = true; + killOwned(child, "SIGTERM"); + hardKill = setTimeout(() => killOwned(child, "SIGKILL"), 5_000); + }, timeout); + + let spawnError; + child.once("error", (error) => { + spawnError = error; + }); + + try { + const result = await new Promise((resolveResult) => + child.once("close", (code, signal) => + resolveResult({ code, signal }), + ), + ); + + if (!logError) { + await new Promise((resolveLog) => { + log.once("error", resolveLog); + log.end(resolveLog); + }); + } + + return { + command, + args, + ...result, + timedOut, + error: spawnError?.message ?? logError?.message, + success: result.code === 0 && !timedOut && !spawnError && !logError, + }; + } finally { + clearTimeout(timer); + clearTimeout(hardKill); + active.delete(child); + } +} + +function killOwned(child, signal) { + if (!child.pid) { + return; + } + + try { + if (process.platform === "win32") { + child.kill(signal); + } else { + process.kill(-child.pid, signal); // the group created by this spawn + } + } catch (error) { + if (error.code !== "ESRCH") { + throw error; + } + } +} + +function positive(value, label) { + const result = Number(value); + if ( + !/^\d+$/.test(String(value)) || + !Number.isSafeInteger(result) || + result < 1 + ) + throw new Error(`${label} must be a positive integer`); + return result; +}; + +async function main() { + const options = parseOptions(); + const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const artifacts = process.env.E2E_ARTIFACTS + ? resolve(process.env.E2E_ARTIFACTS) + : mkdtempSync(join(tmpdir(), "vault-link-e2e-")); + + mkdirSync(artifacts, { recursive: true }); + console.log(`Artifacts: ${artifacts}`); + console.log( + `Seeds: ${options.seed}..${options.seed + options.workers - 1}; iterations: ${options.iterations}`, + ); + + const active = new Set(); + let interrupted = false; + const stop = () => { + interrupted = true; + for (const child of active) killOwned(child, "SIGTERM"); + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + process.on("exit", () => { + for (const child of active) killOwned(child, "SIGKILL"); + }); + + const results = []; + const run = async (name, command, args, cwd = root) => { + if (interrupted) { + throw new Error("Interrupted"); + } + + const logPath = join(artifacts, `${name}.log`); + const result = await runCommand(command, args, { + cwd, + logPath, + timeout: options.timeout, + active, + env: { ...process.env, E2E_ARTIFACTS: artifacts }, + }); + results.push({ name, ...result }); + writeFileSync( + join(artifacts, "results.json"), + JSON.stringify({ options, results }, null, 2), + ); + console.log(`${result.success ? "PASS" : "FAIL"} ${name}: ${logPath}`); + return result.success; + }; + const frontend = join(root, "frontend"); + + // Build only packages in scope and always rebuild the protocol peers. + if ( + !(await run( + "build-server", + "cargo", + ["build", "--release"], + join(root, "sync-server"), + )) + ) { + throw new Error("Server build failed"); + } + + if ( + !(await run( + "build-clients", + "npm", + [ + "run", + "build", + "--workspace", + "sync-client", + "--workspace", + "deterministic-tests", + "--workspace", + "test-client", + ], + frontend, + )) + ) { + throw new Error( + "Client build failed (run npm ci in frontend if dependencies are missing)", + ); + } + + await run( + "server-unit-tests", + "cargo", + ["test"], + join(root, "sync-server"), + ); + await run( + "client-unit-tests", + "npm", + ["run", "test", "--workspace", "sync-client"], + frontend, + ); + await run("harness-self-tests", "npm", ["run", "test:harness"], frontend); + await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend); + + await run("deterministic", process.execPath, [ + join(frontend, "deterministic-tests/dist/cli.js"), + "--concurrency", + String(options.workers), + ]); + + // Execute all suites even if one reports a regression; failure is never + // inferred from log text and successful workers cannot mask failed ones. + await Promise.all( + Array.from({ length: options.workers }, (_, i) => + run(`fuzz-${options.seed + i}`, process.execPath, [ + join(frontend, "test-client/dist/cli.js"), + "--seed", + String(options.seed + i), + "--iterations", + String(options.iterations), + "--artifacts", + artifacts, + ]), + ), + ); + if (interrupted || results.some((result) => !result.success)) { + throw new Error( + `Tests failed; inspect ${join(artifacts, "results.json")}`, + ); + } + + console.log("All harness suites passed"); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/e2e-runner.test.mjs b/scripts/e2e-runner.test.mjs new file mode 100644 index 00000000..ed739b55 --- /dev/null +++ b/scripts/e2e-runner.test.mjs @@ -0,0 +1,160 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseOptions, runCommand } from "./e2e-runner.mjs"; + +test("the clean-install lockfile includes every workspace and its declared dependencies", () => { + const frontend = new URL("../frontend/", import.meta.url); + const readJson = (path) => + JSON.parse(readFileSync(new URL(path, frontend), "utf8")); + const root = readJson("package.json"); + const lock = readJson("package-lock.json"); + assert.deepEqual(lock.packages[""].workspaces, root.workspaces); + for (const workspace of ["", ...root.workspaces]) { + const manifest = workspace + ? readJson(`${workspace}/package.json`) + : root; + const locked = lock.packages[workspace]; + assert(locked, `Missing workspace from lockfile: ${workspace}`); + for (const field of [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ]) + assert.deepEqual( + locked[field] ?? {}, + manifest[field] ?? {}, + `Stale ${workspace || "root"} ${field}; regenerate the lockfile before CI`, + ); + if (workspace) { + assert.equal(locked.version, manifest.version); + assert.deepEqual(lock.packages[`node_modules/${manifest.name}`], { + resolved: workspace, + link: true, + }); + } + } +}); + +test("CI installs dependencies before E2E and uploads the runner's artifacts on failure", () => { + const workflow = readFileSync( + new URL("../.forgejo/workflows/e2e.yml", import.meta.url), + "utf8", + ); + // Guard the workflow/runner contract without requiring a CI service or + // installing dependencies just to run the runner's own tests. + const steps = workflow.split(/^ - /m); + const install = steps.findIndex((step) => /run: npm ci\s*$/m.test(step)); + const e2e = steps.findIndex((step) => + /run: scripts\/e2e\.sh\s*$/m.test(step), + ); + assert(install > 0 && install < e2e, "npm ci must precede the E2E build"); + assert.match(steps[install], /working-directory: frontend\s*$/m); + assert.match(workflow, /^ E2E_ARTIFACTS: logs\/e2e\s*$/m); + assert.match(workflow, /^ E2E_WORKERS: 8\s*$/m); + const upload = steps.find((step) => + step.includes("uses: actions/upload-artifact@"), + ); + assert(upload, "CI must collect diagnostics"); + assert.match(upload, /if: always\(\)/); + assert.match(upload, /path: \$\{\{ env\.E2E_ARTIFACTS \}\}/); + assert.doesNotMatch( + workflow, + /cargo run|SERVER_PID|scripts\/clean-up\.sh/, + "The runner owns server lifecycle and isolated temporary databases", + ); +}); + +test("e2e rejects zero/fractional/garbage workers and seed overflow", () => { + for (const value of ["0", "1.5", "2x", "-1", "33"]) + assert.throws(() => parseOptions({ E2E_WORKERS: value })); + assert.throws(() => + parseOptions({ E2E_WORKERS: "2", E2E_SEED: "4294967295" }), + ); + assert.equal(parseOptions({ E2E_SEED: "0" }).seed, 0); +}); + +test("e2e observes exit codes, spawn errors, logs and bounded timeout", async () => { + const directory = mkdtempSync(join(tmpdir(), "vault-link-runner-test-")); + try { + const options = { + cwd: directory, + logPath: join(directory, "run.log"), + timeout: 5_000, + }; + const failed = await runCommand( + process.execPath, + ["-e", "console.log('last line'); process.exitCode = 7"], + options, + ); + assert.equal(failed.code, 7); + assert.equal(failed.success, false); + assert.equal(readFileSync(options.logPath, "utf8").trim(), "last line"); + const missing = await runCommand( + join(directory, "missing-command"), + [], + options, + ); + assert.equal(missing.success, false); + assert(missing.error); + const badLog = await runCommand( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { ...options, logPath: directory }, + ); + assert(!badLog.success); + assert(badLog.error); + const timedOut = await runCommand( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { ...options, timeout: 40 }, + ); + assert(timedOut.timedOut); + assert(!timedOut.success); + assert( + ( + await runCommand( + process.execPath, + ["-e", "process.exit(0)"], + options, + ) + ).success, + ); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +test("E2E includes both peers' unit suites and the permanent audit regressions", () => { + const runner = readFileSync( + new URL("./e2e-runner.mjs", import.meta.url), + "utf8", + ); + assert.match( + runner.replace(/\s+/g, ""), + /run\("server-unit-tests","cargo",\["test"\]/, + ); + assert.match( + runner.replace(/\s+/g, ""), + /run\("client-unit-tests","npm",\["run","test","--workspace","sync-client"\]/, + ); + const scripts = JSON.parse( + readFileSync( + new URL("../frontend/package.json", import.meta.url), + "utf8", + ), + ).scripts; + for (const file of ["engine-regressions", "sync-audit"]) + assert( + scripts["test:harness"].includes(`'test-support/${file}.test.ts'`), + `${file} must be an individually quoted test argument`, + ); + assert( + scripts["test:protocol"].includes( + "'test-support/protocol-audit.test.ts'", + ), + ); +}); diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 03d6a35f..d0bac3ae 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -1,145 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail -set -e -set -o pipefail - -NO_COLOR=1 -FORCE_COLOR=0 - -./scripts/utils/check-node.sh - -# Check if the argument is provided -if [ $# -eq 0 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Get the number of processes from the first argument -process_count=$1 - -mkdir -p logs - -# Build and restart the server -echo "Building server..." -( - cd sync-server - cargo build --release -) - -# Kill any existing server process -echo "Stopping existing server..." -pkill -f "sync_server" 2>/dev/null || true -sleep 1 - -# Clean databases (uses tmpfs via /dev/shm for zero disk I/O) -echo "Cleaning databases..." -rm -rf /host/tmp/vaultlink-e2e-databases - -# Start the server in the background -echo "Starting server..." -( - cd sync-server - ./target/release/sync_server config-e2e.yml & - echo $! > ../logs/server.pid -) -server_pid=$(cat logs/server.pid) -echo "Server started with PID: $server_pid" - -# Ensure server is killed on script exit -cleanup_server() { - if [ -n "$server_pid" ]; then - echo "Stopping server (PID: $server_pid)..." - kill $server_pid 2>/dev/null || true - wait $server_pid 2>/dev/null || true - server_pid="" - fi -} -trap cleanup_server EXIT - -cd .. - -( - cd frontend - npm ci - npm run build -) - -./scripts/utils/wait-for-server.sh - -pids=() -for i in $(seq 1 $process_count); do - node frontend/test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 & - pid=$! - pids+=($pid) - echo "Started process $i with PID: $pid (log: logs/log_${i}.log)" -done - -print_failed_log() { - for i in $(seq 1 $process_count); do - if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then - # Get the exit code of the process - wait ${pids[$i-1]} - exit_code=$? - - # Only consider non-zero exit codes as failures - if [ $exit_code -ne 0 ]; then - echo "----- Log for process ${pids[$i-1]} (log_${i}.log) -----" - cat "$(pwd)/logs/log_${i}.log" - echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log" - return 0 - else - echo "Process ${pids[$i-1]} completed successfully with exit code 0" - # Mark this PID as processed by setting it to empty - pids[$i-1]="" - fi - fi - done - return 1 -} - -E2E_TIMEOUT=${2:-3600} -start_time=$(date +%s) -echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)" - -# Monitor processes -while true; do - # Script-level timeout to prevent indefinite hangs - current_time=$(date +%s) - elapsed=$((current_time - start_time)) - if [ $elapsed -ge $E2E_TIMEOUT ]; then - echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes." - for pid in "${pids[@]}"; do - if [ -n "$pid" ]; then - kill $pid 2>/dev/null || true - fi - done - exit 1 - fi - - if print_failed_log; then - # Kill remaining processes - for pid in "${pids[@]}"; do - if [ -n "$pid" ]; then - kill $pid 2>/dev/null || true - fi - done - exit 1 - fi - - # Check if all processes have completed - all_done=true - for pid in "${pids[@]}"; do - if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then - all_done=false - break - fi - done - - if $all_done; then - cleanup_server - echo "All processes completed successfully" - exit 0 - fi - - sleep 0.2 -done +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec node "$script_dir/e2e-runner.mjs"