Move e2e harness into mjs

.forgejo/workflows/e2e.yml

@@ -15,6 +15,8 @@ concurrency:
 
 env:
   RUSTFLAGS: "-Dwarnings"
+  E2E_ARTIFACTS: logs/e2e
+  E2E_WORKERS: 8
 
 jobs:
   build:
@@ -30,6 +32,10 @@ jobs:
         with:
           node-version: "25.x"
 
+      - name: Install frontend dependencies
+        working-directory: frontend
+        run: npm ci
+
       - name: Setup Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -44,28 +50,12 @@ jobs:
           sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
 
       - name: E2E tests
-        run: |
+        run: scripts/e2e.sh
-          cd sync-server
-          cargo run config-e2e.yml --color never &
-          SERVER_PID=$!
-          cd ..
-
-          scripts/e2e.sh 8
-          EXIT_CODE=$?
-
-          kill $SERVER_PID 2>/dev/null || true
-          wait $SERVER_PID 2>/dev/null || true
-
-          exit $EXIT_CODE
 
       - name: Upload e2e logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: e2e-logs
-          path: logs/
+          path: ${{ env.E2E_ARTIFACTS }}
           retention-days: 30
-
-      - name: Cleanup
-        if: always()
-        run: scripts/clean-up.sh

scripts/e2e-runner.mjs

@@ -0,0 +1,266 @@
+import { spawn } from "node:child_process";
+import {
+    createWriteStream,
+    mkdirSync,
+    mkdtempSync,
+    writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export function parseOptions(env = process.env) {
+    const workers = positive(env.E2E_WORKERS ?? "1", "E2E_WORKERS");
+
+    const seed = Number(env.E2E_SEED ?? "1");
+
+    if (
+        !/^\d+$/.test(env.E2E_SEED ?? "1") ||
+        !Number.isInteger(seed) ||
+        seed < 0 ||
+        seed + workers - 1 > 0xffffffff
+    ) {
+        throw new Error("E2E_SEED range must fit uint32");
+    }
+
+    return {
+        workers,
+        seed,
+        iterations: positive(env.E2E_ITERATIONS ?? "25", "E2E_ITERATIONS"),
+        timeout:
+            positive(env.E2E_TIMEOUT_SECONDS ?? "900", "E2E_TIMEOUT_SECONDS") *
+            1000,
+    };
+}
+
+export async function runCommand(
+    command,
+    args,
+    { cwd, logPath, timeout, active = new Set(), env = process.env },
+) {
+    const log = createWriteStream(logPath, { flags: "w" });
+    const child = spawn(command, args, {
+        cwd,
+        env: { ...env, NO_COLOR: "1", FORCE_COLOR: "0" },
+        stdio: ["ignore", "pipe", "pipe"],
+        detached: process.platform !== "win32",
+    });
+
+    let logError;
+    log.on("error", (error) => {
+        logError = error;
+        killOwned(child, "SIGTERM");
+    });
+
+    active.add(child);
+    child.stdout.pipe(log, { end: false });
+    child.stderr.pipe(log, { end: false });
+    let timedOut = false;
+    let hardKill;
+
+    const timer = setTimeout(() => {
+        timedOut = true;
+        killOwned(child, "SIGTERM");
+        hardKill = setTimeout(() => killOwned(child, "SIGKILL"), 5_000);
+    }, timeout);
+
+    let spawnError;
+    child.once("error", (error) => {
+        spawnError = error;
+    });
+
+    try {
+        const result = await new Promise((resolveResult) =>
+            child.once("close", (code, signal) =>
+                resolveResult({ code, signal }),
+            ),
+        );
+
+        if (!logError) {
+            await new Promise((resolveLog) => {
+                log.once("error", resolveLog);
+                log.end(resolveLog);
+            });
+        }
+
+        return {
+            command,
+            args,
+            ...result,
+            timedOut,
+            error: spawnError?.message ?? logError?.message,
+            success: result.code === 0 && !timedOut && !spawnError && !logError,
+        };
+    } finally {
+        clearTimeout(timer);
+        clearTimeout(hardKill);
+        active.delete(child);
+    }
+}
+
+function killOwned(child, signal) {
+    if (!child.pid) {
+        return;
+    }
+
+    try {
+        if (process.platform === "win32") {
+            child.kill(signal);
+        } else {
+            process.kill(-child.pid, signal); // the group created by this spawn
+        }
+    } catch (error) {
+        if (error.code !== "ESRCH") {
+            throw error;
+        }
+    }
+}
+
+function positive(value, label) {
+    const result = Number(value);
+    if (
+        !/^\d+$/.test(String(value)) ||
+        !Number.isSafeInteger(result) ||
+        result < 1
+    )
+        throw new Error(`${label} must be a positive integer`);
+    return result;
+};
+
+async function main() {
+    const options = parseOptions();
+    const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+    const artifacts = process.env.E2E_ARTIFACTS
+        ? resolve(process.env.E2E_ARTIFACTS)
+        : mkdtempSync(join(tmpdir(), "vault-link-e2e-"));
+
+    mkdirSync(artifacts, { recursive: true });
+    console.log(`Artifacts: ${artifacts}`);
+    console.log(
+        `Seeds: ${options.seed}..${options.seed + options.workers - 1}; iterations: ${options.iterations}`,
+    );
+
+    const active = new Set();
+    let interrupted = false;
+    const stop = () => {
+        interrupted = true;
+        for (const child of active) killOwned(child, "SIGTERM");
+    };
+    process.on("SIGINT", stop);
+    process.on("SIGTERM", stop);
+    process.on("exit", () => {
+        for (const child of active) killOwned(child, "SIGKILL");
+    });
+
+    const results = [];
+    const run = async (name, command, args, cwd = root) => {
+        if (interrupted) {
+            throw new Error("Interrupted");
+        }
+
+        const logPath = join(artifacts, `${name}.log`);
+        const result = await runCommand(command, args, {
+            cwd,
+            logPath,
+            timeout: options.timeout,
+            active,
+            env: { ...process.env, E2E_ARTIFACTS: artifacts },
+        });
+        results.push({ name, ...result });
+        writeFileSync(
+            join(artifacts, "results.json"),
+            JSON.stringify({ options, results }, null, 2),
+        );
+        console.log(`${result.success ? "PASS" : "FAIL"} ${name}: ${logPath}`);
+        return result.success;
+    };
+    const frontend = join(root, "frontend");
+
+    // Build only packages in scope and always rebuild the protocol peers.
+    if (
+        !(await run(
+            "build-server",
+            "cargo",
+            ["build", "--release"],
+            join(root, "sync-server"),
+        ))
+    ) {
+        throw new Error("Server build failed");
+    }
+
+    if (
+        !(await run(
+            "build-clients",
+            "npm",
+            [
+                "run",
+                "build",
+                "--workspace",
+                "sync-client",
+                "--workspace",
+                "deterministic-tests",
+                "--workspace",
+                "test-client",
+            ],
+            frontend,
+        ))
+    ) {
+        throw new Error(
+            "Client build failed (run npm ci in frontend if dependencies are missing)",
+        );
+    }
+
+    await run(
+        "server-unit-tests",
+        "cargo",
+        ["test"],
+        join(root, "sync-server"),
+    );
+    await run(
+        "client-unit-tests",
+        "npm",
+        ["run", "test", "--workspace", "sync-client"],
+        frontend,
+    );
+    await run("harness-self-tests", "npm", ["run", "test:harness"], frontend);
+    await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend);
+
+    await run("deterministic", process.execPath, [
+        join(frontend, "deterministic-tests/dist/cli.js"),
+        "--concurrency",
+        String(options.workers),
+    ]);
+
+    // Execute all suites even if one reports a regression; failure is never
+    // inferred from log text and successful workers cannot mask failed ones.
+    await Promise.all(
+        Array.from({ length: options.workers }, (_, i) =>
+            run(`fuzz-${options.seed + i}`, process.execPath, [
+                join(frontend, "test-client/dist/cli.js"),
+                "--seed",
+                String(options.seed + i),
+                "--iterations",
+                String(options.iterations),
+                "--artifacts",
+                artifacts,
+            ]),
+        ),
+    );
+    if (interrupted || results.some((result) => !result.success)) {
+        throw new Error(
+            `Tests failed; inspect ${join(artifacts, "results.json")}`,
+        );
+    }
+
+    console.log("All harness suites passed");
+}
+
+if (
+    process.argv[1] &&
+    resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+    main().catch((error) => {
+        console.error(error.message);
+        process.exitCode = 1;
+    });
+}

scripts/e2e-runner.test.mjs

@@ -0,0 +1,160 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { parseOptions, runCommand } from "./e2e-runner.mjs";
+
+test("the clean-install lockfile includes every workspace and its declared dependencies", () => {
+    const frontend = new URL("../frontend/", import.meta.url);
+    const readJson = (path) =>
+        JSON.parse(readFileSync(new URL(path, frontend), "utf8"));
+    const root = readJson("package.json");
+    const lock = readJson("package-lock.json");
+    assert.deepEqual(lock.packages[""].workspaces, root.workspaces);
+    for (const workspace of ["", ...root.workspaces]) {
+        const manifest = workspace
+            ? readJson(`${workspace}/package.json`)
+            : root;
+        const locked = lock.packages[workspace];
+        assert(locked, `Missing workspace from lockfile: ${workspace}`);
+        for (const field of [
+            "dependencies",
+            "devDependencies",
+            "optionalDependencies",
+            "peerDependencies",
+        ])
+            assert.deepEqual(
+                locked[field] ?? {},
+                manifest[field] ?? {},
+                `Stale ${workspace || "root"} ${field}; regenerate the lockfile before CI`,
+            );
+        if (workspace) {
+            assert.equal(locked.version, manifest.version);
+            assert.deepEqual(lock.packages[`node_modules/${manifest.name}`], {
+                resolved: workspace,
+                link: true,
+            });
+        }
+    }
+});
+
+test("CI installs dependencies before E2E and uploads the runner's artifacts on failure", () => {
+    const workflow = readFileSync(
+        new URL("../.forgejo/workflows/e2e.yml", import.meta.url),
+        "utf8",
+    );
+    // Guard the workflow/runner contract without requiring a CI service or
+    // installing dependencies just to run the runner's own tests.
+    const steps = workflow.split(/^      - /m);
+    const install = steps.findIndex((step) => /run: npm ci\s*$/m.test(step));
+    const e2e = steps.findIndex((step) =>
+        /run: scripts\/e2e\.sh\s*$/m.test(step),
+    );
+    assert(install > 0 && install < e2e, "npm ci must precede the E2E build");
+    assert.match(steps[install], /working-directory: frontend\s*$/m);
+    assert.match(workflow, /^  E2E_ARTIFACTS: logs\/e2e\s*$/m);
+    assert.match(workflow, /^  E2E_WORKERS: 8\s*$/m);
+    const upload = steps.find((step) =>
+        step.includes("uses: actions/upload-artifact@"),
+    );
+    assert(upload, "CI must collect diagnostics");
+    assert.match(upload, /if: always\(\)/);
+    assert.match(upload, /path: \$\{\{ env\.E2E_ARTIFACTS \}\}/);
+    assert.doesNotMatch(
+        workflow,
+        /cargo run|SERVER_PID|scripts\/clean-up\.sh/,
+        "The runner owns server lifecycle and isolated temporary databases",
+    );
+});
+
+test("e2e rejects zero/fractional/garbage workers and seed overflow", () => {
+    for (const value of ["0", "1.5", "2x", "-1", "33"])
+        assert.throws(() => parseOptions({ E2E_WORKERS: value }));
+    assert.throws(() =>
+        parseOptions({ E2E_WORKERS: "2", E2E_SEED: "4294967295" }),
+    );
+    assert.equal(parseOptions({ E2E_SEED: "0" }).seed, 0);
+});
+
+test("e2e observes exit codes, spawn errors, logs and bounded timeout", async () => {
+    const directory = mkdtempSync(join(tmpdir(), "vault-link-runner-test-"));
+    try {
+        const options = {
+            cwd: directory,
+            logPath: join(directory, "run.log"),
+            timeout: 5_000,
+        };
+        const failed = await runCommand(
+            process.execPath,
+            ["-e", "console.log('last line'); process.exitCode = 7"],
+            options,
+        );
+        assert.equal(failed.code, 7);
+        assert.equal(failed.success, false);
+        assert.equal(readFileSync(options.logPath, "utf8").trim(), "last line");
+        const missing = await runCommand(
+            join(directory, "missing-command"),
+            [],
+            options,
+        );
+        assert.equal(missing.success, false);
+        assert(missing.error);
+        const badLog = await runCommand(
+            process.execPath,
+            ["-e", "setInterval(() => {}, 1000)"],
+            { ...options, logPath: directory },
+        );
+        assert(!badLog.success);
+        assert(badLog.error);
+        const timedOut = await runCommand(
+            process.execPath,
+            ["-e", "setInterval(() => {}, 1000)"],
+            { ...options, timeout: 40 },
+        );
+        assert(timedOut.timedOut);
+        assert(!timedOut.success);
+        assert(
+            (
+                await runCommand(
+                    process.execPath,
+                    ["-e", "process.exit(0)"],
+                    options,
+                )
+            ).success,
+        );
+    } finally {
+        rmSync(directory, { recursive: true });
+    }
+});
+
+test("E2E includes both peers' unit suites and the permanent audit regressions", () => {
+    const runner = readFileSync(
+        new URL("./e2e-runner.mjs", import.meta.url),
+        "utf8",
+    );
+    assert.match(
+        runner.replace(/\s+/g, ""),
+        /run\("server-unit-tests","cargo",\["test"\]/,
+    );
+    assert.match(
+        runner.replace(/\s+/g, ""),
+        /run\("client-unit-tests","npm",\["run","test","--workspace","sync-client"\]/,
+    );
+    const scripts = JSON.parse(
+        readFileSync(
+            new URL("../frontend/package.json", import.meta.url),
+            "utf8",
+        ),
+    ).scripts;
+    for (const file of ["engine-regressions", "sync-audit"])
+        assert(
+            scripts["test:harness"].includes(`'test-support/${file}.test.ts'`),
+            `${file} must be an individually quoted test argument`,
+        );
+    assert(
+        scripts["test:protocol"].includes(
+            "'test-support/protocol-audit.test.ts'",
+        ),
+    );
+});

scripts/e2e.sh

@@ -1,145 +1,5 @@
-#!/bin/bash
+#!/usr/bin/env bash
+set -euo pipefail
 
-set -e
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-set -o pipefail
+exec node "$script_dir/e2e-runner.mjs"
-
-NO_COLOR=1
-FORCE_COLOR=0
-
-./scripts/utils/check-node.sh
-
-# Check if the argument is provided
-if [ $# -eq 0 ]; then
-    echo "Usage: $0 <number_of_processes>"
-    exit 1
-fi
-
-# Get the number of processes from the first argument
-process_count=$1
-
-mkdir -p logs
-
-# Build and restart the server
-echo "Building server..."
-(
-    cd sync-server
-    cargo build --release
-)
-
-# Kill any existing server process
-echo "Stopping existing server..."
-pkill -f "sync_server" 2>/dev/null || true
-sleep 1
-
-# Clean databases (uses tmpfs via /dev/shm for zero disk I/O)
-echo "Cleaning databases..."
-rm -rf /host/tmp/vaultlink-e2e-databases
-
-# Start the server in the background
-echo "Starting server..."
-(
-    cd sync-server
-    ./target/release/sync_server config-e2e.yml &
-    echo $! > ../logs/server.pid
-)
-server_pid=$(cat logs/server.pid)
-echo "Server started with PID: $server_pid"
-
-# Ensure server is killed on script exit
-cleanup_server() {
-    if [ -n "$server_pid" ]; then
-        echo "Stopping server (PID: $server_pid)..."
-        kill $server_pid 2>/dev/null || true
-        wait $server_pid 2>/dev/null || true
-        server_pid=""
-    fi
-}
-trap cleanup_server EXIT
-
-cd ..
-
-(
-    cd frontend
-    npm ci
-    npm run build
-)
-
-./scripts/utils/wait-for-server.sh
-
-pids=()
-for i in $(seq 1 $process_count); do
-    node frontend/test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 &
-    pid=$!
-    pids+=($pid)
-    echo "Started process $i with PID: $pid (log: logs/log_${i}.log)"
-done
-
-print_failed_log() {
-    for i in $(seq 1 $process_count); do
-        if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then
-            # Get the exit code of the process
-            wait ${pids[$i-1]}
-            exit_code=$?
-
-            # Only consider non-zero exit codes as failures
-            if [ $exit_code -ne 0 ]; then
-                echo "----- Log for process ${pids[$i-1]} (log_${i}.log) -----"
-                cat "$(pwd)/logs/log_${i}.log"
-                echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log"
-                return 0
-            else
-                echo "Process ${pids[$i-1]} completed successfully with exit code 0"
-                # Mark this PID as processed by setting it to empty
-                pids[$i-1]=""
-            fi
-        fi
-    done
-    return 1
-}
-
-E2E_TIMEOUT=${2:-3600}
-start_time=$(date +%s)
-echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)"
-
-# Monitor processes
-while true; do
-    # Script-level timeout to prevent indefinite hangs
-    current_time=$(date +%s)
-    elapsed=$((current_time - start_time))
-    if [ $elapsed -ge $E2E_TIMEOUT ]; then
-        echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes."
-        for pid in "${pids[@]}"; do
-            if [ -n "$pid" ]; then
-                kill $pid 2>/dev/null || true
-            fi
-        done
-        exit 1
-    fi
-
-    if print_failed_log; then
-        # Kill remaining processes
-        for pid in "${pids[@]}"; do
-            if [ -n "$pid" ]; then
-                kill $pid 2>/dev/null || true
-            fi
-        done
-        exit 1
-    fi
-
-    # Check if all processes have completed
-    all_done=true
-    for pid in "${pids[@]}"; do
-        if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then
-            all_done=false
-            break
-        fi
-    done
-
-    if $all_done; then
-        cleanup_server
-        echo "All processes completed successfully"
-        exit 0
-    fi
-
-    sleep 0.2
-done