mor
This commit is contained in:
parent
ff1f474324
commit
e151208e15
7 changed files with 2933 additions and 784 deletions
|
|
@ -2,25 +2,29 @@
|
|||
|
||||
Test-only code shared by the deterministic and seeded clients.
|
||||
|
||||
| Suite | Coverage |
|
||||
| --- | --- |
|
||||
| `harness.test.ts`, `oracle-regressions.test.ts` | Independent content/identity oracles, exercised faults, exclusive operations, persistence ownership, cleanup and seeds |
|
||||
| `file-application.test.ts` | Swaps, file/directory transitions, visible conflict files, live edits, exclusions, and one metadata save per application |
|
||||
| `recovery.test.ts` | Interrupt every application filesystem/save boundary, reopen metadata, modify disk, and verify stable ordinary scans; manifest policy and notification identity |
|
||||
| `engine-regressions.test.ts`, `sync-audit.test.ts` | Offline identities, unknown network outcomes, rejection/retry, exclusions, interrupted in-memory pagination, namespace races and lifecycle changes |
|
||||
| `event-delivery.test.ts`, `polling.test.ts` | WebSocket hints with HTTP-only event reads, missed notifications, opt-in polling, shutdown and retry |
|
||||
| `protocol.test.ts` | Abrupt worker termination after accepted requests, exact retries, real server SIGKILL, CAS and event ordering |
|
||||
| `restart-reconciliation.test.ts` | Interrupted file application followed by user edits, two-client/server convergence and a second restart |
|
||||
| `server-restore.test.ts` | History checkpoints, server backup restores and interrupted metadata resets |
|
||||
| `content-limits.test.ts`, `protocol-audit.test.ts` | Content validation and API invariants |
|
||||
| `real-disk.test.ts` | Production Node adapter direct writes, no-replace moves, unsafe links, and rename cycles |
|
||||
| `production-adapters.test.ts` | Actual CLI processes, raw watchers, Obsidian adapter, complete metadata and restart after SIGKILL |
|
||||
| Suite | Coverage |
|
||||
| -------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `harness.test.ts`, `hardening.test.ts`, `oracle-regressions.test.ts` | Independent content/identity oracles, corruption/duplication mutation probes, exercised faults, exclusive operations, persistence ownership, cleanup and seeds |
|
||||
| `file-application.test.ts` | Swaps, file/directory transitions, visible conflict files, live edits, exclusions, and one metadata save per application |
|
||||
| `recovery.test.ts` | Interrupt every application filesystem/save boundary, reopen metadata, modify disk, and verify stable ordinary scans; manifest policy and notification identity |
|
||||
| `engine-regressions.test.ts`, `sync-audit.test.ts` | Offline identities, unknown network outcomes, rejection/retry, exclusions, interrupted in-memory pagination, namespace races and lifecycle changes |
|
||||
| `event-delivery.test.ts`, `polling.test.ts` | WebSocket hints with HTTP-only event reads, missed notifications, opt-in polling, shutdown and retry |
|
||||
| `protocol.test.ts` | Abrupt worker termination after accepted requests, exact retries, real server SIGKILL, CAS and event ordering |
|
||||
| `restart-reconciliation.test.ts` | Interrupted file application followed by user edits, two-client/server convergence and a second restart |
|
||||
| `server-restore.test.ts` | History checkpoints, server backup restores and interrupted metadata resets |
|
||||
| `content-limits.test.ts`, `protocol-audit.test.ts` | Content validation and API invariants |
|
||||
| `real-disk.test.ts` | Production Node adapter direct writes, no-replace moves, unsafe links, and rename cycles |
|
||||
| `production-adapters.test.ts` | Actual CLI processes, raw watchers, Obsidian adapter, complete metadata and restart after SIGKILL |
|
||||
|
||||
The filesystem interruption tests assert the supported contract: complete metadata
|
||||
remains usable, scans observe the current files, and a second scan is stable.
|
||||
They do not demand the pre-interruption file contents, identities or a replay of
|
||||
unfinished operations. Normal-operation tests still check exact bytes and identities.
|
||||
There is no filesystem journal or hidden recovery archive.
|
||||
The filesystem interruption tests require complete metadata to remain usable,
|
||||
ordinary scans to observe current files, and a second scan to remain stable. They
|
||||
also carry independent preservation obligations across every interruption:
|
||||
unreplaced/moved bytes must survive exactly once, unrelated durable identities must
|
||||
survive, and edits made after interruption must remain intact. The two-client
|
||||
restart regressions check the explicit `USER EDIT` and `AFTER INTERRUPTION` markers
|
||||
exactly once on both replicas through a second restart. An interrupted authorized
|
||||
replacement need not retain the replaced bytes or replay unfinished operations;
|
||||
there is no filesystem journal or hidden recovery archive.
|
||||
|
||||
`MemoryDisk` exposes `before:`, `visible:` and `durable:` boundaries. A process
|
||||
interruption retains visible changes; a simulated power loss rolls unflushed
|
||||
|
|
@ -30,7 +34,7 @@ is not an APFS/ext4 writeback model or a production durability requirement.
|
|||
after replacement. Abandoned filesystem sessions are fenced against later writes.
|
||||
|
||||
`crash-worker.ts` runs a complete client in a worker, which the parent terminates
|
||||
at accepted-response checkpoints without graceful shutdown. A new runtime receives
|
||||
at accepted-response or filesystem/persistence checkpoints without graceful shutdown. A new runtime receives
|
||||
the saved disk and metadata. Real-disk tests exercise the production Node adapter.
|
||||
The production integration suite launches built CLI processes and an Obsidian
|
||||
filesystem adapter against a real server. Obsidian unit tests exercise unsaved editor
|
||||
|
|
|
|||
45
frontend/test-support/checkpoint.ts
Normal file
45
frontend/test-support/checkpoint.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import assert from "node:assert/strict";
|
||||
|
||||
/** A one-shot, observable scheduling boundary. An unexercised gate is a failure. */
|
||||
export class Checkpoint<T> {
|
||||
private readonly reached = Promise.withResolvers<T>();
|
||||
private readonly released = Promise.withResolvers<void>();
|
||||
public fired = false;
|
||||
public resumed = false;
|
||||
public value?: T;
|
||||
|
||||
public async enter(value: T, signal?: AbortSignal | null): Promise<void> {
|
||||
assert(!this.fired, "Checkpoint entered twice");
|
||||
this.fired = true;
|
||||
this.value = structuredClone(value);
|
||||
this.reached.resolve(this.value);
|
||||
signal?.throwIfAborted();
|
||||
let abort = () => {};
|
||||
try {
|
||||
await Promise.race([
|
||||
this.released.promise,
|
||||
new Promise<never>((_, reject) => {
|
||||
abort = () =>
|
||||
reject(
|
||||
signal?.reason ?? new Error("Checkpoint aborted")
|
||||
);
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
})
|
||||
]);
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
}
|
||||
}
|
||||
|
||||
public wait(): Promise<T> {
|
||||
return this.reached.promise;
|
||||
}
|
||||
public resume(): void {
|
||||
this.resumed = true;
|
||||
this.released.resolve();
|
||||
}
|
||||
public assertConsumed(): void {
|
||||
assert(this.fired, "Checkpoint never reached");
|
||||
assert(this.resumed, "Checkpoint never released");
|
||||
}
|
||||
}
|
||||
140
frontend/test-support/hardening.test.ts
Normal file
140
frontend/test-support/hardening.test.ts
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { NetworkFaults } from "./network";
|
||||
import { AssertableState } from "../deterministic-tests/src/utils/assertable-state";
|
||||
import { TESTS } from "../deterministic-tests/src/test-registry";
|
||||
import type { ExpectedDocument } from "../deterministic-tests/src/test-definition";
|
||||
|
||||
const id = "00000000-0000-4000-8000-000000000001";
|
||||
function state(
|
||||
files: Record<string, string>,
|
||||
identities = Object.fromEntries(
|
||||
Object.keys(files).map((path, i) => [
|
||||
i ? id.replace(/1$/, String(i + 1)) : id,
|
||||
path
|
||||
])
|
||||
)
|
||||
) {
|
||||
const map = new Map(Object.entries(files));
|
||||
return new AssertableState({
|
||||
files: map,
|
||||
clientFiles: [map, new Map(map)],
|
||||
canonical: identities,
|
||||
manifests: [identities, identities],
|
||||
bytes: new Map(
|
||||
[...map].map(([path, text]) => [
|
||||
path,
|
||||
new TextEncoder().encode(text)
|
||||
])
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
test("scripted verifiers reject the concrete shared-loss and duplication counterexamples", () => {
|
||||
const cases = [
|
||||
[
|
||||
"multi-file-operations",
|
||||
{ "B.md": "updated", "C.md": "CORRUPTED", "unexpected.md": "EXTRA" }
|
||||
],
|
||||
[
|
||||
"concurrent-edit-exact-same-position",
|
||||
{ "doc.md": "slow slow fast brown fox CORRUPTED" }
|
||||
]
|
||||
] as const;
|
||||
for (const [name, files] of cases) {
|
||||
const step = TESTS[name]!.steps.filter(
|
||||
(s) => s.type === "assert-consistent"
|
||||
).at(-1)!;
|
||||
assert.throws(() => step.verify!(state(files)));
|
||||
}
|
||||
const partial = state({ "a.md": "A", "b.md": "B" });
|
||||
partial.assertContent("a.md", "A");
|
||||
assert.throws(() => partial.assertComplete(), /complete file set/);
|
||||
});
|
||||
|
||||
test("complete document oracles kill lost-byte, duplicate-file and identity-reuse mutations", () => {
|
||||
const expected: ExpectedDocument[] = [
|
||||
{ key: "a", path: "a.md", content: "exact bytes 🦊\r\n" }
|
||||
];
|
||||
const identities = new Map<string, string>();
|
||||
state({ "a.md": "exact bytes 🦊\r\n" }).assertDocuments(
|
||||
expected,
|
||||
identities
|
||||
);
|
||||
const corruptions: Record<string, string>[] = [
|
||||
{ "a.md": "exact bytes 🦊\n" },
|
||||
{ "a.md": "exact bytes 🦊\r\nexact bytes 🦊\r\n" },
|
||||
{ "a.md": "exact bytes 🦊\r\n", "extra.md": "EXTRA" }
|
||||
];
|
||||
for (const corrupted of corruptions)
|
||||
assert.throws(() =>
|
||||
state(corrupted).assertDocuments(expected, identities)
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
state(
|
||||
{ "a.md": "exact bytes 🦊\r\n" },
|
||||
{ [id.replace(/1$/, "2")]: "a.md" }
|
||||
).assertDocuments(expected, identities),
|
||||
/identity changed/
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
state({ "a.md": "exact bytes 🦊\r\n" }).assertDocuments(
|
||||
[{ ...expected[0], key: "recreated" }],
|
||||
identities
|
||||
),
|
||||
/reused/
|
||||
);
|
||||
});
|
||||
|
||||
test("request checkpoints hold the exact boundary and reject unexercised response predicates", async () => {
|
||||
const network = new NetworkFaults();
|
||||
let sent = 0,
|
||||
completed = false;
|
||||
const input = "http://localhost/vaults/test/documents/document";
|
||||
const init = {
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ requestId: "one", parentVersionId: 1 })
|
||||
};
|
||||
const fetcher = network.wrap(async () => {
|
||||
sent++;
|
||||
return Response.json({ type: "StaleBase" });
|
||||
});
|
||||
network.hold({ kind: "content", point: "before" });
|
||||
assert.throws(() => network.resumeRequest(), /before it is reached/);
|
||||
const request = fetcher(input, init).then(() => {
|
||||
completed = true;
|
||||
});
|
||||
await network.waitForHeldRequest();
|
||||
assert.equal(sent, 0);
|
||||
assert.equal(completed, false);
|
||||
assert.throws(() => network.assertConsumed(), /never released/);
|
||||
network.resumeRequest();
|
||||
await request;
|
||||
network.assertConsumed();
|
||||
assert.equal(network.requests[0].responseType, "StaleBase");
|
||||
network.hold({ kind: "content", point: "after", responseType: "Accepted" });
|
||||
assert.equal(
|
||||
network.requestsSinceCheckpoint().length,
|
||||
0,
|
||||
"An earlier response must not satisfy a new race assertion"
|
||||
);
|
||||
await fetcher(input, init);
|
||||
assert.throws(() => network.assertConsumed(), /never reached/);
|
||||
network.releaseRequests();
|
||||
});
|
||||
|
||||
test("scripted races have no sleep steps and the stale-content regression asserts its response", () => {
|
||||
for (const [name, scenario] of Object.entries(TESTS))
|
||||
assert(
|
||||
!scenario!.steps.some((s) => s.type === "sleep"),
|
||||
`${name} still relies on timing`
|
||||
);
|
||||
assert(
|
||||
TESTS["merging-update-response-survives-user-rename"]!.steps.some(
|
||||
(s) =>
|
||||
s.type === "assert-response" && s.responseType === "StaleBase"
|
||||
)
|
||||
);
|
||||
});
|
||||
|
|
@ -84,7 +84,7 @@ for (const phase of ["before", "visible", "durable"] as const) {
|
|||
};
|
||||
await assert.rejects(session.rename("a", "b"), /crash/);
|
||||
assert(fired);
|
||||
disk.boundary = () => { };
|
||||
disk.boundary = () => {};
|
||||
const moved =
|
||||
phase === "durable" || (phase === "visible" && !powerLoss);
|
||||
assert.deepEqual([...disk.userFiles().keys()], [moved ? "b" : "a"]);
|
||||
|
|
@ -168,10 +168,13 @@ test("unconsumed and before-send faults cannot silently pass", async () => {
|
|||
);
|
||||
assert(!called);
|
||||
assert.throws(() => net.assertConsumed(), /never retried/);
|
||||
await net.wrap(async () => new Response())("http://test/file-manifest", {
|
||||
method: "PUT",
|
||||
body: '{"requestId":"r"}'
|
||||
});
|
||||
await net.wrap(async () => Response.json({ type: "Accepted" }))(
|
||||
"http://test/file-manifest",
|
||||
{
|
||||
method: "PUT",
|
||||
body: '{"requestId":"r"}'
|
||||
}
|
||||
);
|
||||
net.assertConsumed();
|
||||
});
|
||||
|
||||
|
|
@ -231,7 +234,7 @@ test("seeded actions are repeatable, including seed zero", () => {
|
|||
test("runner cleanup errors fail an otherwise successful test", async () => {
|
||||
const server = {
|
||||
isRunning: () => true,
|
||||
resume: () => { }
|
||||
resume: () => {}
|
||||
} as unknown as ServerControl;
|
||||
const runner = new TestRunner(
|
||||
server,
|
||||
|
|
@ -241,13 +244,16 @@ test("runner cleanup errors fail an otherwise successful test", async () => {
|
|||
);
|
||||
let cleaned = 0;
|
||||
Object.assign(runner, {
|
||||
initializeAgents: async () => { },
|
||||
initializeAgents: async () => {},
|
||||
agents: [
|
||||
{
|
||||
clientId: 0,
|
||||
database: () => ({ evidence: "before cleanup" }),
|
||||
disk: { image: () => ({ bytes: [0, 255] }) },
|
||||
network: { requests: [{ body: { requestId: "original" } }] },
|
||||
network: {
|
||||
requests: [{ body: { requestId: "original" } }],
|
||||
schedule: []
|
||||
},
|
||||
cleanup: async () => {
|
||||
cleaned++;
|
||||
throw new Error("late background error");
|
||||
|
|
@ -267,7 +273,8 @@ test("runner cleanup errors fail an otherwise successful test", async () => {
|
|||
client: 0,
|
||||
database: { evidence: "before cleanup" },
|
||||
disk: { bytes: [0, 255] },
|
||||
requests: [{ body: { requestId: "original" } }]
|
||||
requests: [{ body: { requestId: "original" } }],
|
||||
schedule: []
|
||||
}
|
||||
]);
|
||||
});
|
||||
|
|
@ -282,7 +289,7 @@ test("disposed harness rejects late websocket reconnects", async () => {
|
|||
test("runner preserves primary failure and cleanup diagnostics", async () => {
|
||||
const server = {
|
||||
isRunning: () => true,
|
||||
resume: () => { }
|
||||
resume: () => {}
|
||||
} as unknown as ServerControl;
|
||||
const runner = new TestRunner(
|
||||
server,
|
||||
|
|
@ -291,7 +298,7 @@ test("runner preserves primary failure and cleanup diagnostics", async () => {
|
|||
"http://unused"
|
||||
);
|
||||
Object.assign(runner, {
|
||||
initializeAgents: async () => { },
|
||||
initializeAgents: async () => {},
|
||||
executeStep: async () => {
|
||||
throw new Error("primary assertion failed");
|
||||
},
|
||||
|
|
@ -300,7 +307,7 @@ test("runner preserves primary failure and cleanup diagnostics", async () => {
|
|||
clientId: 0,
|
||||
database: () => ({}),
|
||||
disk: { image: () => ({}) },
|
||||
network: { requests: [] },
|
||||
network: { requests: [], schedule: [] },
|
||||
cleanup: async () => {
|
||||
throw new Error("cleanup failed too");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { Checkpoint } from "./checkpoint";
|
||||
|
||||
export type RequestKind = "create" | "content" | "manifest";
|
||||
export interface RequestRecord {
|
||||
|
|
@ -7,6 +8,13 @@ export interface RequestRecord {
|
|||
url: string;
|
||||
body: Record<string, unknown>;
|
||||
status?: number;
|
||||
responseType?: string;
|
||||
}
|
||||
|
||||
export interface RequestCheckpoint {
|
||||
kind: RequestKind | "read-content" | "events";
|
||||
point: "before" | "after";
|
||||
responseType?: string;
|
||||
}
|
||||
|
||||
interface InjectedFault {
|
||||
|
|
@ -52,6 +60,79 @@ export function classifyRequest(
|
|||
|
||||
export class NetworkFaults {
|
||||
public readonly requests: RequestRecord[] = [];
|
||||
public readonly schedule: { type: string; detail: unknown }[] = [];
|
||||
private readonly checkpoints: {
|
||||
spec: RequestCheckpoint;
|
||||
gate: Checkpoint<unknown>;
|
||||
firstRequest: number;
|
||||
}[] = [];
|
||||
public hold(spec: RequestCheckpoint): void {
|
||||
assert(
|
||||
spec.point === "after" || spec.responseType === undefined,
|
||||
"Response predicates require an after-response checkpoint"
|
||||
);
|
||||
this.checkpoints.at(-1)?.gate.assertConsumed();
|
||||
this.checkpoints.push({
|
||||
spec,
|
||||
gate: new Checkpoint(),
|
||||
firstRequest: this.requests.length
|
||||
});
|
||||
this.schedule.push({ type: "armed", detail: spec });
|
||||
}
|
||||
public waitForHeldRequest(): Promise<unknown> {
|
||||
const current = this.checkpoints.at(-1);
|
||||
assert(current, "No request checkpoint armed");
|
||||
return current.gate.wait();
|
||||
}
|
||||
public requestsSinceCheckpoint(): RequestRecord[] {
|
||||
const current = this.checkpoints.at(-1);
|
||||
assert(current, "No request checkpoint armed");
|
||||
return this.requests.slice(current.firstRequest);
|
||||
}
|
||||
public resumeRequest(): void {
|
||||
const current = this.checkpoints.at(-1);
|
||||
assert(current, "No request checkpoint armed");
|
||||
assert(
|
||||
current.gate.fired,
|
||||
"Cannot release a request checkpoint before it is reached"
|
||||
);
|
||||
current.gate.resume();
|
||||
this.schedule.push({ type: "released", detail: current.spec });
|
||||
}
|
||||
public releaseRequests(): void {
|
||||
for (const { gate } of this.checkpoints) gate.resume();
|
||||
}
|
||||
public assertCheckpointsConsumed(): void {
|
||||
for (const { gate } of this.checkpoints) gate.assertConsumed();
|
||||
assert(
|
||||
!this.observation ||
|
||||
(this.observation.fired && this.observation.released),
|
||||
"Observation checkpoint was not exercised and released"
|
||||
);
|
||||
}
|
||||
private async requestCheckpoint(
|
||||
kind: RequestCheckpoint["kind"] | undefined,
|
||||
point: RequestCheckpoint["point"],
|
||||
detail: unknown,
|
||||
signal?: AbortSignal | null,
|
||||
responseType?: string
|
||||
): Promise<void> {
|
||||
const current = this.checkpoints.at(-1);
|
||||
if (
|
||||
!current ||
|
||||
current.gate.fired ||
|
||||
current.spec.kind !== kind ||
|
||||
current.spec.point !== point ||
|
||||
(current.spec.responseType !== undefined &&
|
||||
current.spec.responseType !== responseType)
|
||||
)
|
||||
return;
|
||||
this.schedule.push({
|
||||
type: "reached",
|
||||
detail: structuredClone({ spec: current.spec, request: detail })
|
||||
});
|
||||
await current.gate.enter(detail, signal);
|
||||
}
|
||||
private observation?: {
|
||||
released: boolean;
|
||||
fired: boolean;
|
||||
|
|
@ -150,11 +231,7 @@ export class NetworkFaults {
|
|||
await this.armed.reached;
|
||||
}
|
||||
public assertConsumed(): void {
|
||||
assert(
|
||||
!this.observation ||
|
||||
(this.observation.fired && this.observation.released),
|
||||
"Observation checkpoint was not exercised and released"
|
||||
);
|
||||
this.assertCheckpointsConsumed();
|
||||
for (const fault of this.faults) {
|
||||
assert(fault.fired, "Armed network fault never fired");
|
||||
assert(fault.retried, "Interrupted CAS request was never retried");
|
||||
|
|
@ -175,6 +252,13 @@ export class NetworkFaults {
|
|||
(input instanceof Request ? input.signal : undefined);
|
||||
if (observing) await this.observationCheckpoint(signal);
|
||||
const record = classifyRequest(input, init);
|
||||
const kind =
|
||||
record?.kind ??
|
||||
(observing && /\/documents\/.*\/content$/.test(url.pathname)
|
||||
? "read-content"
|
||||
: observing && url.pathname.endsWith("/events-since")
|
||||
? "events"
|
||||
: undefined);
|
||||
if (record) {
|
||||
// A client serializes its persisted CAS requests. Its next CAS
|
||||
// after a lost response must replay the interrupted operation,
|
||||
|
|
@ -192,6 +276,10 @@ export class NetworkFaults {
|
|||
"Retry changed payload/request identity"
|
||||
);
|
||||
this.armed!.retried = true;
|
||||
this.schedule.push({
|
||||
type: `retry:${record.kind}`,
|
||||
detail: structuredClone(record)
|
||||
});
|
||||
}
|
||||
const previous = this.requests.find(
|
||||
(r) => r.body.requestId === record.body.requestId
|
||||
|
|
@ -216,23 +304,45 @@ export class NetworkFaults {
|
|||
const fail = () => {
|
||||
fault!.interrupted = structuredClone(record!);
|
||||
fault!.fired = true;
|
||||
this.schedule.push({
|
||||
type: `drop:${fault!.kind}:${fault!.point}`,
|
||||
detail: structuredClone(record)
|
||||
});
|
||||
fault!.resolve();
|
||||
throw new TypeError(
|
||||
`Injected ${fault!.point}-commit ${fault!.kind} network failure`
|
||||
);
|
||||
};
|
||||
if (matches && fault.point === "before") fail();
|
||||
await this.requestCheckpoint(
|
||||
kind,
|
||||
"before",
|
||||
record ?? { url: url.href },
|
||||
signal
|
||||
);
|
||||
const response = await fetch(input, init);
|
||||
if (observing) await this.observationCheckpoint(signal);
|
||||
if (record) record.status = response.status;
|
||||
if (record) {
|
||||
record.status = response.status;
|
||||
if (response.ok)
|
||||
record.responseType = (
|
||||
(await response.clone().json()) as { type?: string }
|
||||
).type;
|
||||
}
|
||||
await this.requestCheckpoint(
|
||||
kind,
|
||||
"after",
|
||||
record ?? { url: url.href },
|
||||
signal,
|
||||
record?.responseType
|
||||
);
|
||||
// Only drop accepted replies, never confuse a rejected CAS with commit.
|
||||
if (
|
||||
matches &&
|
||||
!fault.fired &&
|
||||
fault.point === "after" &&
|
||||
response.ok &&
|
||||
((await response.clone().json()) as { type?: string }).type ===
|
||||
"Accepted"
|
||||
record?.responseType === "Accepted"
|
||||
) {
|
||||
await response.body?.cancel();
|
||||
fail();
|
||||
|
|
|
|||
|
|
@ -35,10 +35,7 @@ async function fixture(
|
|||
}
|
||||
for (const [id, path] of Object.entries(initial.local))
|
||||
initial.documents[id] = {
|
||||
|
||||
observedHash: (
|
||||
await hashSnapshot((await disk.read(path))!)
|
||||
).hash
|
||||
observedHash: (await hashSnapshot((await disk.read(path))!)).hash
|
||||
};
|
||||
const persistence = new MemoryPersistence({ database: initial });
|
||||
const restart = (changes: LocalChange[] = []) => {
|
||||
|
|
@ -55,7 +52,7 @@ async function fixture(
|
|||
database,
|
||||
config,
|
||||
undefined,
|
||||
{ entries: () => changes, flush: async () => { } }
|
||||
{ entries: () => changes, flush: async () => {} }
|
||||
)
|
||||
};
|
||||
};
|
||||
|
|
@ -64,7 +61,10 @@ async function fixture(
|
|||
persistence,
|
||||
restart,
|
||||
initial,
|
||||
permanentFault: (_label: string): void => { },
|
||||
allowedInterruptedContent: undefined as
|
||||
| { path: string; variants: Uint8Array[] }
|
||||
| undefined,
|
||||
permanentFault: (_label: string): void => {},
|
||||
expectedLocal: undefined as Record<string, string> | undefined
|
||||
};
|
||||
}
|
||||
|
|
@ -126,6 +126,13 @@ async function mergeFixture() {
|
|||
const current = await fixture({ a: { path: "note.md", text: local } });
|
||||
return {
|
||||
...current,
|
||||
allowedInterruptedContent: {
|
||||
path: "note.md",
|
||||
variants: [
|
||||
bytes(local),
|
||||
bytes("local first paragraph\n\nremote last paragraph\n")
|
||||
]
|
||||
},
|
||||
next: structuredClone(current.initial),
|
||||
writes: {
|
||||
a: {
|
||||
|
|
@ -201,7 +208,27 @@ for (const [name, create] of [
|
|||
["permanent destination fallback", permanentDestinationFixture]
|
||||
] as const) {
|
||||
test(`${name}: interrupted operations leave metadata usable by ordinary scans`, async (t) => {
|
||||
const baseline = await create();
|
||||
const protectedFixture = async () => {
|
||||
const current = await create();
|
||||
const content = bytes("UNTOUCHED DURABLE BYTES\0\xff");
|
||||
await current.disk.userWrite("untouched.bin", content);
|
||||
const stored = current.persistence.snapshot();
|
||||
stored.database!.local!.protected = "untouched.bin";
|
||||
stored.database!.documents!.protected = {
|
||||
observedHash: (await hashSnapshot({ content })).hash
|
||||
};
|
||||
await current.persistence.save(stored);
|
||||
current.next.local.protected = "untouched.bin";
|
||||
current.next.documents.protected = {
|
||||
observedHash: (await hashSnapshot({ content })).hash
|
||||
};
|
||||
if (current.expectedLocal)
|
||||
(current.expectedLocal as Record<string, string>).protected =
|
||||
"untouched.bin";
|
||||
current.expected.set("untouched.bin", content);
|
||||
return current;
|
||||
};
|
||||
const baseline = await protectedFixture();
|
||||
const labels: string[] = [];
|
||||
const trace = (label: string) => {
|
||||
if (!label.startsWith("read:")) labels.push(label);
|
||||
|
|
@ -211,7 +238,9 @@ for (const [name, create] of [
|
|||
trace(label);
|
||||
};
|
||||
baseline.persistence.boundary = trace;
|
||||
await baseline.restart().files.applyChanges(baseline.next, baseline.writes);
|
||||
await baseline
|
||||
.restart()
|
||||
.files.applyChanges(baseline.next, baseline.writes);
|
||||
assert.deepEqual(baseline.disk.userFiles(), baseline.expected);
|
||||
assert.deepEqual(
|
||||
baseline.persistence.snapshot().database!.local,
|
||||
|
|
@ -223,7 +252,55 @@ for (const [name, create] of [
|
|||
await t.test(
|
||||
`${powerLoss ? "power" : "process"} interruption at ${labels[cut]}`,
|
||||
async () => {
|
||||
const current = await create();
|
||||
const current = await protectedFixture();
|
||||
// Moving a file does not authorize losing its bytes. Writes
|
||||
// and explicit deletions have separate replacement policies.
|
||||
const required = Object.entries(
|
||||
current.persistence.snapshot().database!.local!
|
||||
)
|
||||
.filter(
|
||||
([id]) =>
|
||||
id in current.next.local &&
|
||||
!(id in current.writes)
|
||||
)
|
||||
.map(
|
||||
([, path]) =>
|
||||
current.disk.userFiles().get(path)!
|
||||
);
|
||||
const preserve = () => {
|
||||
const allowed = current.allowedInterruptedContent;
|
||||
if (allowed) {
|
||||
const content = current.disk
|
||||
.userFiles()
|
||||
.get(allowed.path);
|
||||
assert(
|
||||
content &&
|
||||
allowed.variants.some((variant) =>
|
||||
Buffer.from(variant).equals(
|
||||
Buffer.from(content)
|
||||
)
|
||||
),
|
||||
"Interrupted merge lost or corrupted the user's pre-existing edit"
|
||||
);
|
||||
}
|
||||
const files = [
|
||||
...current.disk.userFiles().values()
|
||||
].map((value) => Buffer.from(value));
|
||||
for (const content of required)
|
||||
assert.equal(
|
||||
files.filter((value) =>
|
||||
value.equals(Buffer.from(content))
|
||||
).length,
|
||||
1,
|
||||
`Interrupted ${name} lost or duplicated existing bytes at ${labels[cut]}`
|
||||
);
|
||||
assert.deepEqual(
|
||||
current.disk
|
||||
.userFiles()
|
||||
.get("after-restart.md"),
|
||||
bytes("user edit")
|
||||
);
|
||||
};
|
||||
let seen = 0;
|
||||
let fired = false;
|
||||
const interrupt = (label: string) => {
|
||||
|
|
@ -242,18 +319,22 @@ for (const [name, create] of [
|
|||
await assert.rejects(
|
||||
current
|
||||
.restart()
|
||||
.files.applyChanges(current.next, current.writes),
|
||||
.files.applyChanges(
|
||||
current.next,
|
||||
current.writes
|
||||
),
|
||||
/interrupted/
|
||||
);
|
||||
assert(fired);
|
||||
current.disk.boundary = () => { };
|
||||
current.persistence.boundary = () => { };
|
||||
current.disk.boundary = () => {};
|
||||
current.persistence.boundary = () => {};
|
||||
// The user may edit after the interruption, including a partial
|
||||
// write or a file whose metadata was never saved.
|
||||
await current.disk.userWrite(
|
||||
"after-restart.md",
|
||||
bytes("user edit")
|
||||
);
|
||||
preserve();
|
||||
const runtime = current.restart();
|
||||
const observed = current.disk.userFiles();
|
||||
await scanChanges(runtime, []);
|
||||
|
|
@ -263,6 +344,12 @@ for (const [name, create] of [
|
|||
"A scan must not replay old filesystem intentions"
|
||||
);
|
||||
assertManifest(runtime.database.state.local);
|
||||
preserve();
|
||||
assert.equal(
|
||||
runtime.database.state.local.protected,
|
||||
"untouched.bin",
|
||||
"Unrelated durable identity changed"
|
||||
);
|
||||
assert.deepEqual(
|
||||
new Set(
|
||||
Object.values(runtime.database.state.local)
|
||||
|
|
@ -294,6 +381,7 @@ for (const [name, create] of [
|
|||
state,
|
||||
"Second startup must be stable"
|
||||
);
|
||||
preserve();
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -399,7 +487,7 @@ test("scan failures cannot masquerade as deletions", async () => {
|
|||
database: runtime.database,
|
||||
changes: [],
|
||||
files: runtime.files,
|
||||
guard: () => { },
|
||||
guard: () => {},
|
||||
ignored: () => false,
|
||||
oversized: () => false
|
||||
}),
|
||||
|
|
@ -426,7 +514,7 @@ for (const phase of ["before", "durable"]) {
|
|||
database: runtime.database,
|
||||
changes,
|
||||
files: runtime.files,
|
||||
guard: () => { },
|
||||
guard: () => {},
|
||||
ignored: () => false,
|
||||
oversized: () => false
|
||||
});
|
||||
|
|
@ -436,7 +524,7 @@ for (const phase of ["before", "durable"]) {
|
|||
if (label === `${phase}:save`) throw new Error("save failure");
|
||||
};
|
||||
await assert.rejects(scan(), /save failure/);
|
||||
current.persistence.boundary = () => { };
|
||||
current.persistence.boundary = () => {};
|
||||
const state = await scan();
|
||||
assert.equal(
|
||||
state.local.a,
|
||||
|
|
@ -459,7 +547,7 @@ async function scanChanges(
|
|||
database: runtime.database,
|
||||
changes,
|
||||
files: runtime.files,
|
||||
guard: () => { },
|
||||
guard: () => {},
|
||||
ignored: () => false,
|
||||
oversized: () => false
|
||||
});
|
||||
|
|
@ -540,7 +628,7 @@ for (const equalContent of [false, true]) {
|
|||
phase === "durable" ? expected : current.initial.local,
|
||||
"The fault must occur on the intended side of the durable save"
|
||||
);
|
||||
current.persistence.boundary = () => { };
|
||||
current.persistence.boundary = () => {};
|
||||
// Model an uncertain save in a live client: durable state reloads,
|
||||
// but the original editor notifications remain in memory. Do not
|
||||
// clear, reconstruct or deduplicate that queue in the harness.
|
||||
|
|
@ -574,7 +662,7 @@ test("normalization-sensitive disk does not lose a distinct decomposed filename"
|
|||
database: runtime.database,
|
||||
changes: [],
|
||||
files: runtime.files,
|
||||
guard: () => { },
|
||||
guard: () => {},
|
||||
ignored: () => false,
|
||||
oversized: () => false
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { fixture, bytes, randomUUID } from "./sync-server-fixture";
|
|||
import { assertCanonical } from "./canonical";
|
||||
import { SyncClient } from "../sync-client/src/sync-client";
|
||||
import { MemoryDisk, MemoryPersistence } from "./storage";
|
||||
import { ContentLedger } from "./oracles";
|
||||
|
||||
for (const powerLoss of [false, true])
|
||||
for (const operation of ["create", "replace", "move", "metadata"] as const)
|
||||
|
|
@ -98,6 +99,9 @@ for (const powerLoss of [false, true])
|
|||
files: () => f.disk.userFiles(),
|
||||
database: () => f.store.snapshot().database
|
||||
};
|
||||
const edits = new ContentLedger();
|
||||
edits.add("USER EDIT");
|
||||
edits.add("AFTER INTERRUPTION");
|
||||
const peerDisk = new MemoryDisk();
|
||||
const peerStore = new MemoryPersistence({
|
||||
settings: f.store.snapshot().settings
|
||||
|
|
@ -119,6 +123,8 @@ for (const powerLoss of [false, true])
|
|||
f.url,
|
||||
"test-token-change-me"
|
||||
);
|
||||
for (const client of clients)
|
||||
edits.assertPreserved(client.files());
|
||||
assert.deepEqual(
|
||||
f.disk.userFiles().get("after.md"),
|
||||
bytes("AFTER INTERRUPTION\n")
|
||||
|
|
@ -141,6 +147,8 @@ for (const powerLoss of [false, true])
|
|||
files,
|
||||
"A second restart must not replay old writes"
|
||||
);
|
||||
for (const client of clients)
|
||||
edits.assertPreserved(client.files());
|
||||
} finally {
|
||||
f.disk.boundary = () => {};
|
||||
f.store.boundary = () => {};
|
||||
|
|
|
|||
Loading…
Reference in a new issue