This commit is contained in:
Andras Schmelczer 2026-09-26 20:22:16 +01:00
commit b07aa589ff
12 changed files with 728 additions and 530 deletions

View file

@ -68,7 +68,7 @@ export async function assertCanonical(
"Snapshot heads differ from membership"
);
for (const [i, state] of states.entries()) {
assert(state?.initialized, `Client ${i} never initialized`);
assert(state?.lastSeenUpdateId !== undefined, `Client ${i} never initialized`);
assertQuiescent(state, entries, snapshot.headEventId);
assert.equal(
state.fileManifest?.fileManifestId,
@ -82,7 +82,7 @@ export async function assertCanonical(
);
for (const id of Object.keys(entries)) {
assert(
state.documents?.[id]?.materialized,
state.documents?.[id]?.observedHash !== undefined,
`Unmaterialized document: ${id}`
);
assert.equal(

View file

@ -75,9 +75,7 @@ test("stale content acknowledgements contain metadata without the excluded paylo
test("highly fragmented local edits use a snapshot within the server operation budget", async () => {
const { fixture } = await import("./sync-fixture");
const { toStoredSnapshot } = await import(
"../sync-client/src/sync-operations/content"
);
const { toStoredSnapshot } = await import("../sync-client/src/snapshot");
const f = await fixture({});
f.contentCache.resize(1024 * 1024);
const original = Array.from({ length: 12_000 }, (_, i) => `word${i}`).join(

View file

@ -4,7 +4,7 @@ import { test } from "node:test";
import { Database, emptyState } from "../sync-client/src/persistence/database";
import { Syncer } from "../sync-client/src/sync-operations/syncer";
import { SyncClient, type StoredClient } from "../sync-client/src/sync-client";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
import { allocatePortablePath } from "../sync-client/src/utils/portable-path";
import { MemoryDisk, MemoryPersistence } from "./storage";
@ -14,10 +14,10 @@ test("offline content notifications need no saves and restart reads the current
const disk = new MemoryDisk();
await disk.userWrite("a.md", bytes("original"));
const initial = emptyState(JSON.stringify(["http://offline.test", "test"]));
initial.initialized = true;
initial.lastSeenUpdateId = 0;
initial.local = { a: "a.md" };
initial.documents.a = {
materialized: true,
observedHash: (await toStoredSnapshot({ content: bytes("original") }))
.hash
};
@ -70,7 +70,7 @@ test("legacy content notifications preserve the acknowledged namespace prefix on
await disk.userWrite("d.md", bytes("A"));
await disk.userWrite("b.md", bytes("B"));
const initial = emptyState(JSON.stringify(["http://offline.test", "test"]));
initial.initialized = true;
initial.lastSeenUpdateId = 0;
initial.local = { a: "c.md", b: "b.md" };
initial.fileManifest.entries = { ...initial.local };
for (const [id, content] of [
@ -78,7 +78,7 @@ test("legacy content notifications preserve the acknowledged namespace prefix on
["b", "B"]
])
initial.documents[id] = {
materialized: true,
observedHash: (await toStoredSnapshot({ content: bytes(content) }))
.hash
};
@ -134,7 +134,7 @@ test("remote binary replacement and deletion do not create backup artifacts", as
});
assert.deepEqual(await f.disk.listFilesRecursively(), ["a.bin"]);
assert.deepEqual(f.disk.userFiles(), new Map([["a.bin", bytes("remote")]]));
await f.internals.incorporateFileManifest({
await f.internals.reconcile({
fileManifestId: 2,
entries: {}
});
@ -202,10 +202,11 @@ test("an unmaterialized file with a directory at its path is deconflicted", asyn
);
await f.disk.userWrite("folder/note.md", bytes("local"));
f.database.state.local.a = "folder";
f.database.state.documents.a = { materialized: false };
f.database.state.documents.a = {};
f.database.state.fileManifest.entries.a = "folder";
await f.database.save();
await f.internals.incorporateContent(head("a", 2, "remote"));
f.database.state.documents.a.remote = head("a", 2, "remote");
await f.internals.reconcile(f.database.state.fileManifest);
assert(
[...f.disk.userFiles().values()].some(
(value) => Buffer.from(value).toString() === "remote"
@ -221,7 +222,7 @@ test("an unmaterialized file with a directory at its path is deconflicted", asyn
test("ignored materialized content survives remote deletion", async () => {
const f = await fixture();
await f.settings.setSettings({ ignorePatterns: ["a.md"] });
await f.internals.incorporateFileManifest({
await f.internals.reconcile({
fileManifestId: 2,
entries: {}
});
@ -231,11 +232,9 @@ test("ignored materialized content survives remote deletion", async () => {
test("unfinished bootstrap cannot be rebound to another vault", async () => {
const database = emptyState(JSON.stringify(["http://example.test", "old"]));
database.bootstrap = {
headEventId: 1,
fileManifest: { fileManifestId: 1, entries: {} },
documents: []
};
// A scan may commit local identities before the first remote application.
database.local = { a: "a.md" };
database.documents.a = { observedHash: "" };
const persistence = new MemoryPersistence({
settings: { remoteUri: "http://example.test", vaultName: "old" },
database
@ -276,7 +275,7 @@ test("a rejected upload is retained locally and does not block another file or a
assert(await f.internals.preparePush());
await f.internals.finishPending();
assert.equal(f.database.state.pending, undefined);
assert.match(f.database.state.documents.a.rejected!.message, /413/);
assert.match(f.database.state.unconfirmed![0].rejection!, /413/);
assert(await f.internals.preparePush());
await f.internals.finishPending();
assert.deepEqual(sent, ["a", "b"]);
@ -286,7 +285,7 @@ test("a rejected upload is retained locally and does not block another file or a
assert(await f.internals.preparePush());
await f.internals.finishPending();
assert.deepEqual(sent, ["a", "b", "a"]);
assert.equal(f.database.state.documents.a.rejected, undefined);
assert.equal(f.database.state.unconfirmed?.find((request) => request.rejection), undefined);
});
test("a transient upload failure retries the exact request and recovers its acknowledgement", async () => {
@ -306,7 +305,7 @@ test("a transient upload failure retries the exact request and recovers its ackn
assert.equal(requests.length, 2);
assert.deepEqual(requests[1], requests[0]);
assert.equal(f.database.state.documents.a.base!.vaultUpdateId, 2);
assert.equal(f.database.state.documents.a.rejected, undefined);
assert.equal(f.database.state.unconfirmed?.find((request) => request.rejection), undefined);
assert.equal(f.database.state.pending, undefined);
});
@ -327,7 +326,7 @@ test("a persisted upload rejection finishes without filesystem writes", async ()
await f.internals.finishPending();
assert.equal(uploads, 1);
assert.equal(f.database.state.pending, undefined);
assert.equal(f.database.state.documents.a.rejected?.message, "HTTP 413");
assert.equal(f.database.state.unconfirmed![0].rejection, "HTTP 413");
assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]]));
});
@ -388,7 +387,6 @@ test("rejected manifests can be replaced after editing the local namespace", asy
};
f.database.state.local = { a: "new.md" };
f.database.state.documents.a = {
materialized: false,
base: { ...head("a", 1, "x"), hash: "hash" }
};
await f.database.save();
@ -405,9 +403,9 @@ test("rejected manifests can be replaced after editing the local namespace", asy
test("offline delete/recreate identities survive restarting before any network connection", async () => {
const key = JSON.stringify(["http://offline.test", "test"]);
const initial = emptyState(key);
initial.initialized = true;
initial.lastSeenUpdateId = 0;
initial.local.a = "a.md";
initial.documents.a = { materialized: true };
initial.documents.a = { observedHash: "" };
initial.fileManifest = { fileManifestId: 1, entries: { a: "a.md" } };
const disk = new MemoryDisk();
await disk.userWrite("a.md", bytes("same"));
@ -465,12 +463,12 @@ test("offline notifications bind an otherwise empty client to its vault", async
test("unignoring a deleted file resumes the deferred remote deletion without resurrection", async () => {
const f = await fixture();
await f.settings.setSettings({ ignorePatterns: ["a.md"] });
await f.internals.incorporateFileManifest({
await f.internals.reconcile({
fileManifestId: 2,
entries: {}
});
await f.settings.setSettings({ ignorePatterns: [] });
await f.internals.incorporateFileManifest(f.database.state.fileManifest);
await f.internals.reconcile(f.database.state.fileManifest);
assert.equal(f.disk.userFiles().size, 0);
assert.deepEqual(f.database.state.local, {});
});
@ -478,7 +476,7 @@ test("unignoring a deleted file resumes the deferred remote deletion without res
test("an ignored file does not move when a remote rename targets its path", async () => {
const f = await fixture({ "a.md": "A", "b.md": "B" });
await f.settings.setSettings({ ignorePatterns: ["a.md"] });
await f.internals.incorporateFileManifest({
await f.internals.reconcile({
fileManifestId: 2,
entries: { a: "c.md", b: "a.md" }
});
@ -623,12 +621,12 @@ test("ignoring a locally renamed file defers its move without forgetting the loc
await f.internals.scan();
f.internals.stopped = false;
assert.equal(await f.internals.preparePush(), false);
await f.internals.incorporateFileManifest({
await f.internals.reconcile({
fileManifestId: 2,
entries: { a: "a.md" }
});
await f.settings.setSettings({ ignorePatterns: [] });
await f.internals.incorporateFileManifest(f.database.state.fileManifest);
await f.internals.reconcile(f.database.state.fileManifest);
assert.equal(f.database.state.local.a, "ignored.md");
assert.equal(
Buffer.from(
@ -648,7 +646,7 @@ test("a notified deletion of an ignored file does not delete its server membersh
assert.equal(await f.internals.preparePush(), false);
assert.equal(f.database.state.local.a, undefined);
await f.settings.setSettings({ ignorePatterns: [] });
await f.internals.incorporateFileManifest(f.database.state.fileManifest);
await f.internals.reconcile(f.database.state.fileManifest);
assert(await f.internals.preparePush());
assert.equal(f.database.state.pending?.type, "fileManifest");
});
@ -656,7 +654,7 @@ test("a notified deletion of an ignored file does not delete its server membersh
test("an incoming ignored document cannot force an existing ignored file to move", async () => {
const f = await fixture();
await f.settings.setSettings({ ignorePatterns: ["a.md", "*conflict*"] });
await f.internals.incorporateFileManifest({
await f.internals.reconcile({
fileManifestId: 2,
entries: { a: "b.md", other: "a.md" }
});
@ -687,7 +685,7 @@ test("rejected content is reported to callers and an explicit retry clears the r
test("notification saves cannot roll back an uncertain engine commit, even if the first reload fails", async () => {
const key = JSON.stringify(["http://offline.test", "test"]);
const initial = emptyState(key);
initial.initialized = true;
initial.lastSeenUpdateId = 0;
const disk = new MemoryDisk();
const store = new MemoryPersistence({
database: initial,
@ -755,3 +753,40 @@ test(
assert(notified);
}
);
for (const edited of [false, true]) {
test(`hash-only history recovery preserves ${edited ? "an edit made after reset" : "a clean file"} across restart`, async () => {
const remote = head("a", 2, "restored");
const service = {
getVaultSnapshot: async () => ({
headEventId: 3,
fileManifest: { fileManifestId: 3, entries: { a: "a.md" } },
documents: [remote]
}),
getDocumentVersionContent: async ({ vaultUpdateId }: { vaultUpdateId: number }) => {
assert.equal(vaultUpdateId, 2, "never read a version from the abandoned history");
return bytes("restored");
}
};
let f = await fixture({ "a.md": "old branch" }, service);
const hash = f.database.state.documents.a.observedHash!;
f.database.state.documents.a.base = { vaultUpdateId: 50, hash };
await (f.syncer as unknown as { recoverServerHistory(): Promise<void> }).recoverServerHistory();
assert.equal(f.database.state.lastSeenUpdateId, undefined);
assert.deepEqual(f.database.state.documents.a, {
observedHash: hash,
base: { vaultUpdateId: null, hash }
});
if (edited) await f.disk.userWrite("a.md", bytes("my new edit"));
f = await fixture({}, service, f);
await (f.syncer as unknown as { bootstrap(): Promise<void> }).bootstrap();
assert.equal(f.database.state.lastSeenUpdateId, 3);
assert.deepEqual((await f.disk.read("a.md"))?.content, bytes("restored"));
const other = Object.keys(f.database.state.local).filter((id) => id !== "a");
assert.equal(other.length, edited ? 1 : 0);
if (edited) {
assert.deepEqual((await f.disk.read(f.database.state.local[other[0]]))?.content, bytes("my new edit"));
assert.equal(f.database.state.documents[other[0]].base, undefined);
}
});
}

View file

@ -30,7 +30,7 @@ for (const [count, delivery] of [
async () => {
const logger = new Logger();
const initial = emptyState("event-delivery");
initial.initialized = true;
initial.lastSeenUpdateId = 0;
const persistence = new MemoryPersistence({ database: initial });
const database = new Database(
initial,

View file

@ -1,7 +1,7 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { fixture, bytes } from "./sync-fixture";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
import { LocalChangesDuringReconciliation } from "../sync-client/src/errors/errors";
for (const equal of [false, true])

View file

@ -34,13 +34,13 @@ test("canonical checks independently verify both persisted content hashes", asyn
};
const manifest = { fileManifestId: 2, entries: { [id]: "a.md" } };
const state: Partial<StoredDatabase> = {
initialized: true,
local: manifest.entries,
fileManifest: manifest,
lastSeenUpdateId: 2,
documents: {
[id]: {
materialized: true,
observedHash: hash,
base: { ...document, hash }
}
@ -297,7 +297,7 @@ test("canonical checks follow pages without weakening gap or duplicate-request d
const client = () => ({
files: () => new Map<string, Uint8Array>(),
database: () => ({
initialized: true,
local: {},
documents: {},
fileManifest: manifest,

View file

@ -8,7 +8,7 @@ import {
import { Settings } from "../sync-client/src/persistence/settings";
import { FileOperations } from "../sync-client/src/file-operations/file-operations";
import { Syncer } from "../sync-client/src/sync-operations/syncer";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
import { Logger } from "../sync-client/src/tracing/logger";
import { SyncHistory } from "../sync-client/src/tracing/sync-history";
import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache";
@ -87,14 +87,12 @@ async function fixture(syncIntervalMs: number | undefined) {
}
};
const initial = emptyState("polling-test");
initial.initialized = true;
initial.lastSeenUpdateId = 2;
initial.local = { [documentId]: "note.bin" };
initial.fileManifest = { fileManifestId: 2, entries: { ...initial.local } };
initial.remoteHeads = { [documentId]: latest };
const hash = (await toStoredSnapshot({ content: bytes("base") })).hash;
initial.documents[documentId] = {
materialized: true,
remote: latest,
observedHash: hash,
base: { ...latest, hash }
};

View file

@ -148,7 +148,7 @@ for (const [kind, format, originalPath] of [
)[0];
originalParent =
initialized.stored.database!.documents![originalId]
.base!.vaultUpdateId;
.base!.vaultUpdateId!;
disk.restore(initialized.disk);
uploaded += "local-update\n";
await disk.userWrite(

View file

@ -7,7 +7,7 @@ import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesyste
import { Database, emptyState } from "../sync-client/src/persistence/database";
import { FileOperations } from "../sync-client/src/file-operations/file-operations";
import type { ServerConfig } from "../sync-client/src/services/server-config";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
test(
"real filesystem: direct updates, unsafe entries, and file swaps",
@ -101,7 +101,7 @@ test(
initial.local = { a: "a.md", b: "b.md" };
for (const [id, relative] of Object.entries(initial.local))
initial.documents[id] = {
materialized: true,
observedHash: (
await toStoredSnapshot(
(await disk.read(relative))!

View file

@ -7,7 +7,7 @@ import {
type StoredDatabase
} from "../sync-client/src/persistence/database";
import { FileOperations } from "../sync-client/src/file-operations/file-operations";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
import type { ServerConfig } from "../sync-client/src/services/server-config";
import { mergeFileManifests } from "../sync-client/src/sync-operations/file-manifest";
import { assertManifest, pathDecision } from "./oracles";
@ -35,7 +35,7 @@ async function fixture(
}
for (const [id, path] of Object.entries(initial.local))
initial.documents[id] = {
materialized: true,
observedHash: (
await toStoredSnapshot((await disk.read(path))!)
).hash
@ -73,7 +73,7 @@ async function swapFixture() {
const current = await fixture();
const next = structuredClone(current.initial);
next.local = { a: "b.md", b: "a.md", c: "nested/c.md" };
next.documents.c = { materialized: false };
next.documents.c = {};
const writes = {
c: { replacement: await toStoredSnapshot({ content: bytes("C") }) }
};
@ -274,7 +274,7 @@ for (const [name, create] of [
)) {
assert(
runtime.database.state.documents[id]
.materialized
.observedHash !== undefined
);
assert.equal(
runtime.database.state.documents[id]
@ -560,7 +560,7 @@ test("normalization-sensitive disk does not lose a distinct decomposed filename"
const runtime = current.restart();
const next = structuredClone(runtime.database.state);
next.local.composed = "é.md";
next.documents.composed = { materialized: true };
next.documents.composed = { observedHash: "" };
await runtime.database.commit(next);
await current.disk.userWrite("e\u0301.md", bytes("decomposed"));
await scanLocalFiles({

View file

@ -5,7 +5,7 @@ import { test } from "node:test";
import type { Database } from "../sync-client/src/persistence/database";
import { emptyState } from "../sync-client/src/persistence/database";
import { SyncClient } from "../sync-client/src/sync-client";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners";
import type { SyncService } from "../sync-client/src/services/sync-service";
import type { WebSocketManager } from "../sync-client/src/services/websocket-manager";
@ -50,8 +50,8 @@ test("ignored untracked ancestor must remain in place", async () => {
);
await f.disk.userWrite("private", bytes("LOCAL PRIVATE"));
await f.settings.setSettings({ ignorePatterns: ["private"] });
f.database.state.remoteHeads.a = head("a", 2, "remote");
await f.internals.incorporateFileManifest({
(f.database.state.documents.a ??= {}).remote = head("a", 2, "remote");
await f.internals.reconcile({
fileManifestId: 3,
entries: { a: "private/note.md" }
});
@ -69,8 +69,8 @@ test("oversized untracked file must remain in place", async () => {
);
await f.disk.userWrite("a.md", bytes("LOCAL OVERSIZED"));
await f.settings.setSettings({ maxFileSizeMB: 5 / (1024 * 1024) });
f.database.state.remoteHeads.a = head("a", 2, "R");
await f.internals.incorporateFileManifest({
(f.database.state.documents.a ??= {}).remote = head("a", 2, "R");
await f.internals.reconcile({
fileManifestId: 3,
entries: { a: "a.md" }
});
@ -133,7 +133,7 @@ test("rejected unknown success must use submitted content as merge base", async
test("bootstrap adopts same-path identity even after multiple notifications", async () => {
const f = await fixture({});
f.database.state.initialized = false;
delete f.database.state.lastSeenUpdateId;
f.database.state.fileManifest = { fileManifestId: 0, entries: {} };
await f.disk.userWrite("a.md", bytes("local a"));
await f.syncer.syncLocallyCreatedFile("a.md");
@ -168,7 +168,7 @@ test("oversized remote head must not cause identical stale uploads forever", asy
...head("a", 1, "base"),
hash: (await toStoredSnapshot({ content: bytes("base") })).hash
};
f.database.state.remoteHeads.a = head("a", 2, remoteText);
(f.database.state.documents.a ??= {}).remote = head("a", 2, remoteText);
await f.settings.setSettings({ maxFileSizeMB: 10 / (1024 * 1024) });
f.internals.stopped = false;
// Even a small local edit must wait until its required remote base is readable.
@ -480,13 +480,13 @@ for (const interrupted of [false, true]) {
...head("a", 1, "base"),
hash: (await toStoredSnapshot({ content: bytes("base") })).hash
};
f.database.state.remoteHeads.a = head("a", 1, "base");
(f.database.state.documents.a ??= {}).remote = head("a", 1, "base");
f.database.state.lastSeenUpdateId = 1;
await f.database.save();
const saved = f.persistence.snapshot();
const catchUp = async () =>
f.internals.incorporateEventBatch(
await service.getEvents!(f.database.state.lastSeenUpdateId)
await service.getEvents!(f.database.state.lastSeenUpdateId!)
);
if (interrupted) {
await assert.rejects(catchUp(), /interrupted page fetch/);
@ -515,8 +515,8 @@ test("an ignored occupant arriving after the scan keeps its path and privacy", a
}
);
await f.settings.setSettings({ ignorePatterns: ["private"] });
f.database.state.remoteHeads.a = head("a", 2, "remote");
await f.internals.incorporateFileManifest({
(f.database.state.documents.a ??= {}).remote = head("a", 2, "remote");
await f.internals.reconcile({
fileManifestId: 3,
entries: { a: "private/note.md" }
});
@ -818,10 +818,15 @@ for (const boundary of ["before:save", "durable:save"]) {
const { database } = f.client as unknown as { database: Database };
await database.commit({
...database.state,
rejectedManifest: {
entries: { retired: "old.md" },
message: "retired rejection"
}
unconfirmed: [{
type: "fileManifest",
request: {
requestId: "rejected",
parentFileManifestId: 0,
entries: { retired: "old.md" }
},
rejection: "retired rejection"
}]
});
let saves = 0,
failed = false;

View file

@ -6,7 +6,7 @@ import {
import { Settings } from "../sync-client/src/persistence/settings";
import { FileOperations } from "../sync-client/src/file-operations/file-operations";
import { Syncer } from "../sync-client/src/sync-operations/syncer";
import { toStoredSnapshot } from "../sync-client/src/sync-operations/content";
import { toStoredSnapshot } from "../sync-client/src/snapshot";
import { Logger } from "../sync-client/src/tracing/logger";
import { SyncHistory } from "../sync-client/src/tracing/sync-history";
import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache";
@ -37,13 +37,13 @@ export async function fixture(
? (recovered.persistence.snapshot().database as StoredDatabase)
: emptyState("test");
if (!recovered) {
initial.initialized = true;
initial.lastSeenUpdateId = 0;
for (const [path, content] of Object.entries(contents)) {
await disk.userWrite(path, bytes(content));
const id = path.split(".")[0];
initial.local[id] = path;
initial.documents[id] = {
materialized: true,
observedHash: (
await toStoredSnapshot({ content: bytes(content) })
).hash
@ -110,11 +110,10 @@ export async function fixture(
batch: import("../sync-client/src/services/types/EventBatch").EventBatch
) => Promise<void>;
stopped: boolean;
incorporateFileManifest: (remote: {
reconcile: (remote: {
fileManifestId: number;
entries: Record<string, string>;
}) => Promise<void>;
incorporateContent: (remote: ReturnType<typeof head>) => Promise<void>;
};
return {
disk,