power loss
This commit is contained in:
parent
0f6b618c0b
commit
d56084e4bc
22 changed files with 18667 additions and 3902 deletions
|
|
@ -0,0 +1,428 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
Database,
|
||||
emptyState,
|
||||
type StoredDatabase
|
||||
} from "../persistence/database";
|
||||
import type { ServerConfig } from "../services/server-config";
|
||||
import { toStoredSnapshot } from "../sync-operations/content";
|
||||
import { isInternalPath } from "../utils/portable-path";
|
||||
import { Logger } from "../tracing/logger";
|
||||
import { FileOperations, type FileWrite } from "./file-operations";
|
||||
import type {
|
||||
FileSnapshot,
|
||||
FileSystemOperations
|
||||
} from "./filesystem-operations";
|
||||
|
||||
const bytes = (text: string) => new TextEncoder().encode(text);
|
||||
const text = (snapshot: FileSnapshot) =>
|
||||
new TextDecoder().decode(snapshot.content);
|
||||
const parent = (path: string) => path.split("/").slice(0, -1).join("/");
|
||||
|
||||
/** Atomic mutations with an injectable interruption before or after durability. */
|
||||
class MemoryFiles implements FileSystemOperations {
|
||||
public readonly files = new Map<string, FileSnapshot>();
|
||||
public readonly directories = new Set([""]);
|
||||
public boundary: (label: string) => void = () => {};
|
||||
|
||||
public change(label: string, action: () => void): void {
|
||||
this.boundary(`before ${label}`);
|
||||
action();
|
||||
this.boundary(`after ${label}`);
|
||||
}
|
||||
|
||||
public put(path: string, value: string): void {
|
||||
for (
|
||||
let directory = parent(path);
|
||||
directory;
|
||||
directory = parent(directory)
|
||||
)
|
||||
this.directories.add(directory);
|
||||
this.files.set(path, { content: bytes(value) });
|
||||
}
|
||||
|
||||
public async listFilesRecursively(root = ""): Promise<string[]> {
|
||||
assert(this.directories.has(root), `Cannot list non-directory ${root}`);
|
||||
return [...this.files.keys()].filter(
|
||||
(path) => !root || path.startsWith(`${root}/`)
|
||||
);
|
||||
}
|
||||
|
||||
public async readSnapshot(path: string): Promise<FileSnapshot | undefined> {
|
||||
assert(!this.directories.has(path), `Cannot read directory ${path}`);
|
||||
return structuredClone(this.files.get(path));
|
||||
}
|
||||
|
||||
public async stat(path: string) {
|
||||
for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor))
|
||||
assert(
|
||||
!this.files.has(ancestor),
|
||||
`Cannot traverse file ${ancestor}`
|
||||
);
|
||||
const snapshot = this.files.get(path);
|
||||
if (snapshot)
|
||||
return { kind: "file" as const, size: snapshot.content.length };
|
||||
if (this.directories.has(path))
|
||||
return { kind: "directory" as const, size: 0 };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
public async exists(path: string): Promise<boolean> {
|
||||
return this.files.has(path) || this.directories.has(path);
|
||||
}
|
||||
|
||||
public async createDirectory(path: string): Promise<void> {
|
||||
this.change(`mkdir ${path}`, () => {
|
||||
for (
|
||||
let directory = path;
|
||||
directory;
|
||||
directory = parent(directory)
|
||||
) {
|
||||
assert(
|
||||
!this.files.has(directory),
|
||||
`File blocks directory ${directory}`
|
||||
);
|
||||
this.directories.add(directory);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async write(path: string, snapshot: FileSnapshot): Promise<void> {
|
||||
this.change(`write ${path}`, () => {
|
||||
assert(
|
||||
!this.files.has(path) && !this.directories.has(path),
|
||||
`Destination exists: ${path}`
|
||||
);
|
||||
assert(
|
||||
this.directories.has(parent(path)),
|
||||
`Parent missing: ${path}`
|
||||
);
|
||||
this.files.set(path, structuredClone(snapshot));
|
||||
});
|
||||
}
|
||||
|
||||
public async rename(from: string, to: string): Promise<void> {
|
||||
this.change(`rename ${from} -> ${to}`, () => {
|
||||
assert(this.files.has(from), `Source missing: ${from}`);
|
||||
assert(
|
||||
!this.files.has(to) && !this.directories.has(to),
|
||||
`Destination exists: ${to}`
|
||||
);
|
||||
assert(this.directories.has(parent(to)), `Parent missing: ${to}`);
|
||||
this.files.set(to, this.files.get(from)!);
|
||||
this.files.delete(from);
|
||||
});
|
||||
}
|
||||
|
||||
public async flushPaths(paths: readonly string[]): Promise<void> {
|
||||
this.change(`flush ${paths.join(",")}`, () => {});
|
||||
}
|
||||
|
||||
public async delete(path: string): Promise<void> {
|
||||
this.change(`rmdir ${path}`, () => {
|
||||
assert(!this.files.has(path));
|
||||
assert(
|
||||
![...this.files.keys()].some((file) =>
|
||||
file.startsWith(`${path}/`)
|
||||
)
|
||||
);
|
||||
for (const directory of this.directories)
|
||||
if (directory === path || directory.startsWith(`${path}/`))
|
||||
this.directories.delete(directory);
|
||||
});
|
||||
}
|
||||
|
||||
public async deleteFile(path: string): Promise<void> {
|
||||
this.change(`unlink ${path}`, () => {
|
||||
assert(!this.directories.has(path));
|
||||
this.files.delete(path);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function setup(
|
||||
contents: Record<string, string>,
|
||||
local: Record<string, string>
|
||||
) {
|
||||
const fs = new MemoryFiles();
|
||||
for (const [path, value] of Object.entries(contents)) fs.put(path, value);
|
||||
let durable: StoredDatabase = emptyState("test");
|
||||
durable.local = local;
|
||||
for (const [id, path] of Object.entries(local))
|
||||
durable.documents[id] = {
|
||||
materialized: true,
|
||||
observedHash: (await toStoredSnapshot(fs.files.get(path)!)).hash
|
||||
};
|
||||
const config = {
|
||||
getConfig: async () => ({ mergeableFileExtensions: ["md"] })
|
||||
} as ServerConfig;
|
||||
const restart = () => {
|
||||
const database = new Database(
|
||||
new Logger(),
|
||||
structuredClone(durable),
|
||||
async (next) => {
|
||||
fs.change("save", () => {
|
||||
durable = structuredClone(next);
|
||||
});
|
||||
},
|
||||
"test",
|
||||
async () => structuredClone(durable)
|
||||
);
|
||||
return {
|
||||
database,
|
||||
operations: new FileOperations(fs, database, config)
|
||||
};
|
||||
};
|
||||
return { fs, restart, ...restart() };
|
||||
}
|
||||
|
||||
async function scenario() {
|
||||
const fixture = await setup(
|
||||
{
|
||||
"a.md": "A",
|
||||
"b.md": "B",
|
||||
"c.md": "C",
|
||||
"d.md": "Unmanaged",
|
||||
"e.md": "E",
|
||||
"folder/keep.md": "Keep",
|
||||
"deleted.md": "Deleted"
|
||||
},
|
||||
{ a: "a.md", b: "b.md", c: "c.md", e: "e.md", deleted: "deleted.md" }
|
||||
);
|
||||
const next = structuredClone(fixture.database.state);
|
||||
next.local = {
|
||||
a: "b.md",
|
||||
b: "a.md",
|
||||
c: "d.md",
|
||||
e: "folder",
|
||||
added: "new.md"
|
||||
};
|
||||
next.documents.added = { materialized: false };
|
||||
const writes: Record<string, FileWrite> = {
|
||||
a: {
|
||||
expected: await fixture.operations.snapshot("a.md"),
|
||||
replacement: await toStoredSnapshot({ content: bytes("A updated") })
|
||||
},
|
||||
added: {
|
||||
replacement: await toStoredSnapshot({
|
||||
content: bytes("Downloaded")
|
||||
})
|
||||
}
|
||||
};
|
||||
return { ...fixture, next, writes };
|
||||
}
|
||||
|
||||
async function verifyScenario(fs: MemoryFiles, database: Database) {
|
||||
assert.equal(database.state.application, undefined);
|
||||
for (const [id, expected] of Object.entries({
|
||||
a: "A updated",
|
||||
b: "B",
|
||||
c: "C",
|
||||
e: "E",
|
||||
added: "Downloaded"
|
||||
})) {
|
||||
const snapshot = fs.files.get(database.state.local[id])!;
|
||||
assert.equal(text(snapshot), expected, id);
|
||||
assert.equal(database.state.documents[id].materialized, true, id);
|
||||
assert.equal(
|
||||
database.state.documents[id].observedHash,
|
||||
(await toStoredSnapshot(snapshot)).hash,
|
||||
id
|
||||
);
|
||||
}
|
||||
assert.equal(database.state.local.a, "b.md");
|
||||
assert.equal(database.state.local.b, "a.md");
|
||||
assert.equal(database.state.local.c, "d.md");
|
||||
assert.notEqual(database.state.local.e, "folder");
|
||||
assert.equal(text(fs.files.get("folder/keep.md")!), "Keep");
|
||||
assert.equal(fs.files.has("deleted.md"), false);
|
||||
assert.equal(database.state.local.deleted, undefined);
|
||||
const visible = [...fs.files].filter(([path]) => !isInternalPath(path));
|
||||
assert.deepEqual(
|
||||
visible.map(([, value]) => text(value)).sort(),
|
||||
["A updated", "B", "C", "Downloaded", "E", "Keep", "Unmanaged"].sort()
|
||||
);
|
||||
assert.equal(
|
||||
[...fs.files.keys()].some((path) =>
|
||||
path.startsWith(".vault-link-sync/transactions/")
|
||||
),
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
describe("filesystem journal recovery", () => {
|
||||
it("stages swaps, preserves unexpected files/directories, and cleans recovery artifacts", async () => {
|
||||
const { operations, database, fs, next, writes } = await scenario();
|
||||
await operations.apply(next, writes);
|
||||
await verifyScenario(fs, database);
|
||||
fs.boundary = () => {
|
||||
throw new Error("Completed recovery must not write");
|
||||
};
|
||||
await operations.recover();
|
||||
});
|
||||
|
||||
it("resumes after interruption at every filesystem and metadata mutation boundary", async (t) => {
|
||||
const baseline = await scenario();
|
||||
const boundaries: string[] = [];
|
||||
baseline.fs.boundary = (label) => boundaries.push(label);
|
||||
await baseline.operations.apply(baseline.next, baseline.writes);
|
||||
for (const [index, label] of boundaries.entries()) {
|
||||
await t.test(`${index}: ${label}`, async () => {
|
||||
const fixture = await scenario();
|
||||
const interrupted = new Error("Interrupted");
|
||||
let count = 0;
|
||||
fixture.fs.boundary = () => {
|
||||
if (count++ === index) throw interrupted;
|
||||
};
|
||||
await assert.rejects(
|
||||
fixture.operations.apply(fixture.next, fixture.writes),
|
||||
(error) => error === interrupted
|
||||
);
|
||||
fixture.fs.boundary = () => {};
|
||||
const { database, operations } = fixture.restart();
|
||||
if (
|
||||
!database.state.application &&
|
||||
database.state.local.a === "a.md"
|
||||
)
|
||||
await operations.apply(
|
||||
structuredClone(fixture.next),
|
||||
fixture.writes
|
||||
);
|
||||
else await operations.recover();
|
||||
await verifyScenario(fixture.fs, database);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("merges an edit made after planning exactly once across an interrupted install", async () => {
|
||||
const base = "First paragraph.\n\nLast paragraph.\n";
|
||||
const local = "Local first paragraph.\n\nLast paragraph.\n";
|
||||
const remote = "First paragraph.\n\nRemote last paragraph.\n";
|
||||
const fixture = await setup({ "note.md": base }, { note: "note.md" });
|
||||
const expected = await fixture.operations.snapshot("note.md");
|
||||
let edited = false;
|
||||
const interrupted = new Error("Interrupted after install");
|
||||
fixture.fs.boundary = (label) => {
|
||||
if (!edited && label === "after save") {
|
||||
fixture.fs.put("note.md", local);
|
||||
edited = true;
|
||||
}
|
||||
if (
|
||||
label.startsWith("after rename ") &&
|
||||
label.endsWith(" -> note.md")
|
||||
)
|
||||
throw interrupted;
|
||||
};
|
||||
await assert.rejects(
|
||||
fixture.operations.apply(structuredClone(fixture.database.state), {
|
||||
note: {
|
||||
expected,
|
||||
replacement: await toStoredSnapshot({
|
||||
content: bytes(remote)
|
||||
})
|
||||
}
|
||||
}),
|
||||
(error) => error === interrupted
|
||||
);
|
||||
fixture.fs.boundary = () => {};
|
||||
const { operations, database } = fixture.restart();
|
||||
await operations.recover();
|
||||
assert.equal(
|
||||
text(fixture.fs.files.get("note.md")!),
|
||||
"Local first paragraph.\n\nRemote last paragraph.\n"
|
||||
);
|
||||
assert.equal(
|
||||
database.state.documents.note.observedHash,
|
||||
(await operations.snapshot("note.md"))!.hash
|
||||
);
|
||||
assert.equal(
|
||||
[...fixture.fs.files.keys()].some((path) =>
|
||||
path.startsWith(".vault-link-sync/transactions/")
|
||||
),
|
||||
false
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves an external deletion between planning and staging", async () => {
|
||||
const fixture = await setup(
|
||||
{ "note.md": "Local" },
|
||||
{ note: "note.md" }
|
||||
);
|
||||
const expected = await fixture.operations.snapshot("note.md");
|
||||
fixture.fs.boundary = (label) => {
|
||||
if (label === "after save") fixture.fs.files.delete("note.md");
|
||||
};
|
||||
await fixture.operations.apply(
|
||||
structuredClone(fixture.database.state),
|
||||
{
|
||||
note: {
|
||||
expected,
|
||||
replacement: await toStoredSnapshot({
|
||||
content: bytes("Remote")
|
||||
})
|
||||
}
|
||||
}
|
||||
);
|
||||
assert.equal(fixture.database.state.local.note, undefined);
|
||||
assert.equal(fixture.fs.files.has("note.md"), false);
|
||||
});
|
||||
|
||||
it("preserves a file created after the destination check", async () => {
|
||||
const fixture = await setup(
|
||||
{ "source.md": "Incoming" },
|
||||
{ doc: "source.md" }
|
||||
);
|
||||
const next = structuredClone(fixture.database.state);
|
||||
next.local.doc = "target.md";
|
||||
fixture.fs.boundary = (label) => {
|
||||
if (
|
||||
label.startsWith("before rename ") &&
|
||||
label.endsWith(" -> target.md")
|
||||
)
|
||||
fixture.fs.put("target.md", "Concurrent");
|
||||
};
|
||||
await assert.rejects(
|
||||
fixture.operations.apply(next),
|
||||
/Destination exists/
|
||||
);
|
||||
fixture.fs.boundary = () => {};
|
||||
const { operations, database } = fixture.restart();
|
||||
await operations.recover();
|
||||
assert.equal(text(fixture.fs.files.get("target.md")!), "Incoming");
|
||||
const displaced = Object.entries(database.state.local).find(
|
||||
([id]) => id !== "doc"
|
||||
)!;
|
||||
assert.equal(text(fixture.fs.files.get(displaced[1])!), "Concurrent");
|
||||
});
|
||||
|
||||
it("preserves an unexpected file blocking a destination's ancestor", async () => {
|
||||
const fixture = await setup(
|
||||
{ "source.md": "Incoming", target: "Occupant" },
|
||||
{ doc: "source.md" }
|
||||
);
|
||||
const next = structuredClone(fixture.database.state);
|
||||
next.local.doc = "target/child.md";
|
||||
await fixture.operations.apply(next);
|
||||
assert.equal(
|
||||
text(fixture.fs.files.get("target/child.md")!),
|
||||
"Incoming"
|
||||
);
|
||||
const displaced = Object.entries(fixture.database.state.local).find(
|
||||
([id]) => id !== "doc"
|
||||
)!;
|
||||
assert.equal(text(fixture.fs.files.get(displaced[1])!), "Occupant");
|
||||
});
|
||||
|
||||
it("handles paths becoming directories and directories becoming files", async () => {
|
||||
const fixture = await setup(
|
||||
{ a: "A", "b/deep/child": "B" },
|
||||
{ a: "a", b: "b/deep/child" }
|
||||
);
|
||||
const next = structuredClone(fixture.database.state);
|
||||
next.local = { a: "b", b: "a/deep/child" };
|
||||
await fixture.operations.apply(next);
|
||||
assert.equal(text(fixture.fs.files.get("b")!), "A");
|
||||
assert.equal(text(fixture.fs.files.get("a/deep/child")!), "B");
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,37 +1,45 @@
|
|||
import type { RelativePath } from "../persistence/database";
|
||||
import type { CursorPosition } from "reconcile-text";
|
||||
|
||||
import type { TextWithCursors } from "reconcile-text";
|
||||
|
||||
export interface FileSnapshot {
|
||||
content: Uint8Array;
|
||||
/** Optional editor selections, moved with the file and applied with its bytes. */
|
||||
cursors?: CursorPosition[];
|
||||
}
|
||||
|
||||
/** Required adapter contract for API v4; there is deliberately no weak fallback.
|
||||
* Paths are vault-relative. Never follow symlinks (including ancestor symlinks).
|
||||
* Only independent regular files are supported; reject hardlinks/special files.
|
||||
* Every mutation must flush file data and affected directories before resolving.
|
||||
* A successful scan is complete: unreadable directories must reject, not vanish.
|
||||
*/
|
||||
export interface FileSystemOperations {
|
||||
// List all files under root that should be synced. If root is undefined, return every file.
|
||||
listFilesRecursively: (
|
||||
root: RelativePath | undefined
|
||||
) => Promise<RelativePath[]>;
|
||||
|
||||
// Read the content of a file.
|
||||
read: (path: RelativePath) => Promise<Uint8Array>;
|
||||
|
||||
// Create or overwrite a file with the given content.
|
||||
write: (path: RelativePath, content: Uint8Array) => Promise<void>;
|
||||
|
||||
// Atomically update the content of a text file.
|
||||
atomicUpdateText: (
|
||||
path: RelativePath,
|
||||
updater: (current: TextWithCursors) => TextWithCursors
|
||||
) => Promise<string>;
|
||||
|
||||
// Get the size of a file in bytes.
|
||||
getFileSize: (path: RelativePath) => Promise<number>;
|
||||
|
||||
// Check if a file exists.
|
||||
exists: (path: RelativePath) => Promise<boolean>;
|
||||
|
||||
// Create a directory at the specified path. All parent directories must already exist.
|
||||
createDirectory: (path: RelativePath) => Promise<void>;
|
||||
|
||||
// Delete a file. It is expected that the path points to an existing file.
|
||||
delete: (path: RelativePath) => Promise<void>;
|
||||
|
||||
// Rename a file. It is expected that the oldPath points to an existing file and the newPath does not exist.
|
||||
rename: (oldPath: RelativePath, newPath: RelativePath) => Promise<void>;
|
||||
listFilesRecursively: (root?: string) => Promise<string[]>;
|
||||
/** Coherent content snapshot; undefined only for an absent file. Reject read races. */
|
||||
readSnapshot: (path: string) => Promise<FileSnapshot | undefined>;
|
||||
/** Reject symlinks and special files rather than following them. */
|
||||
stat: (
|
||||
path: string
|
||||
) => Promise<{ kind: "file" | "directory"; size: number } | undefined>;
|
||||
/** Includes directories. */
|
||||
exists: (path: string) => Promise<boolean>;
|
||||
/** Recursive, idempotent, durable directory creation. */
|
||||
createDirectory: (path: string) => Promise<void>;
|
||||
/** Atomically create a complete file: never replace an existing object.
|
||||
* A crash leaves the path absent or complete, never a partial file.
|
||||
* Flush data and directory entries before resolving. */
|
||||
write: (path: string, snapshot: FileSnapshot) => Promise<void>;
|
||||
/** Atomic, durable rename on the same filesystem; fail if destination exists.
|
||||
* Flush the source's data and both affected directories before resolving. */
|
||||
rename: (from: string, to: string) => Promise<void>;
|
||||
/** Flush existing files and all affected ancestor directories, even when a
|
||||
* named file is absent. Recovery uses this after a mutation became visible
|
||||
* before its caller learned that it was durable. */
|
||||
flushPaths: (paths: readonly string[]) => Promise<void>;
|
||||
/** Idempotently unlink one regular file and flush its parent directory.
|
||||
* The sync engine uses this only for its own internal recovery artifacts. */
|
||||
deleteFile: (path: string) => Promise<void>;
|
||||
/** Prune an implicit directory tree using rmdir only, durably. Reject
|
||||
* regular files and never unlink descendant files; reject if files remain.
|
||||
* Partial removal of empty child directories on failure is harmless. */
|
||||
delete: (path: string) => Promise<void>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,374 +1,235 @@
|
|||
import type { CursorPosition } from "reconcile-text";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import { EMPTY_HASH } from "../utils/hash";
|
||||
import { CoveredValues } from "../utils/data-structures/min-covered";
|
||||
import { awaitAll } from "../utils/await-all";
|
||||
import { removeFromArray } from "../utils/remove-from-array";
|
||||
import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse";
|
||||
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
|
||||
import type { FileManifest } from "../services/types/FileManifest";
|
||||
import type { FileManifestUpdateResponse } from "../services/types/FileManifestUpdateResponse";
|
||||
import type { PushFileManifest } from "../services/types/PushFileManifest";
|
||||
import type { PutFileContent } from "../services/types/PutFileContent";
|
||||
import type { VaultSnapshot } from "../services/types/VaultSnapshot";
|
||||
import type { EventRecord } from "../services/types/EventRecord";
|
||||
import type { FileManifestEntries } from "../types/file-manifest-entries";
|
||||
|
||||
export type VaultUpdateId = number;
|
||||
export type DocumentId = string;
|
||||
export type RelativePath = string;
|
||||
|
||||
export interface StoredSnapshot {
|
||||
contentBase64: string;
|
||||
hash: string;
|
||||
cursors?: CursorPosition[];
|
||||
}
|
||||
|
||||
export interface DocumentMetadata {
|
||||
parentVersionId: VaultUpdateId;
|
||||
parentVersionId: number;
|
||||
hash: string;
|
||||
remoteRelativePath?: RelativePath;
|
||||
remoteRelativePath?: string;
|
||||
}
|
||||
|
||||
export interface StoredDocumentMetadata {
|
||||
relativePath: RelativePath;
|
||||
documentId: DocumentId;
|
||||
parentVersionId: VaultUpdateId;
|
||||
remoteRelativePath?: RelativePath;
|
||||
hash: string;
|
||||
export interface DocumentState {
|
||||
/** Local clean base retained while rebasing after a server restore. */
|
||||
recoveryBase?: StoredSnapshot;
|
||||
base?: DocumentVersionWithoutContent & { hash: string };
|
||||
observedHash?: string;
|
||||
materialized: boolean;
|
||||
/** First contact with existing local content: there is no common ancestor. */
|
||||
bootstrap?: boolean;
|
||||
/** Do not resubmit an unchanged payload rejected by the server. */
|
||||
rejected?: { hash: string; message: string };
|
||||
}
|
||||
|
||||
export interface StoredDatabase {
|
||||
documents: StoredDocumentMetadata[];
|
||||
lastSeenUpdateId: VaultUpdateId | undefined;
|
||||
hasInitialSyncCompleted: boolean;
|
||||
export type PendingRequest = { rejection?: string } & (
|
||||
| {
|
||||
type: "content";
|
||||
documentId: string;
|
||||
request: PutFileContent;
|
||||
snapshot: StoredSnapshot;
|
||||
response?: DocumentUpdateResponse;
|
||||
}
|
||||
| {
|
||||
type: "fileManifest";
|
||||
request: PushFileManifest;
|
||||
response?: FileManifestUpdateResponse;
|
||||
}
|
||||
);
|
||||
|
||||
export interface EngineState {
|
||||
/** Durable reset intent; clear the transport checkpoint before bootstrap. */
|
||||
historyRecovery?: boolean;
|
||||
vaultKey: string;
|
||||
initialized: boolean;
|
||||
fileManifest: FileManifest;
|
||||
local: FileManifestEntries;
|
||||
documents: Record<string, DocumentState>;
|
||||
lastSeenUpdateId: number;
|
||||
remoteHeads: Record<string, DocumentVersionWithoutContent>;
|
||||
bootstrap?: VaultSnapshot;
|
||||
/** Fold pages durably without applying intermediate remote snapshots. */
|
||||
eventReplay?: {
|
||||
after: number;
|
||||
manifest: FileManifest;
|
||||
heads: Record<string, DocumentVersionWithoutContent>;
|
||||
receipts: EventRecord[];
|
||||
};
|
||||
pending?: PendingRequest;
|
||||
/** A rejected retry does not prove an earlier attempt failed to commit. */
|
||||
unconfirmed?: PendingRequest[];
|
||||
/** Commits consumption of the durable notification queue with the identity map. */
|
||||
lastAppliedLocalChangeId?: string;
|
||||
/** Physical exclusions, including files with no sync identity. */
|
||||
protectedPaths?: string[];
|
||||
/** Original server paths while local files are excluded; null means absent. */
|
||||
excluded?: Record<string, string | null>;
|
||||
rejectedManifest?: { entries: FileManifestEntries; message: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a document in the database.
|
||||
*
|
||||
* It is mutable and its content should always represent the latest
|
||||
* state of the document on disk based on the update events we have seen.
|
||||
*/
|
||||
export interface FileStep {
|
||||
documentId: string;
|
||||
from?: string;
|
||||
to?: string;
|
||||
expected?: StoredSnapshot;
|
||||
replacement?: StoredSnapshot;
|
||||
staged: string;
|
||||
output: string;
|
||||
/** A permanent destination error has already selected a root fallback. */
|
||||
replanned?: boolean;
|
||||
/** Retained input of a later step for this identity; never projected as live. */
|
||||
superseded?: boolean;
|
||||
phase: "planned" | "staged" | "prepared" | "installing" | "installed";
|
||||
}
|
||||
|
||||
export interface ApplicationJournal {
|
||||
id: string;
|
||||
extensions: string[];
|
||||
next: EngineState;
|
||||
steps: FileStep[];
|
||||
}
|
||||
|
||||
export interface StoredDatabase extends EngineState {
|
||||
application?: ApplicationJournal;
|
||||
}
|
||||
|
||||
// Read-only view used by cursor tracking and status reporting.
|
||||
export interface DocumentRecord {
|
||||
relativePath: RelativePath;
|
||||
documentId: DocumentId;
|
||||
metadata: DocumentMetadata | undefined;
|
||||
documentId: string;
|
||||
relativePath: string;
|
||||
metadata?: DocumentMetadata;
|
||||
isDeleted: boolean;
|
||||
updates: Promise<unknown>[];
|
||||
parallelVersion: number;
|
||||
}
|
||||
|
||||
export function emptyState(vaultKey: string): StoredDatabase {
|
||||
return {
|
||||
vaultKey,
|
||||
initialized: false,
|
||||
fileManifest: { fileManifestId: 0, entries: {} },
|
||||
local: {},
|
||||
documents: {},
|
||||
remoteHeads: {},
|
||||
lastSeenUpdateId: 0
|
||||
};
|
||||
}
|
||||
|
||||
function canRebindEmpty(state: Partial<StoredDatabase> | undefined): boolean {
|
||||
return (
|
||||
state !== undefined &&
|
||||
!state.initialized &&
|
||||
!state.pending &&
|
||||
!state.application &&
|
||||
!state.bootstrap &&
|
||||
!state.eventReplay &&
|
||||
(state.unconfirmed?.length ?? 0) === 0 &&
|
||||
Object.keys(state.local ?? {}).length === 0
|
||||
);
|
||||
}
|
||||
|
||||
export class Database {
|
||||
private documents: DocumentRecord[];
|
||||
private lastSeenUpdateIds: CoveredValues;
|
||||
private hasInitialSyncCompleted: boolean;
|
||||
|
||||
public state: StoredDatabase;
|
||||
private needsReload = false;
|
||||
public constructor(
|
||||
private readonly logger: Logger,
|
||||
initialState: Partial<StoredDatabase> | undefined,
|
||||
private readonly saveData: (data: StoredDatabase) => Promise<void>
|
||||
initial: Partial<StoredDatabase> | undefined,
|
||||
private readonly saveData: (data: StoredDatabase) => Promise<void>,
|
||||
vaultKey: string,
|
||||
private readonly loadData: () => Promise<StoredDatabase | undefined>
|
||||
) {
|
||||
initialState ??= {};
|
||||
|
||||
this.documents =
|
||||
initialState.documents?.map(
|
||||
({ relativePath, documentId, ...metadata }) => ({
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata,
|
||||
isDeleted: false,
|
||||
updates: [],
|
||||
parallelVersion: 0
|
||||
})
|
||||
) ?? [];
|
||||
|
||||
this.ensureConsistency();
|
||||
this.logger.debug(`Loaded ${this.documents.length} documents`);
|
||||
|
||||
const { lastSeenUpdateId } = initialState;
|
||||
this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`);
|
||||
this.lastSeenUpdateIds = new CoveredValues(
|
||||
Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1
|
||||
);
|
||||
|
||||
this.documents.forEach((doc) => {
|
||||
this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId);
|
||||
});
|
||||
|
||||
this.hasInitialSyncCompleted =
|
||||
initialState.hasInitialSyncCompleted ?? false;
|
||||
this.logger.debug(
|
||||
`Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
|
||||
);
|
||||
}
|
||||
|
||||
public get length(): number {
|
||||
return this.documents.length;
|
||||
}
|
||||
|
||||
public get resolvedDocuments(): DocumentRecord[] {
|
||||
const paths = new Map<string, DocumentRecord[]>();
|
||||
this.documents
|
||||
// eslint-disable-next-line no-restricted-syntax -- Type narrowing, not removing a specific item
|
||||
.filter(({ metadata }) => metadata !== undefined)
|
||||
.forEach((record) =>
|
||||
paths.set(record.relativePath, [
|
||||
record,
|
||||
...(paths.get(record.relativePath) ?? [])
|
||||
])
|
||||
);
|
||||
|
||||
return Array.from(paths.values()).map((records) => {
|
||||
records.sort(
|
||||
(a, b) => b.parallelVersion - a.parallelVersion // descending
|
||||
);
|
||||
|
||||
if (
|
||||
records.length > 1 &&
|
||||
records.some((current, i) =>
|
||||
i === 0
|
||||
? false
|
||||
: records[i - 1].parallelVersion ===
|
||||
current.parallelVersion
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
`Multiple documents with the same parallel version and path at ${records[0].relativePath}`
|
||||
);
|
||||
}
|
||||
return records[0];
|
||||
});
|
||||
}
|
||||
|
||||
public updateDocumentMetadata(
|
||||
metadata: {
|
||||
parentVersionId: VaultUpdateId;
|
||||
hash: string;
|
||||
remoteRelativePath: RelativePath;
|
||||
},
|
||||
toUpdate: DocumentRecord
|
||||
): void {
|
||||
if (!this.documents.includes(toUpdate)) {
|
||||
throw new Error("Document not found in database");
|
||||
}
|
||||
|
||||
toUpdate.metadata = metadata;
|
||||
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public removeDocumentPromise(promise: Promise<unknown>): void {
|
||||
const entry = this.documents.find(({ updates }) =>
|
||||
updates.includes(promise)
|
||||
);
|
||||
|
||||
if (entry === undefined) {
|
||||
// This method should be idempotent and tolerant of
|
||||
// stragglers calling it after the databse has been reset.
|
||||
return;
|
||||
}
|
||||
|
||||
removeFromArray(entry.updates, promise);
|
||||
// No need to save as Promises don't get serialized
|
||||
}
|
||||
|
||||
public removeDocument(find: DocumentRecord): void {
|
||||
removeFromArray(this.documents, find);
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public getLatestDocumentByRelativePath(
|
||||
find: RelativePath
|
||||
): DocumentRecord | undefined {
|
||||
const candidates = this.documents.filter(
|
||||
({ relativePath }) => relativePath === find
|
||||
);
|
||||
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
public async getResolvedDocumentByRelativePath(
|
||||
relativePath: RelativePath,
|
||||
promise: Promise<unknown>
|
||||
): Promise<DocumentRecord> {
|
||||
const entry = this.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
if (entry === undefined) {
|
||||
const hasState = initial && Object.keys(initial).length > 0;
|
||||
if (
|
||||
hasState &&
|
||||
initial.vaultKey !== vaultKey &&
|
||||
!canRebindEmpty(initial)
|
||||
) {
|
||||
throw new Error(
|
||||
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
|
||||
this.documents,
|
||||
null,
|
||||
2
|
||||
)}`
|
||||
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
|
||||
);
|
||||
}
|
||||
|
||||
const currentPromises = entry.updates;
|
||||
entry.updates = [...currentPromises, promise];
|
||||
await awaitAll(currentPromises);
|
||||
|
||||
return entry;
|
||||
this.state = hasState
|
||||
? { ...(initial as StoredDatabase), vaultKey }
|
||||
: emptyState(vaultKey);
|
||||
}
|
||||
|
||||
public createNewPendingDocument(
|
||||
documentId: DocumentId,
|
||||
relativePath: RelativePath,
|
||||
promise: Promise<unknown>
|
||||
): DocumentRecord {
|
||||
this.logger.debug(
|
||||
`Creating new pending document: ${relativePath} (${documentId})`
|
||||
);
|
||||
const previousEntry =
|
||||
this.getLatestDocumentByRelativePath(relativePath);
|
||||
|
||||
const entry = {
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata: undefined,
|
||||
isDeleted: false,
|
||||
updates: [promise],
|
||||
parallelVersion:
|
||||
previousEntry?.parallelVersion === undefined
|
||||
? 0
|
||||
: previousEntry.parallelVersion + 1
|
||||
};
|
||||
|
||||
this.documents.push(entry);
|
||||
this.saveInTheBackground();
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
public createNewEmptyDocument(
|
||||
documentId: DocumentId,
|
||||
parentVersionId: VaultUpdateId,
|
||||
relativePath: RelativePath
|
||||
): DocumentRecord {
|
||||
const entry = {
|
||||
relativePath,
|
||||
documentId,
|
||||
metadata: {
|
||||
parentVersionId,
|
||||
hash: EMPTY_HASH,
|
||||
remoteRelativePath: relativePath
|
||||
},
|
||||
isDeleted: false,
|
||||
updates: [],
|
||||
parallelVersion: 0
|
||||
};
|
||||
|
||||
this.documents.push(entry);
|
||||
this.saveInTheBackground();
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
public getDocumentByDocumentId(
|
||||
find: DocumentId
|
||||
): DocumentRecord | undefined {
|
||||
return this.documents.find(({ documentId }) => documentId === find);
|
||||
}
|
||||
|
||||
public move(
|
||||
oldRelativePath: RelativePath,
|
||||
newRelativePath: RelativePath
|
||||
): void {
|
||||
const oldDocument =
|
||||
this.getLatestDocumentByRelativePath(oldRelativePath);
|
||||
|
||||
if (oldDocument === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
const newDocument =
|
||||
this.getLatestDocumentByRelativePath(newRelativePath);
|
||||
if (newDocument?.isDeleted === false) {
|
||||
/** Bind an empty database to the configured vault. */
|
||||
public async bindVault(vaultKey: string): Promise<void> {
|
||||
if (this.state.vaultKey === vaultKey) return;
|
||||
if (!canRebindEmpty(this.state))
|
||||
throw new Error(
|
||||
`Document already exists at new location: ${newRelativePath}`
|
||||
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
|
||||
);
|
||||
}
|
||||
|
||||
oldDocument.relativePath = newRelativePath;
|
||||
// We're in a strange state where the target of the move has just got deleted,
|
||||
// however, its metadata might already have a bunch of updates queued up for
|
||||
// the document at the new location. We need to keep these updates.
|
||||
oldDocument.parallelVersion =
|
||||
newDocument !== undefined ? newDocument.parallelVersion + 1 : 0;
|
||||
|
||||
this.saveInTheBackground();
|
||||
await this.commit({ ...this.state, vaultKey });
|
||||
}
|
||||
|
||||
public delete(relativePath: RelativePath): void {
|
||||
const candidate = this.getLatestDocumentByRelativePath(relativePath);
|
||||
if (candidate === undefined) {
|
||||
public async commit(next: StoredDatabase): Promise<void> {
|
||||
const snapshot = structuredClone(next);
|
||||
if (this.needsReload)
|
||||
throw new Error(
|
||||
`Document not found by relative path: ${relativePath}`
|
||||
"Reload durable state before retrying an uncertain save"
|
||||
);
|
||||
}
|
||||
candidate.isDeleted = true;
|
||||
this.needsReload = true;
|
||||
await this.saveData(snapshot);
|
||||
this.state = snapshot;
|
||||
this.needsReload = false;
|
||||
}
|
||||
|
||||
public getHasInitialSyncCompleted(): boolean {
|
||||
return this.hasInitialSyncCompleted;
|
||||
}
|
||||
|
||||
public setHasInitialSyncCompleted(value: boolean): void {
|
||||
this.hasInitialSyncCompleted = value;
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public getLastSeenUpdateId(): VaultUpdateId {
|
||||
return this.lastSeenUpdateIds.min;
|
||||
}
|
||||
|
||||
public addSeenUpdateId(value: number): void {
|
||||
const previousMin = this.lastSeenUpdateIds.min;
|
||||
this.lastSeenUpdateIds.add(value);
|
||||
if (previousMin !== this.lastSeenUpdateIds.min) {
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
}
|
||||
|
||||
public setLastSeenUpdateId(value: number): void {
|
||||
this.lastSeenUpdateIds.min = value;
|
||||
this.saveInTheBackground();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this.documents = [];
|
||||
this.lastSeenUpdateIds = new CoveredValues(
|
||||
0 // the first updateId will be 1 which is the first integer after -1
|
||||
);
|
||||
this.hasInitialSyncCompleted = false;
|
||||
this.saveInTheBackground();
|
||||
public async recoverPersistence(): Promise<void> {
|
||||
if (!this.needsReload) return;
|
||||
const saved = await this.loadData();
|
||||
if (saved) this.state = saved;
|
||||
this.needsReload = false;
|
||||
}
|
||||
|
||||
public async save(): Promise<void> {
|
||||
return this.saveData({
|
||||
documents: this.resolvedDocuments.map(
|
||||
({ relativePath, documentId, metadata }) => ({
|
||||
documentId,
|
||||
relativePath,
|
||||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
||||
...metadata! // `resolvedDocuments` only returns docs with metadata set
|
||||
})
|
||||
),
|
||||
lastSeenUpdateId: this.lastSeenUpdateIds.min,
|
||||
hasInitialSyncCompleted: this.hasInitialSyncCompleted
|
||||
});
|
||||
await this.commit(this.state);
|
||||
}
|
||||
|
||||
private ensureConsistency(): void {
|
||||
const idToPath = new Map<string, string[]>();
|
||||
|
||||
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
|
||||
idToPath.set(documentId, [
|
||||
...(idToPath.get(documentId) ?? []),
|
||||
relativePath
|
||||
]);
|
||||
});
|
||||
|
||||
const duplicates = Array.from(idToPath.entries())
|
||||
.filter(([_, paths]) => paths.length > 1)
|
||||
.map(([id, paths]) => `${id} (${paths.join(", ")})`);
|
||||
|
||||
if (duplicates.length > 0) {
|
||||
throw new Error(
|
||||
"Document IDs are not unique, found duplicates: " +
|
||||
duplicates.join("; ")
|
||||
);
|
||||
}
|
||||
public get length(): number {
|
||||
return Object.keys(this.state.local).length;
|
||||
}
|
||||
|
||||
private saveInTheBackground(): void {
|
||||
this.ensureConsistency();
|
||||
void this.save().catch((error: unknown) => {
|
||||
this.logger.error(`Error saving data: ${error}`);
|
||||
});
|
||||
public getLastSeenUpdateId(): number {
|
||||
return this.state.lastSeenUpdateId;
|
||||
}
|
||||
|
||||
public getDocumentByDocumentId(id: string): DocumentRecord | undefined {
|
||||
const relativePath = this.state.local[id];
|
||||
if (relativePath === undefined) return undefined;
|
||||
const base = this.state.documents[id]?.base;
|
||||
return {
|
||||
documentId: id,
|
||||
relativePath,
|
||||
isDeleted: false,
|
||||
metadata: base && {
|
||||
parentVersionId: base.vaultUpdateId,
|
||||
hash: base.hash,
|
||||
remoteRelativePath: this.state.fileManifest.entries[id]
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public getLatestDocumentByRelativePath(
|
||||
path: string
|
||||
): DocumentRecord | undefined {
|
||||
const id = Object.keys(this.state.local).find(
|
||||
(id) => this.state.local[id] === path
|
||||
);
|
||||
return id === undefined ? undefined : this.getDocumentByDocumentId(id);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import type { FetchController } from "./fetch-controller";
|
|||
import type { Logger } from "../tracing/logger";
|
||||
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
|
||||
import type { DocumentVersion } from "./types/DocumentVersion";
|
||||
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
|
||||
import type { EventBatch } from "./types/EventBatch";
|
||||
import type { FileManifest } from "./types/FileManifest";
|
||||
import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse";
|
||||
|
|
@ -10,7 +11,11 @@ import type { PutFileContent } from "./types/PutFileContent";
|
|||
import type { PushFileManifest } from "./types/PushFileManifest";
|
||||
import type { VaultSnapshot } from "./types/VaultSnapshot";
|
||||
import type { PingResponse } from "./types/PingResponse";
|
||||
import { AuthenticationError, PermanentSyncError } from "../errors/errors";
|
||||
import {
|
||||
AuthenticationError,
|
||||
PermanentSyncError,
|
||||
ServerHistoryChangedError
|
||||
} from "../errors/errors";
|
||||
|
||||
export class SyncService {
|
||||
private readonly client: typeof fetch;
|
||||
|
|
@ -21,7 +26,11 @@ export class SyncService {
|
|||
fetchController: FetchController,
|
||||
private readonly settings: Settings,
|
||||
logger: Logger,
|
||||
fetchImplementation: typeof fetch = globalThis.fetch
|
||||
fetchImplementation: typeof fetch = globalThis.fetch,
|
||||
private readonly history?: {
|
||||
get: () => string | undefined;
|
||||
save: (checkpoint: string | undefined) => Promise<void>;
|
||||
}
|
||||
) {
|
||||
this.rawFetch = async (...args) => fetchImplementation(...args);
|
||||
this.client = fetchController.getControlledFetchImplementation(
|
||||
|
|
@ -30,6 +39,42 @@ export class SyncService {
|
|||
);
|
||||
}
|
||||
|
||||
public get verifiesHistory(): boolean {
|
||||
return this.history !== undefined;
|
||||
}
|
||||
public get hasHistoryCheckpoint(): boolean {
|
||||
return this.history?.get() !== undefined;
|
||||
}
|
||||
|
||||
public async resetHistory(): Promise<void> {
|
||||
await this.history?.save(undefined);
|
||||
}
|
||||
|
||||
private historyHeaders(): Record<string, string> {
|
||||
const checkpoint = this.history?.get();
|
||||
return checkpoint ? { "X-Vault-Link-History": checkpoint } : {};
|
||||
}
|
||||
|
||||
private async recordHistory(response: Response): Promise<void> {
|
||||
const checkpoint = response.headers.get("x-vault-link-history");
|
||||
if (checkpoint && this.history) {
|
||||
const previous = this.history.get();
|
||||
if (
|
||||
!previous ||
|
||||
Number(checkpoint.split(":")[0]) >=
|
||||
Number(previous.split(":")[0])
|
||||
)
|
||||
await this.history.save(checkpoint);
|
||||
}
|
||||
}
|
||||
|
||||
private checkHistory(response: Response): void {
|
||||
if (response.headers.get("x-vault-link-history-mismatch") === "1")
|
||||
throw new ServerHistoryChangedError(
|
||||
"Server history changed; recovering local work"
|
||||
);
|
||||
}
|
||||
|
||||
public getUrl(path: string): string {
|
||||
const { remoteUri, vaultName } = this.settings.getSettings();
|
||||
return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`;
|
||||
|
|
@ -49,6 +94,7 @@ export class SyncService {
|
|||
this.settings.getSettings().requestTimeoutMs
|
||||
),
|
||||
headers: {
|
||||
...(raw ? {} : this.historyHeaders()),
|
||||
Authorization: `Bearer ${this.settings.getSettings().token}`,
|
||||
"Device-Id": this.deviceId,
|
||||
"Content-Type": "application/json"
|
||||
|
|
@ -56,6 +102,7 @@ export class SyncService {
|
|||
}
|
||||
);
|
||||
|
||||
this.checkHistory(response);
|
||||
if (!response.ok) {
|
||||
this.throwForErrorResponse(
|
||||
response,
|
||||
|
|
@ -63,13 +110,11 @@ export class SyncService {
|
|||
);
|
||||
}
|
||||
|
||||
if (!raw) await this.recordHistory(response);
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
private throwForErrorResponse(
|
||||
response: Response,
|
||||
message: string
|
||||
): never {
|
||||
private throwForErrorResponse(response: Response, message: string): never {
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw new AuthenticationError(message);
|
||||
}
|
||||
|
|
@ -117,6 +162,10 @@ export class SyncService {
|
|||
return this.request(`/documents/${id}`);
|
||||
}
|
||||
|
||||
public async metadata(id: string): Promise<DocumentVersionWithoutContent> {
|
||||
return this.request(`/documents/${id}/metadata`);
|
||||
}
|
||||
|
||||
public async getDocumentVersionContent({
|
||||
documentId,
|
||||
vaultUpdateId
|
||||
|
|
@ -133,11 +182,13 @@ export class SyncService {
|
|||
this.settings.getSettings().requestTimeoutMs
|
||||
),
|
||||
headers: {
|
||||
...this.historyHeaders(),
|
||||
Authorization: `Bearer ${this.settings.getSettings().token}`
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
this.checkHistory(response);
|
||||
if (!response.ok) {
|
||||
this.throwForErrorResponse(
|
||||
response,
|
||||
|
|
@ -145,6 +196,7 @@ export class SyncService {
|
|||
);
|
||||
}
|
||||
|
||||
await this.recordHistory(response);
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { DocumentVersion } from "./DocumentVersion";
|
||||
import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent";
|
||||
|
||||
/**
|
||||
* Response to an update document request.
|
||||
*/
|
||||
export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersion;
|
||||
export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersionWithoutContent;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
import type { EventRecord } from "./EventRecord";
|
||||
|
||||
export type EventBatch = { headEventId: number, events: Array<EventRecord>, };
|
||||
export type EventBatch = { headEventId: number,
|
||||
/**
|
||||
* Last event in this page; `head_event_id` is the vault's current head.
|
||||
*/
|
||||
endEventId?: number, events: Array<EventRecord>, };
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
|
||||
import { describe, it, beforeEach } from "node:test";
|
||||
import assert from "node:assert";
|
||||
import { createServer } from "node:net";
|
||||
import { WebSocketManager } from "./websocket-manager";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import type { Settings } from "../persistence/settings";
|
||||
import { Logger } from "../tracing/logger";
|
||||
import { Settings } from "../persistence/settings";
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
const WebSocket = require("ws") as typeof globalThis.WebSocket;
|
||||
|
||||
|
|
@ -337,3 +338,179 @@ describe("WebSocketManager", () => {
|
|||
await manager.stop();
|
||||
});
|
||||
});
|
||||
|
||||
it(
|
||||
"refused WebSocket connections and replacing a connecting socket do not emit unhandled errors",
|
||||
{ timeout: 5_000 },
|
||||
async () => {
|
||||
const server = createServer();
|
||||
await new Promise<void>((resolve) =>
|
||||
server.listen(0, "127.0.0.1", resolve)
|
||||
);
|
||||
const address = server.address();
|
||||
assert(address && typeof address !== "string");
|
||||
await new Promise<void>((resolve, reject) =>
|
||||
server.close((error) => (error ? reject(error) : resolve()))
|
||||
);
|
||||
const logger = new Logger();
|
||||
const settings = new Settings(
|
||||
logger,
|
||||
{
|
||||
remoteUri: `http://127.0.0.1:${address.port}`,
|
||||
webSocketRetryIntervalMs: 10
|
||||
},
|
||||
async () => {}
|
||||
);
|
||||
const manager = new WebSocketManager(
|
||||
"test",
|
||||
logger,
|
||||
settings,
|
||||
WebSocket
|
||||
);
|
||||
let closes = 0;
|
||||
const disconnected = Promise.withResolvers<void>();
|
||||
manager.onWebSocketStatusChanged.add((connected) => {
|
||||
if (!connected && ++closes >= 2) disconnected.resolve();
|
||||
});
|
||||
try {
|
||||
manager.start();
|
||||
manager.start();
|
||||
await disconnected.promise;
|
||||
assert.equal(manager.isWebSocketConnected, false);
|
||||
} finally {
|
||||
await manager.stop();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
for (const failedAttempt of [1, 2]) {
|
||||
it(`recovers from a WebSocket constructor failure on attempt ${failedAttempt}`, async (context) => {
|
||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||
let attempts = 0;
|
||||
class FailingSocket extends MockWebSocket {
|
||||
static OPEN = 1;
|
||||
constructor(url: string) {
|
||||
if (++attempts === failedAttempt)
|
||||
throw new Error("socket allocation failed");
|
||||
super(url);
|
||||
}
|
||||
}
|
||||
const logger = new Logger();
|
||||
const settings = new Settings(
|
||||
logger,
|
||||
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
||||
async () => {}
|
||||
);
|
||||
const manager = new WebSocketManager(
|
||||
"test",
|
||||
logger,
|
||||
settings,
|
||||
FailingSocket as unknown as typeof WebSocket
|
||||
);
|
||||
try {
|
||||
assert.doesNotThrow(() => manager.start());
|
||||
context.mock.timers.tick(0);
|
||||
if (failedAttempt === 2) {
|
||||
const socket = (
|
||||
manager as unknown as { webSocket: MockWebSocket }
|
||||
).webSocket;
|
||||
socket.close();
|
||||
}
|
||||
assert.doesNotThrow(() => context.mock.timers.tick(1000));
|
||||
context.mock.timers.tick(1000);
|
||||
assert.equal(attempts, failedAttempt + 1);
|
||||
assert.equal(manager.isWebSocketConnected, true);
|
||||
} finally {
|
||||
await manager.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const closeBehavior of ["throws", "stays open"]) {
|
||||
it(`stop retires a socket even when close ${closeBehavior}`, async (context) => {
|
||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||
class BrokenCloseSocket extends MockWebSocket {
|
||||
static OPEN = 1;
|
||||
close() {
|
||||
if (closeBehavior === "throws") throw new Error("close failed");
|
||||
}
|
||||
}
|
||||
const logger = new Logger();
|
||||
const settings = new Settings(
|
||||
logger,
|
||||
{ remoteUri: "http://test" },
|
||||
async () => {}
|
||||
);
|
||||
const manager = new WebSocketManager(
|
||||
"test",
|
||||
logger,
|
||||
settings,
|
||||
BrokenCloseSocket as unknown as typeof WebSocket
|
||||
);
|
||||
let connected = false,
|
||||
delivered = 0;
|
||||
manager.onWebSocketStatusChanged.add((status) => {
|
||||
connected = status;
|
||||
});
|
||||
manager.onRemoteVaultUpdateReceived.add(async () => {
|
||||
delivered++;
|
||||
});
|
||||
manager.start();
|
||||
context.mock.timers.tick(0);
|
||||
const socket = (manager as unknown as { webSocket: MockWebSocket })
|
||||
.webSocket;
|
||||
const stopping = manager.stop();
|
||||
context.mock.timers.tick(10_000);
|
||||
await stopping;
|
||||
assert.equal(manager.isWebSocketConnected, false);
|
||||
assert.equal(connected, false);
|
||||
socket.simulateMessage({
|
||||
type: "vaultEvents",
|
||||
headEventId: 0,
|
||||
events: []
|
||||
});
|
||||
await manager.waitUntilFinished();
|
||||
assert.equal(delivered, 0);
|
||||
});
|
||||
}
|
||||
|
||||
it("a handshake send failure retires the connection without escaping the open callback", async (context) => {
|
||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||
class BrokenSendSocket extends MockWebSocket {
|
||||
static OPEN = 1;
|
||||
send() {
|
||||
throw new Error("transport closed during open");
|
||||
}
|
||||
}
|
||||
const logger = new Logger();
|
||||
const settings = new Settings(
|
||||
logger,
|
||||
{ remoteUri: "http://test" },
|
||||
async () => {}
|
||||
);
|
||||
const manager = new WebSocketManager(
|
||||
"test",
|
||||
logger,
|
||||
settings,
|
||||
BrokenSendSocket as unknown as typeof WebSocket
|
||||
);
|
||||
manager.onWebSocketStatusChanged.add((connected) => {
|
||||
if (connected)
|
||||
manager.sendHandshakeMessage({
|
||||
type: "handshake",
|
||||
token: "test",
|
||||
deviceId: "test",
|
||||
lastSeenVaultUpdateId: 0
|
||||
});
|
||||
});
|
||||
const statuses: boolean[] = [];
|
||||
manager.onWebSocketStatusChanged.add((status) => statuses.push(status));
|
||||
try {
|
||||
manager.start();
|
||||
assert.doesNotThrow(() => context.mock.timers.tick(0));
|
||||
assert.equal(manager.isWebSocketConnected, false);
|
||||
assert.equal(statuses.at(-1), false);
|
||||
} finally {
|
||||
await manager.stop();
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ export class WebSocketManager {
|
|||
|
||||
private isStopped = true;
|
||||
private resolveDisconnectingPromise: null | (() => unknown) = null;
|
||||
private receiveTimeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
private reconnectTimeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
private readonly outstandingPromises: Promise<unknown>[] = [];
|
||||
|
|
@ -56,12 +57,14 @@ export class WebSocketManager {
|
|||
|
||||
public get isWebSocketConnected(): boolean {
|
||||
return (
|
||||
this.webSocket?.readyState ===
|
||||
this.webSocketFactoryImplementation.OPEN
|
||||
this.webSocket !== undefined &&
|
||||
this.webSocket.readyState ===
|
||||
this.webSocketFactoryImplementation.OPEN
|
||||
);
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
if (!this.isStopped) return;
|
||||
this.isStopped = false;
|
||||
this.initializeWebSocket();
|
||||
}
|
||||
|
|
@ -71,13 +74,20 @@ export class WebSocketManager {
|
|||
this.resolveDisconnectingPromise = resolve;
|
||||
|
||||
this.isStopped = true;
|
||||
clearTimeout(this.receiveTimeoutId);
|
||||
|
||||
if (this.reconnectTimeoutId !== undefined) {
|
||||
clearTimeout(this.reconnectTimeoutId);
|
||||
this.reconnectTimeoutId = undefined;
|
||||
}
|
||||
|
||||
this.webSocket?.close(1000, "WebSocketManager has been stopped");
|
||||
const socket = this.webSocket;
|
||||
try {
|
||||
socket?.close(1000, "WebSocketManager has been stopped");
|
||||
} catch (error) {
|
||||
this.logger.warn(`Failed to close WebSocket: ${String(error)}`);
|
||||
if (socket) this.reconnect(socket);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/init-declarations
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
|
|
@ -107,6 +117,9 @@ export class WebSocketManager {
|
|||
if (timeoutId !== undefined) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
// A throwing, silent or timed-out close must still fence callbacks
|
||||
// before a stopped client can release its state or restart.
|
||||
if (socket) this.reconnect(socket);
|
||||
}
|
||||
|
||||
await this.waitUntilFinished();
|
||||
|
|
@ -132,6 +145,7 @@ export class WebSocketManager {
|
|||
this.logger.error(
|
||||
`Failed to send handshake message: ${String(error)}`
|
||||
);
|
||||
this.reconnect(webSocket);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
|
@ -162,7 +176,66 @@ export class WebSocketManager {
|
|||
}
|
||||
}
|
||||
|
||||
private expectResponse(socket: WebSocket): void {
|
||||
clearTimeout(this.receiveTimeoutId);
|
||||
// Cursor heartbeats are acknowledged every 15 seconds, including when
|
||||
// there are no open editors. An OPEN TCP socket alone proves nothing.
|
||||
this.receiveTimeoutId = setTimeout(() => {
|
||||
if (this.webSocket !== socket || this.isStopped) return;
|
||||
this.logger.warn("WebSocket receive deadline expired");
|
||||
this.reconnect(socket);
|
||||
}, 45_000);
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Browser timers have no unref method.
|
||||
this.receiveTimeoutId.unref?.();
|
||||
}
|
||||
|
||||
private reconnect(socket: WebSocket): void {
|
||||
if (this.webSocket !== socket) return;
|
||||
clearTimeout(this.receiveTimeoutId);
|
||||
socket.onopen = null;
|
||||
socket.onclose = null;
|
||||
socket.onmessage = null;
|
||||
socket.onerror = (): void => {
|
||||
/* Retired transports may still emit errors. */
|
||||
};
|
||||
this.webSocket = undefined;
|
||||
this.onWebSocketStatusChanged.trigger(false);
|
||||
try {
|
||||
socket.close();
|
||||
} catch {
|
||||
/* The transport may already be gone. */
|
||||
}
|
||||
if (this.isStopped) {
|
||||
this.resolveDisconnectingPromise?.();
|
||||
this.resolveDisconnectingPromise = null;
|
||||
} else {
|
||||
this.scheduleReconnect();
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
if (this.isStopped) return;
|
||||
clearTimeout(this.reconnectTimeoutId);
|
||||
this.reconnectTimeoutId = setTimeout(() => {
|
||||
this.reconnectTimeoutId = undefined;
|
||||
this.initializeWebSocket();
|
||||
}, this.settings.getSettings().webSocketRetryIntervalMs);
|
||||
}
|
||||
|
||||
private initializeWebSocket(): void {
|
||||
if (this.isStopped) return;
|
||||
try {
|
||||
this.openWebSocket();
|
||||
} catch (error) {
|
||||
this.logger.warn(`Cannot initialize WebSocket: ${String(error)}`);
|
||||
this.webSocket = undefined;
|
||||
clearTimeout(this.receiveTimeoutId);
|
||||
this.onWebSocketStatusChanged.trigger(false);
|
||||
this.scheduleReconnect();
|
||||
}
|
||||
}
|
||||
|
||||
private openWebSocket(): void {
|
||||
// Clean up old WebSocket handlers to prevent race conditions
|
||||
if (this.webSocket) {
|
||||
try {
|
||||
|
|
@ -170,7 +243,9 @@ export class WebSocketManager {
|
|||
this.webSocket.onopen = null;
|
||||
this.webSocket.onclose = null;
|
||||
this.webSocket.onmessage = null;
|
||||
this.webSocket.onerror = null;
|
||||
this.webSocket.onerror = (): void => {
|
||||
/* Retired transports may still emit errors. */
|
||||
};
|
||||
this.webSocket.close();
|
||||
} catch (e) {
|
||||
this.logger.error(
|
||||
|
|
@ -188,7 +263,14 @@ export class WebSocketManager {
|
|||
|
||||
this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`);
|
||||
|
||||
this.webSocket = new this.webSocketFactoryImplementation(wsUri);
|
||||
const socket = new this.webSocketFactoryImplementation(wsUri);
|
||||
this.webSocket = socket;
|
||||
this.expectResponse(socket);
|
||||
|
||||
this.webSocket.onerror = (): void => {
|
||||
this.logger.warn("WebSocket transport error");
|
||||
this.reconnect(socket);
|
||||
};
|
||||
|
||||
this.webSocket.onopen = (): void => {
|
||||
// Check if we've been stopped while connecting
|
||||
|
|
@ -200,7 +282,18 @@ export class WebSocketManager {
|
|||
return;
|
||||
}
|
||||
this.logger.info("WebSocket connection opened");
|
||||
this.onWebSocketStatusChanged.trigger(true);
|
||||
try {
|
||||
this.onWebSocketStatusChanged.trigger(true);
|
||||
// A handshake failure can retire this socket from inside an
|
||||
// observer. Finish delivery with its actual connection state.
|
||||
if (this.webSocket !== socket)
|
||||
this.onWebSocketStatusChanged.trigger(false);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`WebSocket handshake failed: ${String(error)}`
|
||||
);
|
||||
this.reconnect(socket);
|
||||
}
|
||||
};
|
||||
|
||||
this.webSocket.onmessage = (event): void => {
|
||||
|
|
@ -210,6 +303,8 @@ export class WebSocketManager {
|
|||
event.data
|
||||
) as WebSocketServerMessage;
|
||||
|
||||
if (["vaultEvents", "cursorPositions"].includes(message.type))
|
||||
this.expectResponse(socket);
|
||||
// Track the message handling promise
|
||||
const messageHandlingPromise = this.handleWebSocketMessage(
|
||||
message
|
||||
|
|
@ -238,17 +333,7 @@ export class WebSocketManager {
|
|||
this.logger.warn(
|
||||
`WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})`
|
||||
);
|
||||
this.onWebSocketStatusChanged.trigger(false);
|
||||
|
||||
if (this.isStopped) {
|
||||
this.resolveDisconnectingPromise?.();
|
||||
this.resolveDisconnectingPromise = null;
|
||||
} else {
|
||||
this.reconnectTimeoutId = setTimeout(() => {
|
||||
this.reconnectTimeoutId = undefined;
|
||||
this.initializeWebSocket();
|
||||
}, this.settings.getSettings().webSocketRetryIntervalMs);
|
||||
}
|
||||
this.reconnect(socket);
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,62 +1,246 @@
|
|||
import type { PersistenceProvider } from "./persistence/persistence";
|
||||
import type { HistoryEntry, HistoryStats } from "./tracing/sync-history";
|
||||
import { SyncHistory } from "./tracing/sync-history";
|
||||
import { Logger, LogLevel, LogLine } from "./tracing/logger";
|
||||
import type { RelativePath, StoredDatabase } from "./persistence/database";
|
||||
import { Database } from "./persistence/database";
|
||||
import * as Sentry from "@sentry/browser";
|
||||
import type { SyncSettings } from "./persistence/settings";
|
||||
import { DEFAULT_SETTINGS, Settings } from "./persistence/settings";
|
||||
import { SyncService } from "./services/sync-service";
|
||||
import { Syncer } from "./sync-operations/syncer";
|
||||
import { setUpTelemetry } from "./utils/set-up-telemetry";
|
||||
import type { PersistenceProvider } from "./persistence/persistence";
|
||||
import { Database, type StoredDatabase } from "./persistence/database";
|
||||
import {
|
||||
Settings,
|
||||
DEFAULT_SETTINGS,
|
||||
type SyncSettings
|
||||
} from "./persistence/settings";
|
||||
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
|
||||
import { FileOperations } from "./file-operations/file-operations";
|
||||
import { Logger, LogLevel } from "./tracing/logger";
|
||||
import { SyncHistory } from "./tracing/sync-history";
|
||||
import { FetchController } from "./services/fetch-controller";
|
||||
import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer";
|
||||
import { rateLimit } from "./utils/rate-limit";
|
||||
import type { NetworkConnectionStatus } from "./types/network-connection-status";
|
||||
import { DocumentSyncStatus } from "./types/document-sync-status";
|
||||
import { WebSocketManager } from "./services/websocket-manager";
|
||||
import { createClientId } from "./utils/create-client-id";
|
||||
import { CursorTracker } from "./sync-operations/cursor-tracker";
|
||||
import type { CursorSpan } from "./services/types/CursorSpan";
|
||||
import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
|
||||
import { FileChangeNotifier } from "./sync-operations/file-change-notifier";
|
||||
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
||||
import { setUpTelemetry } from "./utils/set-up-telemetry";
|
||||
import { DIFF_CACHE_SIZE_MB } from "./consts";
|
||||
import { SyncService } from "./services/sync-service";
|
||||
import { ServerConfig } from "./services/server-config";
|
||||
import type { EventListeners } from "./utils/data-structures/event-listeners";
|
||||
import { WebSocketManager } from "./services/websocket-manager";
|
||||
import { Syncer } from "./sync-operations/syncer";
|
||||
import { CursorTracker } from "./sync-operations/cursor-tracker";
|
||||
import { FileChangeNotifier } from "./sync-operations/file-change-notifier";
|
||||
import { createClientId } from "./utils/create-client-id";
|
||||
import { DocumentSyncStatus } from "./types/document-sync-status";
|
||||
import type { CursorSpan } from "./services/types/CursorSpan";
|
||||
import { isInternalPath } from "./utils/portable-path";
|
||||
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
||||
import { globsToRegexes } from "./utils/globs-to-regexes";
|
||||
import { Lock } from "./utils/data-structures/locks";
|
||||
import type { LocalChange } from "./sync-operations/local-changes";
|
||||
|
||||
type StoredClient = Partial<{
|
||||
settings: Partial<SyncSettings>;
|
||||
database: Partial<StoredDatabase>;
|
||||
localChanges: LocalChange[];
|
||||
localChangesVaultKey: string;
|
||||
historyCheckpoint: { vaultKey: string; checkpoint?: string };
|
||||
}>;
|
||||
|
||||
const vaultKey = (settings: SyncSettings): string =>
|
||||
JSON.stringify([
|
||||
settings.remoteUri.replace(/\/$/u, ""),
|
||||
settings.vaultName
|
||||
]);
|
||||
|
||||
export class SyncClient {
|
||||
private hasStartedOfflineSync = false;
|
||||
private hasFinishedOfflineSync = false;
|
||||
private hasStarted = false;
|
||||
private hasBeenDestroyed = false;
|
||||
private readonly lifecycle = new Lock();
|
||||
private destroying?: Promise<void>;
|
||||
private started = false;
|
||||
private destroyed = false;
|
||||
private unloadTelemetry?: () => void;
|
||||
private isDestroying = false;
|
||||
private readonly eventUnsubscribers: (() => void)[] = [];
|
||||
|
||||
private readonly unsubscribeLog: () => void;
|
||||
private constructor(
|
||||
public readonly logger: Logger,
|
||||
private readonly history: SyncHistory,
|
||||
private readonly settings: Settings,
|
||||
private readonly database: Database,
|
||||
private readonly syncer: Syncer,
|
||||
private readonly webSocketManager: WebSocketManager,
|
||||
public readonly logger: Logger,
|
||||
private readonly fetchController: FetchController,
|
||||
private readonly serverConfig: ServerConfig,
|
||||
private readonly cursorTracker: CursorTracker,
|
||||
private readonly fileChangeNotifier: FileChangeNotifier,
|
||||
private readonly contentCache: FixedSizeDocumentCache,
|
||||
private readonly fileOperations: FileOperations,
|
||||
private readonly serverConfig: ServerConfig,
|
||||
private readonly persistence: PersistenceProvider<
|
||||
Partial<{
|
||||
settings: Partial<SyncSettings>;
|
||||
database: Partial<StoredDatabase>;
|
||||
}>
|
||||
>
|
||||
) {}
|
||||
private readonly persistence: PersistenceProvider<StoredClient>
|
||||
) {
|
||||
syncer.onServerHistoryChanged.add(() => {
|
||||
cursorTracker.reset();
|
||||
});
|
||||
syncer.onReadyForEvents.add(() => {
|
||||
if (
|
||||
this.started &&
|
||||
!this.destroyed &&
|
||||
settings.getSettings().isSyncEnabled
|
||||
)
|
||||
webSocketManager.start();
|
||||
});
|
||||
this.unsubscribeLog = logger.onLogEmitted.add((log) => {
|
||||
if (log.level === LogLevel.ERROR && Sentry.isInitialized())
|
||||
Sentry.captureMessage(log.message);
|
||||
});
|
||||
}
|
||||
|
||||
public static async create({
|
||||
fs,
|
||||
persistence,
|
||||
fetch,
|
||||
webSocket
|
||||
}: {
|
||||
fs: FileSystemOperations;
|
||||
persistence: PersistenceProvider<StoredClient>;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
webSocket?: typeof globalThis.WebSocket;
|
||||
/** V4 preserves bytes and line endings as stored, on every platform. */
|
||||
nativeLineEndings?: string;
|
||||
}): Promise<SyncClient> {
|
||||
const logger = new Logger();
|
||||
let stored = (await persistence.load()) ?? {};
|
||||
let saving: Promise<void> = Promise.resolve();
|
||||
let reloadBeforeSave = false;
|
||||
// Settings and engine state share a store. Serialize complete durable
|
||||
// replacements so one cannot overwrite a newer save from the other.
|
||||
const save = async (update: StoredClient): Promise<void> => {
|
||||
const operation = saving.then(async () => {
|
||||
if (reloadBeforeSave) {
|
||||
stored = (await persistence.load()) ?? {};
|
||||
reloadBeforeSave = false;
|
||||
}
|
||||
const next = { ...stored, ...structuredClone(update) };
|
||||
try {
|
||||
await persistence.save(next);
|
||||
stored = next;
|
||||
} catch (error) {
|
||||
// A failed save may have committed. Do not let a queued
|
||||
// settings/notification save roll back that engine state.
|
||||
reloadBeforeSave = true;
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
saving = operation.catch(() => {});
|
||||
return operation;
|
||||
};
|
||||
const settings = new Settings(logger, stored.settings, async (data) =>
|
||||
save({ settings: data })
|
||||
);
|
||||
const database = new Database(
|
||||
logger,
|
||||
stored.database,
|
||||
async (data) => save({ database: data }),
|
||||
vaultKey(settings.getSettings()),
|
||||
async () => {
|
||||
await saving;
|
||||
stored = (await persistence.load()) ?? {};
|
||||
return stored.database as StoredDatabase | undefined;
|
||||
}
|
||||
);
|
||||
if (
|
||||
stored.localChanges?.length &&
|
||||
stored.localChangesVaultKey !== vaultKey(settings.getSettings())
|
||||
) {
|
||||
throw new Error(
|
||||
"Offline notifications belong to another vault; use a separate state store"
|
||||
);
|
||||
}
|
||||
const fetchController = new FetchController(
|
||||
settings.getSettings().isSyncEnabled,
|
||||
logger
|
||||
);
|
||||
const deviceId = createClientId();
|
||||
const service = new SyncService(
|
||||
deviceId,
|
||||
fetchController,
|
||||
settings,
|
||||
logger,
|
||||
fetch,
|
||||
{
|
||||
get: () =>
|
||||
stored.historyCheckpoint?.vaultKey ===
|
||||
vaultKey(settings.getSettings())
|
||||
? stored.historyCheckpoint.checkpoint
|
||||
: undefined,
|
||||
save: async (checkpoint) =>
|
||||
save({
|
||||
historyCheckpoint: {
|
||||
vaultKey: vaultKey(settings.getSettings()),
|
||||
checkpoint
|
||||
}
|
||||
})
|
||||
}
|
||||
);
|
||||
const serverConfig = new ServerConfig(service);
|
||||
const notifier = new FileChangeNotifier();
|
||||
const localChanges = stored.localChanges ?? [];
|
||||
let syncer: Syncer;
|
||||
const files = new FileOperations(
|
||||
fs,
|
||||
database,
|
||||
serverConfig,
|
||||
(paths) => {
|
||||
for (const path of paths) notifier.notifyOfFileChange(path);
|
||||
},
|
||||
{
|
||||
entries: () => localChanges,
|
||||
flush: async () => syncer?.flushLocalChanges()
|
||||
},
|
||||
(path, size) => {
|
||||
const current = settings.getSettings();
|
||||
return (
|
||||
size > current.maxFileSizeMB * 1024 * 1024 ||
|
||||
globsToRegexes(current.ignorePatterns, logger).some(
|
||||
(pattern) => pattern.test(path)
|
||||
)
|
||||
);
|
||||
}
|
||||
);
|
||||
const websocket = new WebSocketManager(
|
||||
deviceId,
|
||||
logger,
|
||||
settings,
|
||||
webSocket
|
||||
);
|
||||
const history = new SyncHistory(logger);
|
||||
const contentCache = new FixedSizeDocumentCache(
|
||||
settings.getSettings().diffCacheSizeMB * 1024 * 1024
|
||||
);
|
||||
syncer = new Syncer(
|
||||
deviceId,
|
||||
logger,
|
||||
database,
|
||||
settings,
|
||||
service,
|
||||
websocket,
|
||||
files,
|
||||
serverConfig,
|
||||
history,
|
||||
contentCache,
|
||||
{
|
||||
entries: localChanges,
|
||||
save: async (entries) =>
|
||||
save({
|
||||
localChanges: entries,
|
||||
localChangesVaultKey: vaultKey(settings.getSettings())
|
||||
})
|
||||
}
|
||||
);
|
||||
const cursors = new CursorTracker(database, websocket, files, notifier);
|
||||
return new SyncClient(
|
||||
logger,
|
||||
history,
|
||||
settings,
|
||||
database,
|
||||
syncer,
|
||||
websocket,
|
||||
fetchController,
|
||||
serverConfig,
|
||||
cursors,
|
||||
notifier,
|
||||
contentCache,
|
||||
persistence
|
||||
);
|
||||
}
|
||||
|
||||
private check(): void {
|
||||
if (this.destroyed) throw new Error("SyncClient has been destroyed");
|
||||
}
|
||||
|
||||
public get documentCount(): number {
|
||||
return this.database.length;
|
||||
|
|
@ -66,478 +250,229 @@ export class SyncClient {
|
|||
return this.webSocketManager.isWebSocketConnected;
|
||||
}
|
||||
|
||||
public get onSyncHistoryUpdated(): EventListeners<
|
||||
(stats: HistoryStats) => unknown
|
||||
> {
|
||||
this.checkIfDestroyed("onSyncHistoryUpdated getter");
|
||||
public get onSyncHistoryUpdated() {
|
||||
return this.history.onHistoryUpdated;
|
||||
}
|
||||
|
||||
public get onSettingsChanged(): EventListeners<
|
||||
(newSettings: SyncSettings, oldSettings: SyncSettings) => unknown
|
||||
> {
|
||||
this.checkIfDestroyed("onSettingsChanged getter");
|
||||
public get onSettingsChanged() {
|
||||
return this.settings.onSettingsChanged;
|
||||
}
|
||||
|
||||
public get onRemainingOperationsCountChanged(): EventListeners<
|
||||
(remainingOperationsCount: number) => unknown
|
||||
> {
|
||||
this.checkIfDestroyed("onRemainingOperationsCountChanged getter");
|
||||
public get onRemainingOperationsCountChanged() {
|
||||
return this.syncer.onRemainingOperationsCountChanged;
|
||||
}
|
||||
|
||||
public get onWebSocketStatusChanged(): EventListeners<
|
||||
(isConnected: boolean) => unknown
|
||||
> {
|
||||
this.checkIfDestroyed("onWebSocketStatusChanged getter");
|
||||
public get onWebSocketStatusChanged() {
|
||||
return this.webSocketManager.onWebSocketStatusChanged;
|
||||
}
|
||||
|
||||
public get onRemoteCursorsUpdated(): EventListeners<
|
||||
(cursors: MaybeOutdatedClientCursors[]) => unknown
|
||||
> {
|
||||
this.checkIfDestroyed("onRemoteCursorsUpdated getter");
|
||||
public get onRemoteCursorsUpdated() {
|
||||
return this.cursorTracker.onRemoteCursorsUpdated;
|
||||
}
|
||||
|
||||
public static async create({
|
||||
fs,
|
||||
persistence,
|
||||
fetch,
|
||||
webSocket,
|
||||
nativeLineEndings = "\n"
|
||||
}: {
|
||||
fs: FileSystemOperations;
|
||||
persistence: PersistenceProvider<
|
||||
Partial<{
|
||||
settings: Partial<SyncSettings>;
|
||||
database: Partial<StoredDatabase>;
|
||||
}>
|
||||
>;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
webSocket?: typeof globalThis.WebSocket;
|
||||
nativeLineEndings?: string;
|
||||
}): Promise<SyncClient> {
|
||||
const logger = new Logger();
|
||||
|
||||
const deviceId = createClientId();
|
||||
|
||||
logger.info(`Creating SyncClient with client id ${deviceId}`);
|
||||
|
||||
const history = new SyncHistory(logger);
|
||||
|
||||
let state = (await persistence.load()) ?? {
|
||||
settings: undefined,
|
||||
database: undefined
|
||||
};
|
||||
|
||||
const settings = new Settings(
|
||||
logger,
|
||||
state.settings,
|
||||
async (data): Promise<void> => {
|
||||
state = { ...state, settings: data };
|
||||
// we're not rate-limiting settings saves as (1) we need to initialise the settings to know the rate limit
|
||||
// and (2) settings changes are infrequent enough that rate-limiting is not necessary
|
||||
await persistence.save(state);
|
||||
}
|
||||
);
|
||||
|
||||
const rateLimitedSave = rateLimit(
|
||||
persistence.save,
|
||||
() => settings.getSettings().minimumSaveIntervalMs
|
||||
);
|
||||
|
||||
const database = new Database(
|
||||
logger,
|
||||
state.database,
|
||||
async (data): Promise<void> => {
|
||||
state = { ...state, database: data };
|
||||
await rateLimitedSave(state);
|
||||
}
|
||||
);
|
||||
|
||||
const fetchController = new FetchController(
|
||||
settings.getSettings().isSyncEnabled,
|
||||
logger
|
||||
);
|
||||
|
||||
const syncService = new SyncService(
|
||||
deviceId,
|
||||
fetchController,
|
||||
settings,
|
||||
logger,
|
||||
fetch
|
||||
);
|
||||
|
||||
const serverConfig = new ServerConfig(syncService);
|
||||
|
||||
const fileOperations = new FileOperations(
|
||||
logger,
|
||||
database,
|
||||
fs,
|
||||
serverConfig,
|
||||
nativeLineEndings
|
||||
);
|
||||
|
||||
const contentCache = new FixedSizeDocumentCache(
|
||||
1024 * 1024 * DIFF_CACHE_SIZE_MB
|
||||
);
|
||||
const unrestrictedSyncer = new UnrestrictedSyncer(
|
||||
logger,
|
||||
database,
|
||||
settings,
|
||||
syncService,
|
||||
fileOperations,
|
||||
history,
|
||||
contentCache,
|
||||
serverConfig
|
||||
);
|
||||
|
||||
const webSocketManager = new WebSocketManager(
|
||||
deviceId,
|
||||
logger,
|
||||
settings,
|
||||
webSocket
|
||||
);
|
||||
|
||||
const syncer = new Syncer(
|
||||
deviceId,
|
||||
logger,
|
||||
database,
|
||||
settings,
|
||||
syncService,
|
||||
webSocketManager,
|
||||
fileOperations,
|
||||
unrestrictedSyncer
|
||||
);
|
||||
|
||||
const fileChangeNotifier = new FileChangeNotifier();
|
||||
const cursorTracker = new CursorTracker(
|
||||
database,
|
||||
webSocketManager,
|
||||
fileOperations,
|
||||
fileChangeNotifier
|
||||
);
|
||||
const client = new SyncClient(
|
||||
history,
|
||||
settings,
|
||||
database,
|
||||
syncer,
|
||||
webSocketManager,
|
||||
logger,
|
||||
fetchController,
|
||||
cursorTracker,
|
||||
fileChangeNotifier,
|
||||
contentCache,
|
||||
fileOperations,
|
||||
serverConfig,
|
||||
persistence
|
||||
);
|
||||
|
||||
logger.info("SyncClient created successfully");
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
this.checkIfDestroyed("start");
|
||||
|
||||
if (this.hasStarted) {
|
||||
throw new Error("SyncClient has already been started");
|
||||
}
|
||||
this.hasStarted = true;
|
||||
|
||||
if (
|
||||
!this.unloadTelemetry &&
|
||||
this.settings.getSettings().enableTelemetry
|
||||
) {
|
||||
this.unloadTelemetry = setUpTelemetry();
|
||||
}
|
||||
|
||||
this.eventUnsubscribers.push(
|
||||
this.settings.onSettingsChanged.add((newSettings, oldSettings) => {
|
||||
if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) {
|
||||
this.fetchController.canFetch = newSettings.isSyncEnabled;
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
this.eventUnsubscribers.push(
|
||||
this.logger.onLogEmitted.add((log): void => {
|
||||
if (log.level === LogLevel.ERROR && Sentry.isInitialized()) {
|
||||
Sentry.captureMessage(log.message);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
this.eventUnsubscribers.push(
|
||||
this.settings.onSettingsChanged.add(
|
||||
this.onSettingsChange.bind(this)
|
||||
)
|
||||
);
|
||||
|
||||
if (this.settings.getSettings().isSyncEnabled) {
|
||||
this.logger.info("Starting SyncClient");
|
||||
await this.startSyncing();
|
||||
this.logger.info("SyncClient has successfully started");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reload settings from disk overriding current in-memory settings.
|
||||
* Missing values will be filled in from DEFAULT_SETTINGS rather than
|
||||
* retaining current in-memory settings.
|
||||
*/
|
||||
public async reloadSettings(): Promise<void> {
|
||||
this.checkIfDestroyed("reloadSettings");
|
||||
|
||||
const state = (await this.persistence.load()) ?? {
|
||||
settings: undefined
|
||||
};
|
||||
|
||||
const settings = {
|
||||
...DEFAULT_SETTINGS,
|
||||
...(state.settings ?? {})
|
||||
};
|
||||
|
||||
await this.setSettings(settings);
|
||||
}
|
||||
|
||||
public async checkConnection(): Promise<NetworkConnectionStatus> {
|
||||
this.checkIfDestroyed("checkConnection");
|
||||
|
||||
const server = await this.serverConfig.checkConnection(true);
|
||||
return {
|
||||
isSuccessful: server.isSuccessful,
|
||||
serverMessage: server.message,
|
||||
isWebSocketConnected: this.webSocketManager.isWebSocketConnected
|
||||
};
|
||||
}
|
||||
|
||||
public getHistoryEntries(): readonly HistoryEntry[] {
|
||||
public getHistoryEntries() {
|
||||
return this.history.entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the in-flight operations to finish, reset all tracking,
|
||||
* and the local database but retain the settings.
|
||||
* The SyncClient can be used again after calling this method.
|
||||
*/
|
||||
public async reset(): Promise<void> {
|
||||
this.checkIfDestroyed("reset");
|
||||
|
||||
this.logger.info(
|
||||
"Stopping SyncClient to apply changed connection settings"
|
||||
);
|
||||
await this.pause();
|
||||
|
||||
// clear all local state
|
||||
this.logger.info("Resetting SyncClient's local state");
|
||||
this.database.reset();
|
||||
await this.database.save(); // ensure the new database reads as empty
|
||||
this.resetInMemoryState();
|
||||
this.hasStartedOfflineSync = false;
|
||||
this.hasFinishedOfflineSync = false;
|
||||
this.serverConfig.reset();
|
||||
|
||||
await this.startSyncing();
|
||||
}
|
||||
|
||||
public getSettings(): SyncSettings {
|
||||
return this.settings.getSettings();
|
||||
}
|
||||
|
||||
public async setSetting<T extends keyof SyncSettings>(
|
||||
key: T,
|
||||
value: SyncSettings[T]
|
||||
): Promise<void> {
|
||||
this.checkIfDestroyed("setSetting");
|
||||
|
||||
await this.settings.setSetting(key, value);
|
||||
}
|
||||
|
||||
public async setSettings(value: Partial<SyncSettings>): Promise<void> {
|
||||
this.checkIfDestroyed("setSettings");
|
||||
|
||||
await this.settings.setSettings(value);
|
||||
}
|
||||
|
||||
public async syncLocallyCreatedFile(
|
||||
relativePath: RelativePath
|
||||
): Promise<void> {
|
||||
this.checkIfDestroyed("syncLocallyCreatedFile");
|
||||
|
||||
this.fileChangeNotifier.notifyOfFileChange(relativePath);
|
||||
return this.syncer.syncLocallyCreatedFile(relativePath);
|
||||
}
|
||||
|
||||
public async syncLocallyDeletedFile(
|
||||
relativePath: RelativePath
|
||||
): Promise<void> {
|
||||
this.checkIfDestroyed("syncLocallyDeletedFile");
|
||||
|
||||
this.fileChangeNotifier.notifyOfFileChange(relativePath);
|
||||
return this.syncer.syncLocallyDeletedFile(relativePath);
|
||||
}
|
||||
|
||||
public async syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath
|
||||
}: {
|
||||
oldPath?: RelativePath;
|
||||
relativePath: RelativePath;
|
||||
}): Promise<void> {
|
||||
this.checkIfDestroyed("syncLocallyUpdatedFile");
|
||||
|
||||
this.fileChangeNotifier.notifyOfFileChange(relativePath);
|
||||
return this.syncer.syncLocallyUpdatedFile({
|
||||
oldPath,
|
||||
relativePath
|
||||
});
|
||||
}
|
||||
|
||||
public getDocumentSyncingStatus(
|
||||
relativePath: RelativePath
|
||||
): DocumentSyncStatus {
|
||||
this.checkIfDestroyed("getDocumentSyncingStatus");
|
||||
|
||||
if (!this.settings.getSettings().isSyncEnabled) {
|
||||
return DocumentSyncStatus.SYNCING_IS_DISABLED;
|
||||
}
|
||||
|
||||
if (!this.syncer.isFirstSyncComplete || !this.hasFinishedOfflineSync) {
|
||||
return DocumentSyncStatus.SYNCING;
|
||||
}
|
||||
|
||||
const document =
|
||||
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||
if (document === undefined) {
|
||||
return DocumentSyncStatus.SYNCING;
|
||||
}
|
||||
return document.updates.length > 0
|
||||
? DocumentSyncStatus.SYNCING
|
||||
: DocumentSyncStatus.UP_TO_DATE;
|
||||
}
|
||||
|
||||
public async updateLocalCursors(
|
||||
documentToCursors: Record<RelativePath, CursorSpan[]>
|
||||
): Promise<void> {
|
||||
this.checkIfDestroyed("updateLocalCursors");
|
||||
|
||||
await this.cursorTracker.sendLocalCursorsToServer(documentToCursors);
|
||||
}
|
||||
|
||||
public async waitUntilFinished(): Promise<void> {
|
||||
this.checkIfDestroyed("waitUntilIdle");
|
||||
await this.syncer.waitUntilFinished();
|
||||
await this.webSocketManager.waitUntilFinished();
|
||||
await this.database.save(); // flush all changes to disk
|
||||
}
|
||||
|
||||
/**
|
||||
* Completely destroy the SyncClient, cancelling all in-progress operations.
|
||||
* After calling this method, the SyncClient cannot be used again.
|
||||
*/
|
||||
public async destroy(): Promise<void> {
|
||||
this.checkIfDestroyed("destroy");
|
||||
|
||||
// Prevent concurrent destroy calls
|
||||
if (this.isDestroying) {
|
||||
this.logger.warn(
|
||||
"destroy() called while already destroying, ignoring"
|
||||
public async start(): Promise<void> {
|
||||
await this.lifecycle.withLock(async () => {
|
||||
this.check();
|
||||
if (this.started)
|
||||
throw new Error("SyncClient has already been started");
|
||||
await this.syncer.recoverLocalState();
|
||||
await this.database.bindVault(
|
||||
vaultKey(this.settings.getSettings())
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.isDestroying = true;
|
||||
|
||||
// cancel everything that's in progress
|
||||
await this.pause();
|
||||
|
||||
this.hasBeenDestroyed = true;
|
||||
|
||||
this.resetInMemoryState();
|
||||
|
||||
// Clean up event listeners to prevent memory leaks
|
||||
this.eventUnsubscribers.forEach((unsubscribe) => {
|
||||
unsubscribe();
|
||||
this.started = true;
|
||||
this.configureTelemetry();
|
||||
if (this.settings.getSettings().isSyncEnabled) await this.resume();
|
||||
});
|
||||
this.eventUnsubscribers.length = 0;
|
||||
|
||||
this.logger.info("SyncClient has been successfully disposed");
|
||||
|
||||
this.unloadTelemetry?.();
|
||||
}
|
||||
|
||||
private async startSyncing(): Promise<void> {
|
||||
this.checkIfDestroyed("startSyncing");
|
||||
private async resume(): Promise<void> {
|
||||
if (this.destroyed) return;
|
||||
if (!this.settings.getSettings().isSyncEnabled) return;
|
||||
await this.database.bindVault(vaultKey(this.settings.getSettings()));
|
||||
this.fetchController.finishReset();
|
||||
|
||||
await this.serverConfig.initialize();
|
||||
this.webSocketManager.start();
|
||||
|
||||
if (!this.hasStartedOfflineSync) {
|
||||
this.hasStartedOfflineSync = true;
|
||||
await this.syncer.scheduleSyncForOfflineChanges();
|
||||
}
|
||||
|
||||
this.hasFinishedOfflineSync = true;
|
||||
this.fetchController.canFetch = true;
|
||||
this.syncer.start();
|
||||
await this.syncer.waitUntilFinished();
|
||||
}
|
||||
|
||||
private async pause(): Promise<void> {
|
||||
const stopped = this.syncer.stop();
|
||||
this.fetchController.startReset();
|
||||
await this.webSocketManager.stop();
|
||||
await this.waitUntilFinished();
|
||||
await stopped;
|
||||
}
|
||||
|
||||
private resetInMemoryState(): void {
|
||||
this.history.reset();
|
||||
this.contentCache.reset();
|
||||
// don't reset the logger
|
||||
this.cursorTracker.reset();
|
||||
this.syncer.reset();
|
||||
this.fileOperations.reset();
|
||||
/** Restart transports; never discard pending requests or filesystem journals. */
|
||||
public async reset(): Promise<void> {
|
||||
await this.lifecycle.withLock(async () => {
|
||||
this.check();
|
||||
await this.pause();
|
||||
this.serverConfig.reset();
|
||||
this.cursorTracker.reset();
|
||||
await this.syncer.retryRejectedRequests();
|
||||
if (this.started && this.settings.getSettings().isSyncEnabled)
|
||||
await this.resume();
|
||||
});
|
||||
}
|
||||
|
||||
private async onSettingsChange(
|
||||
newSettings: SyncSettings,
|
||||
oldSettings: SyncSettings
|
||||
): Promise<void> {
|
||||
this.checkIfDestroyed("onSettingsChange");
|
||||
public async setSettings(change: Partial<SyncSettings>): Promise<void> {
|
||||
const update = structuredClone(change);
|
||||
const notification = await this.lifecycle.withLock(async () => {
|
||||
this.check();
|
||||
const old = this.settings.getSettings();
|
||||
const next = { ...old, ...update };
|
||||
if (
|
||||
vaultKey(next) !== vaultKey(old) &&
|
||||
(this.started ||
|
||||
this.database.state.initialized ||
|
||||
this.database.state.bootstrap ||
|
||||
this.database.state.pending ||
|
||||
this.database.state.application ||
|
||||
this.syncer.hasLocalChanges ||
|
||||
Object.keys(this.database.state.local).length)
|
||||
)
|
||||
throw new Error(
|
||||
"Changing vaults requires a separate state store and client; pending recovery data must not be reset"
|
||||
);
|
||||
|
||||
if (
|
||||
newSettings.vaultName !== oldSettings.vaultName ||
|
||||
newSettings.remoteUri !== oldSettings.remoteUri
|
||||
) {
|
||||
await this.reset();
|
||||
}
|
||||
|
||||
if (newSettings.isSyncEnabled !== oldSettings.isSyncEnabled) {
|
||||
if (newSettings.isSyncEnabled) {
|
||||
await this.startSyncing();
|
||||
} else {
|
||||
await this.pause();
|
||||
try {
|
||||
if (this.started) await this.pause();
|
||||
await this.database.recoverPersistence();
|
||||
await this.settings.setSettings(update, false);
|
||||
await this.syncer.retryRejectedRequests();
|
||||
this.contentCache.resize(next.diffCacheSizeMB * 1024 * 1024);
|
||||
await this.database.bindVault(vaultKey(next));
|
||||
} catch (error) {
|
||||
this.serverConfig.reset();
|
||||
this.contentCache.resize(
|
||||
this.settings.getSettings().diffCacheSizeMB * 1024 * 1024
|
||||
);
|
||||
this.configureTelemetry();
|
||||
if (this.started && this.settings.getSettings().isSyncEnabled)
|
||||
await this.resume().catch((resumeError: unknown) => {
|
||||
this.logger.error(
|
||||
`Settings recovery could not resume sync: ${String(resumeError)}`
|
||||
);
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
this.serverConfig.reset();
|
||||
this.configureTelemetry();
|
||||
if (this.started && next.isSyncEnabled) await this.resume();
|
||||
return [this.settings.getSettings(), old] as const;
|
||||
});
|
||||
await this.settings.onSettingsChanged.triggerAsync(...notification);
|
||||
}
|
||||
|
||||
if (newSettings.diffCacheSizeMB !== oldSettings.diffCacheSizeMB) {
|
||||
this.contentCache.resize(newSettings.diffCacheSizeMB * 1024 * 1024);
|
||||
}
|
||||
public async setSetting<K extends keyof SyncSettings>(
|
||||
key: K,
|
||||
value: SyncSettings[K]
|
||||
): Promise<void> {
|
||||
await this.setSettings({ [key]: value });
|
||||
}
|
||||
|
||||
if (newSettings.enableTelemetry !== oldSettings.enableTelemetry) {
|
||||
if (newSettings.enableTelemetry) {
|
||||
this.unloadTelemetry = setUpTelemetry();
|
||||
} else {
|
||||
public async reloadSettings(): Promise<void> {
|
||||
const stored = await this.persistence.load();
|
||||
await this.setSettings({ ...DEFAULT_SETTINGS, ...stored?.settings });
|
||||
}
|
||||
|
||||
public async checkConnection() {
|
||||
const result = await this.serverConfig.checkConnection(true);
|
||||
return {
|
||||
isSuccessful: result.isSuccessful,
|
||||
serverMessage: result.message,
|
||||
isWebSocketConnected: this.isWebSocketConnected
|
||||
};
|
||||
}
|
||||
|
||||
/** Notifications enqueue work. Use waitUntilFinished() to await convergence. */
|
||||
public async syncLocallyCreatedFile(path: string): Promise<void> {
|
||||
this.check();
|
||||
if (isInternalPath(path)) return;
|
||||
this.fileChangeNotifier.notifyOfFileChange(path);
|
||||
await this.syncer.syncLocallyCreatedFile(path);
|
||||
}
|
||||
|
||||
public async syncLocallyDeletedFile(path: string): Promise<void> {
|
||||
this.check();
|
||||
if (isInternalPath(path)) return;
|
||||
this.fileChangeNotifier.notifyOfFileChange(path);
|
||||
await this.syncer.syncLocallyDeletedFile(path);
|
||||
}
|
||||
|
||||
public async syncLocallyUpdatedFile(change: {
|
||||
oldPath?: string;
|
||||
relativePath: string;
|
||||
}): Promise<void> {
|
||||
this.check();
|
||||
if (
|
||||
isInternalPath(change.relativePath) ||
|
||||
(change.oldPath && isInternalPath(change.oldPath))
|
||||
)
|
||||
return;
|
||||
this.fileChangeNotifier.notifyOfFileChange(change.relativePath);
|
||||
await this.syncer.syncLocallyUpdatedFile(change);
|
||||
}
|
||||
|
||||
public async updateLocalCursors(
|
||||
cursors: Record<string, CursorSpan[]>
|
||||
): Promise<void> {
|
||||
this.check();
|
||||
await this.cursorTracker.sendLocalCursorsToServer(cursors);
|
||||
}
|
||||
|
||||
public getDocumentSyncingStatus(path: string): DocumentSyncStatus {
|
||||
this.check();
|
||||
if (!this.settings.getSettings().isSyncEnabled)
|
||||
return DocumentSyncStatus.SYNCING_IS_DISABLED;
|
||||
return this.syncer.isDocumentUpToDate(path)
|
||||
? DocumentSyncStatus.UP_TO_DATE
|
||||
: DocumentSyncStatus.SYNCING;
|
||||
}
|
||||
|
||||
public async waitUntilFinished(): Promise<void> {
|
||||
this.check();
|
||||
await this.syncer.waitUntilFinished();
|
||||
}
|
||||
|
||||
private configureTelemetry(): void {
|
||||
if (
|
||||
this.settings.getSettings().enableTelemetry &&
|
||||
typeof window !== "undefined"
|
||||
) {
|
||||
this.unloadTelemetry ??= setUpTelemetry();
|
||||
} else {
|
||||
this.unloadTelemetry?.();
|
||||
this.unloadTelemetry = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
public async destroy(): Promise<void> {
|
||||
if (this.destroying) return this.destroying;
|
||||
this.destroyed = true;
|
||||
// Interrupt a start/reset awaiting network progress before waiting for
|
||||
// its lifecycle operation to finish.
|
||||
this.fetchController.startReset();
|
||||
const stopped = this.syncer.stop();
|
||||
this.destroying = this.lifecycle.withLock(async () => {
|
||||
try {
|
||||
await this.webSocketManager.stop();
|
||||
await stopped;
|
||||
await this.syncer.flushLocalChanges();
|
||||
} finally {
|
||||
this.cursorTracker.reset();
|
||||
this.unsubscribeLog();
|
||||
this.unloadTelemetry?.();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private checkIfDestroyed(origin: string): void {
|
||||
if (this.hasBeenDestroyed) {
|
||||
throw new Error(
|
||||
`SyncClient has been destroyed and can no longer be used; called from ${origin}`
|
||||
);
|
||||
}
|
||||
});
|
||||
return this.destroying;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ it("retains future cursors and publishes them when content catches up", async ()
|
|||
})
|
||||
} as unknown as Database;
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteCursorsUpdateReceived: new EventListeners<
|
||||
(cursors: ClientCursors[]) => Promise<void>
|
||||
>(),
|
||||
|
|
@ -65,3 +68,209 @@ it("retains future cursors and publishes them when content catches up", async ()
|
|||
"note.md"
|
||||
);
|
||||
});
|
||||
|
||||
it("refreshes unchanged cursors on reconnect and heartbeat, and follows the document through renames", async (context) => {
|
||||
context.mock.timers.enable({ apis: ["setInterval"] });
|
||||
const content = new TextEncoder().encode("hello");
|
||||
let path = "note.md";
|
||||
const record = () => ({
|
||||
documentId: "document",
|
||||
relativePath: path,
|
||||
metadata: { parentVersionId: 1, hash: contentHash }
|
||||
});
|
||||
const contentHash = await hash(content);
|
||||
const database = {
|
||||
getLatestDocumentByRelativePath: (candidate: string) =>
|
||||
candidate === path ? record() : undefined,
|
||||
getDocumentByDocumentId: () => record()
|
||||
} as unknown as Database;
|
||||
const sent: {
|
||||
documentsWithCursors: {
|
||||
relative_path: string;
|
||||
vault_update_id: number | null;
|
||||
}[];
|
||||
}[] = [];
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteCursorsUpdateReceived: new EventListeners(),
|
||||
updateLocalCursors: (value: (typeof sent)[number]) => {
|
||||
sent.push(value);
|
||||
}
|
||||
} as unknown as WebSocketManager;
|
||||
const notifier = new FileChangeNotifier();
|
||||
const tracker = new CursorTracker(
|
||||
database,
|
||||
websocket,
|
||||
{ read: async () => content } as unknown as FileOperations,
|
||||
notifier
|
||||
);
|
||||
const drain = async () => {
|
||||
for (let i = 0; i < 20; i++)
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
};
|
||||
try {
|
||||
await tracker.sendLocalCursorsToServer({
|
||||
"note.md": [{ start: 1, end: 2 }]
|
||||
});
|
||||
assert.equal(sent.length, 1);
|
||||
await tracker.sendLocalCursorsToServer({
|
||||
"note.md": [{ start: 1, end: 2 }]
|
||||
});
|
||||
assert.equal(sent.length, 1);
|
||||
websocket.onWebSocketStatusChanged.trigger(true);
|
||||
await drain();
|
||||
assert.equal(sent.length, 2);
|
||||
context.mock.timers.tick(15_000);
|
||||
await drain();
|
||||
assert.equal(sent.length, 3);
|
||||
path = "renamed.md";
|
||||
await notifier.onFileChanged.triggerAsync(path);
|
||||
assert.equal(sent.at(-1)!.documentsWithCursors[0].relative_path, path);
|
||||
websocket.onWebSocketStatusChanged.trigger(false);
|
||||
const count = sent.length;
|
||||
context.mock.timers.tick(30_000);
|
||||
await drain();
|
||||
assert.equal(sent.length, count);
|
||||
websocket.onWebSocketStatusChanged.trigger(true);
|
||||
await drain();
|
||||
assert.equal(sent.length, count + 1);
|
||||
} finally {
|
||||
tracker.reset();
|
||||
context.mock.timers.reset();
|
||||
}
|
||||
});
|
||||
|
||||
for (const interruption of ["disconnect", "reset"] as const) {
|
||||
it(`an in-flight cursor read cannot resurrect presence after ${interruption}`, async () => {
|
||||
const content = new TextEncoder().encode("current");
|
||||
const contentHash = await hash(content);
|
||||
const entered = Promise.withResolvers<void>();
|
||||
const release = Promise.withResolvers<void>();
|
||||
const database = {
|
||||
getDocumentByDocumentId: () => ({
|
||||
documentId: "document",
|
||||
relativePath: "note.md",
|
||||
metadata: { parentVersionId: 1, hash: contentHash }
|
||||
})
|
||||
} as unknown as Database;
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteCursorsUpdateReceived: new EventListeners<
|
||||
(cursors: ClientCursors[]) => Promise<void>
|
||||
>(),
|
||||
updateLocalCursors: () => {}
|
||||
} as unknown as WebSocketManager;
|
||||
const tracker = new CursorTracker(
|
||||
database,
|
||||
websocket,
|
||||
{
|
||||
read: async () => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
return content;
|
||||
}
|
||||
} as unknown as FileOperations,
|
||||
new FileChangeNotifier()
|
||||
);
|
||||
let received: ClientCursors[] = [];
|
||||
tracker.onRemoteCursorsUpdated.add((cursors) => {
|
||||
received = cursors;
|
||||
});
|
||||
const cursor: ClientCursors = {
|
||||
userName: "Other",
|
||||
deviceId: "device",
|
||||
documentsWithCursors: [
|
||||
{
|
||||
document_id: "document",
|
||||
relative_path: "note.md",
|
||||
vault_update_id: 1,
|
||||
cursors: [{ start: 0, end: 1 }]
|
||||
}
|
||||
]
|
||||
};
|
||||
const first = websocket.onRemoteCursorsUpdateReceived.triggerAsync([
|
||||
cursor
|
||||
]);
|
||||
await entered.promise;
|
||||
const queued = websocket.onRemoteCursorsUpdateReceived.triggerAsync([
|
||||
cursor
|
||||
]);
|
||||
if (interruption === "reset") tracker.reset();
|
||||
else websocket.onWebSocketStatusChanged.trigger(false);
|
||||
assert.deepEqual(received, []);
|
||||
release.resolve();
|
||||
await first;
|
||||
await queued;
|
||||
assert.deepEqual(
|
||||
received,
|
||||
[],
|
||||
"a retired connection must not restore its cursors"
|
||||
);
|
||||
tracker.reset();
|
||||
});
|
||||
}
|
||||
|
||||
it("a remote cursor becomes outdated if the document advances during its snapshot read", async () => {
|
||||
const content = new TextEncoder().encode("old");
|
||||
let metadata = { parentVersionId: 1, hash: await hash(content) };
|
||||
const entered = Promise.withResolvers<void>(),
|
||||
release = Promise.withResolvers<void>();
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteCursorsUpdateReceived: new EventListeners<
|
||||
(cursors: ClientCursors[]) => Promise<void>
|
||||
>(),
|
||||
updateLocalCursors: () => {}
|
||||
} as unknown as WebSocketManager;
|
||||
const tracker = new CursorTracker(
|
||||
{
|
||||
getDocumentByDocumentId: () => ({
|
||||
documentId: "document",
|
||||
relativePath: "note.md",
|
||||
metadata
|
||||
})
|
||||
} as unknown as Database,
|
||||
websocket,
|
||||
{
|
||||
read: async () => {
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
return content;
|
||||
}
|
||||
} as unknown as FileOperations,
|
||||
new FileChangeNotifier()
|
||||
);
|
||||
let outdated: boolean | undefined;
|
||||
tracker.onRemoteCursorsUpdated.add((cursors) => {
|
||||
outdated = cursors[0]?.isOutdated;
|
||||
});
|
||||
const update = websocket.onRemoteCursorsUpdateReceived.triggerAsync([
|
||||
{
|
||||
userName: "Other",
|
||||
deviceId: "device",
|
||||
documentsWithCursors: [
|
||||
{
|
||||
document_id: "document",
|
||||
relative_path: "note.md",
|
||||
vault_update_id: 1,
|
||||
cursors: [{ start: 0, end: 1 }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]);
|
||||
await entered.promise;
|
||||
metadata = {
|
||||
parentVersionId: 2,
|
||||
hash: await hash(new TextEncoder().encode("new content"))
|
||||
};
|
||||
release.resolve();
|
||||
await update;
|
||||
assert.equal(outdated, true);
|
||||
tracker.reset();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ export class CursorTracker {
|
|||
>();
|
||||
|
||||
private readonly updateLock = new Lock();
|
||||
private sessionGeneration = 0;
|
||||
|
||||
private knownRemoteCursors: (ClientCursors & {
|
||||
upToDateness: DocumentUpToDateness;
|
||||
|
|
@ -31,6 +32,10 @@ export class CursorTracker {
|
|||
private lastLocalCursorState: DocumentWithCursors[] = [];
|
||||
private lastLocalCursorStateWithoutDirtyDocuments: DocumentWithCursors[] =
|
||||
[];
|
||||
private localCursors: Record<string, CursorSpan[]> = {};
|
||||
private localCursorDocuments: Record<string, string> = {};
|
||||
private cursorTimer?: ReturnType<typeof setInterval>;
|
||||
private readonly sendLock = new Lock();
|
||||
|
||||
public constructor(
|
||||
private readonly database: Database,
|
||||
|
|
@ -38,9 +43,31 @@ export class CursorTracker {
|
|||
private readonly fileOperations: FileOperations,
|
||||
private readonly fileChangeNotifier: FileChangeNotifier
|
||||
) {
|
||||
this.webSocketManager.onWebSocketStatusChanged.add((connected) => {
|
||||
this.sessionGeneration++;
|
||||
clearInterval(this.cursorTimer);
|
||||
this.lastLocalCursorState = [];
|
||||
this.lastLocalCursorStateWithoutDirtyDocuments = [];
|
||||
if (!connected) {
|
||||
this.knownRemoteCursors = [];
|
||||
this.onRemoteCursorsUpdated.trigger([]);
|
||||
return;
|
||||
}
|
||||
const refresh = () => {
|
||||
void this.publishLocalCursors(true).catch(() => {
|
||||
// A concurrent move/deletion can invalidate an editor view;
|
||||
// the next notification or heartbeat retries the snapshot.
|
||||
});
|
||||
};
|
||||
refresh();
|
||||
this.cursorTimer = setInterval(refresh, 15_000);
|
||||
this.cursorTimer.unref?.();
|
||||
});
|
||||
this.webSocketManager.onRemoteCursorsUpdateReceived.add(
|
||||
async (clientCursors) => {
|
||||
const generation = this.sessionGeneration;
|
||||
await this.updateLock.withLock(async () => {
|
||||
if (generation !== this.sessionGeneration) return;
|
||||
// The latest message will contain all active clients, so we can delete the ones
|
||||
// from the local list which are no longer active.
|
||||
const allIds = new Set(
|
||||
|
|
@ -51,11 +78,7 @@ export class CursorTracker {
|
|||
allIds.has(c.deviceId)
|
||||
);
|
||||
|
||||
for (const cursor of clientCursors.filter((client) =>
|
||||
client.documentsWithCursors.every(
|
||||
(doc) => doc.vault_update_id != null
|
||||
)
|
||||
)) {
|
||||
for (const cursor of clientCursors) {
|
||||
updatedKnownRemoteCursors.push({
|
||||
...cursor,
|
||||
upToDateness:
|
||||
|
|
@ -63,17 +86,20 @@ export class CursorTracker {
|
|||
});
|
||||
}
|
||||
|
||||
if (generation !== this.sessionGeneration) return;
|
||||
this.knownRemoteCursors = updatedKnownRemoteCursors;
|
||||
this.onRemoteCursorsUpdated.trigger(
|
||||
this.getRelevantAndPruneKnownClientCursors()
|
||||
);
|
||||
});
|
||||
|
||||
this.onRemoteCursorsUpdated.trigger(
|
||||
this.getRelevantAndPruneKnownClientCursors()
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
this.fileChangeNotifier.onFileChanged.add(async (relativePath) => {
|
||||
const generation = this.sessionGeneration;
|
||||
await this.publishLocalCursors(true);
|
||||
const cursors = await this.updateLock.withLock(async () => {
|
||||
if (generation !== this.sessionGeneration) return [];
|
||||
for (const clientCursor of this.knownRemoteCursors) {
|
||||
if (
|
||||
clientCursor.documentsWithCursors.some(
|
||||
|
|
@ -88,10 +114,13 @@ export class CursorTracker {
|
|||
await this.getDocumentsUpToDateness(clientCursor);
|
||||
}
|
||||
}
|
||||
return this.getRelevantAndPruneKnownClientCursors();
|
||||
return generation === this.sessionGeneration
|
||||
? this.getRelevantAndPruneKnownClientCursors()
|
||||
: [];
|
||||
});
|
||||
|
||||
this.onRemoteCursorsUpdated.trigger(cursors);
|
||||
if (generation === this.sessionGeneration)
|
||||
this.onRemoteCursorsUpdated.trigger(cursors);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -101,71 +130,106 @@ export class CursorTracker {
|
|||
public async sendLocalCursorsToServer(
|
||||
documentToCursors: Record<RelativePath, CursorSpan[]>
|
||||
): Promise<void> {
|
||||
const documentsWithCursors: DocumentWithCursors[] = [];
|
||||
this.localCursors = structuredClone(documentToCursors);
|
||||
this.localCursorDocuments = {};
|
||||
await this.publishLocalCursors(false);
|
||||
}
|
||||
|
||||
for (const [relativePath, cursors] of Object.entries(
|
||||
documentToCursors
|
||||
)) {
|
||||
const record =
|
||||
this.database.getLatestDocumentByRelativePath(relativePath);
|
||||
private async publishLocalCursors(force: boolean): Promise<void> {
|
||||
const generation = this.sessionGeneration;
|
||||
await this.sendLock.withLock(async () => {
|
||||
if (generation !== this.sessionGeneration) return;
|
||||
const documentsWithCursors: DocumentWithCursors[] = [];
|
||||
|
||||
if (!record) {
|
||||
continue; // Let's wait for the file to be created before sending cursors
|
||||
for (const [editorPath, cursors] of Object.entries(
|
||||
this.localCursors
|
||||
)) {
|
||||
const id = this.localCursorDocuments[editorPath];
|
||||
const record = id
|
||||
? this.database.getDocumentByDocumentId(id)
|
||||
: this.database.getLatestDocumentByRelativePath(editorPath);
|
||||
|
||||
if (!record) {
|
||||
continue; // Let's wait for the file to be created before sending cursors
|
||||
}
|
||||
this.localCursorDocuments[editorPath] = record.documentId;
|
||||
const { relativePath } = record;
|
||||
|
||||
if (!record.metadata) {
|
||||
continue; // this is a new document, no need to sync the cursors
|
||||
}
|
||||
|
||||
documentsWithCursors.push({
|
||||
relative_path: relativePath,
|
||||
document_id: record.documentId,
|
||||
vault_update_id: record.metadata.parentVersionId,
|
||||
cursors: cursors.map(({ start, end }) => ({
|
||||
start: Math.min(start, end),
|
||||
end: Math.max(start, end)
|
||||
})) // the client might send directional selections
|
||||
});
|
||||
}
|
||||
|
||||
if (!record.metadata) {
|
||||
continue; // this is a new document, no need to sync the cursors
|
||||
if (
|
||||
!force &&
|
||||
JSON.stringify(this.lastLocalCursorState) ===
|
||||
JSON.stringify(documentsWithCursors)
|
||||
) {
|
||||
// Caching step to avoid reading the edited files all the time
|
||||
return;
|
||||
}
|
||||
this.lastLocalCursorState = structuredClone(documentsWithCursors);
|
||||
|
||||
for (const doc of documentsWithCursors) {
|
||||
let readContent: Uint8Array;
|
||||
try {
|
||||
readContent = await this.fileOperations.read(
|
||||
doc.relative_path
|
||||
);
|
||||
} catch {
|
||||
doc.vault_update_id = null;
|
||||
continue;
|
||||
}
|
||||
const digest = await hash(readContent);
|
||||
const record = this.database.getLatestDocumentByRelativePath(
|
||||
doc.relative_path
|
||||
);
|
||||
if (
|
||||
record?.documentId !== doc.document_id ||
|
||||
record.relativePath !== doc.relative_path ||
|
||||
record.metadata?.parentVersionId !== doc.vault_update_id ||
|
||||
record.metadata.hash !== digest
|
||||
) {
|
||||
doc.vault_update_id = null;
|
||||
}
|
||||
}
|
||||
|
||||
documentsWithCursors.push({
|
||||
relative_path: relativePath,
|
||||
document_id: record.documentId,
|
||||
vault_update_id: record.metadata.parentVersionId,
|
||||
cursors: cursors.map(({ start, end }) => ({
|
||||
start: Math.min(start, end),
|
||||
end: Math.max(start, end)
|
||||
})) // the client might send directional selections
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
JSON.stringify(this.lastLocalCursorState) ===
|
||||
JSON.stringify(documentsWithCursors)
|
||||
) {
|
||||
// Caching step to avoid reading the edited files all the time
|
||||
return;
|
||||
}
|
||||
this.lastLocalCursorState = documentsWithCursors;
|
||||
|
||||
for (const doc of documentsWithCursors) {
|
||||
const readContent = await this.fileOperations.read(
|
||||
doc.relative_path
|
||||
);
|
||||
const record = this.database.getLatestDocumentByRelativePath(
|
||||
doc.relative_path
|
||||
);
|
||||
if (record?.metadata?.hash !== (await hash(readContent))) {
|
||||
doc.vault_update_id = null;
|
||||
if (generation !== this.sessionGeneration) return;
|
||||
if (
|
||||
!force &&
|
||||
JSON.stringify(
|
||||
this.lastLocalCursorStateWithoutDirtyDocuments
|
||||
) === JSON.stringify(documentsWithCursors)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
JSON.stringify(this.lastLocalCursorStateWithoutDirtyDocuments) ===
|
||||
JSON.stringify(documentsWithCursors)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
this.lastLocalCursorStateWithoutDirtyDocuments =
|
||||
documentsWithCursors;
|
||||
|
||||
this.lastLocalCursorStateWithoutDirtyDocuments = documentsWithCursors;
|
||||
|
||||
this.webSocketManager.updateLocalCursors({ documentsWithCursors });
|
||||
this.webSocketManager.updateLocalCursors({ documentsWithCursors });
|
||||
});
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this.sessionGeneration++;
|
||||
clearInterval(this.cursorTimer);
|
||||
this.knownRemoteCursors = [];
|
||||
this.onRemoteCursorsUpdated.trigger([]);
|
||||
this.lastLocalCursorState = [];
|
||||
this.lastLocalCursorStateWithoutDirtyDocuments = [];
|
||||
this.updateLock.reset();
|
||||
// Keep serialization intact while old reads drain. A generation fence
|
||||
// discards both active and queued work from the retired connection.
|
||||
}
|
||||
|
||||
private getRelevantAndPruneKnownClientCursors(): MaybeOutdatedClientCursors[] {
|
||||
|
|
@ -245,6 +309,8 @@ export class CursorTracker {
|
|||
private async getDocumentUpToDateness(
|
||||
document: DocumentWithCursors
|
||||
): Promise<DocumentUpToDateness> {
|
||||
if (document.vault_update_id === null)
|
||||
return DocumentUpToDateness.Prior;
|
||||
const record = this.database.getDocumentByDocumentId(
|
||||
document.document_id
|
||||
);
|
||||
|
|
@ -255,13 +321,11 @@ export class CursorTracker {
|
|||
}
|
||||
|
||||
if (
|
||||
(record.metadata?.parentVersionId ?? 0) <
|
||||
(document.vault_update_id ?? 0)
|
||||
(record.metadata?.parentVersionId ?? 0) < document.vault_update_id
|
||||
) {
|
||||
return DocumentUpToDateness.Later;
|
||||
} else if (
|
||||
(document.vault_update_id ?? 0) <
|
||||
(record.metadata?.parentVersionId ?? 0)
|
||||
document.vault_update_id < (record.metadata?.parentVersionId ?? 0)
|
||||
) {
|
||||
// the document of the cursor must be from the past
|
||||
return DocumentUpToDateness.Prior;
|
||||
|
|
@ -276,7 +340,14 @@ export class CursorTracker {
|
|||
return DocumentUpToDateness.Prior;
|
||||
}
|
||||
|
||||
return record.metadata?.hash === (await hash(currentContent))
|
||||
const contentHash = await hash(currentContent);
|
||||
const current = this.database.getDocumentByDocumentId(
|
||||
document.document_id
|
||||
);
|
||||
return current?.metadata?.parentVersionId ===
|
||||
document.vault_update_id &&
|
||||
current.relativePath === record.relativePath &&
|
||||
current.metadata.hash === contentHash
|
||||
? DocumentUpToDateness.UpToDate
|
||||
: DocumentUpToDateness.Prior;
|
||||
}
|
||||
|
|
|
|||
69
frontend/sync-client/src/sync-operations/file-manifest.ts
Normal file
69
frontend/sync-client/src/sync-operations/file-manifest.ts
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import type { FileManifestEntries } from "../types/file-manifest-entries";
|
||||
import {
|
||||
allocatePortablePath,
|
||||
validatePortablePaths
|
||||
} from "../utils/portable-path";
|
||||
|
||||
export function validateFileManifest(entries: FileManifestEntries): void {
|
||||
validatePortablePaths(Object.values(entries));
|
||||
}
|
||||
|
||||
export function sameFileManifest(
|
||||
a: FileManifestEntries,
|
||||
b: FileManifestEntries
|
||||
): boolean {
|
||||
return (
|
||||
Object.keys(a).length === Object.keys(b).length &&
|
||||
Object.keys(a).every((id) => a[id] === b[id])
|
||||
);
|
||||
}
|
||||
|
||||
export function resolvePaths(
|
||||
proposed: FileManifestEntries,
|
||||
remote: FileManifestEntries,
|
||||
protectedIds: readonly string[] = [],
|
||||
reservedPaths: readonly string[] = []
|
||||
): FileManifestEntries {
|
||||
const result: FileManifestEntries = {};
|
||||
for (const id of protectedIds)
|
||||
if (proposed[id] !== undefined) result[id] = proposed[id];
|
||||
const occupied = { ...result };
|
||||
for (const [index, path] of reservedPaths.entries())
|
||||
if (!Object.values(result).includes(path))
|
||||
occupied[`reserved-${index}`] = path;
|
||||
const ids = Object.keys(proposed).sort(
|
||||
(a, b) =>
|
||||
Number(remote[b] === proposed[b]) -
|
||||
Number(remote[a] === proposed[a]) ||
|
||||
(a < b ? -1 : a > b ? 1 : 0)
|
||||
);
|
||||
for (const id of ids.filter(
|
||||
(candidate) => result[candidate] === undefined
|
||||
)) {
|
||||
result[id] = allocatePortablePath(proposed[id], id, occupied);
|
||||
occupied[id] = result[id];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function mergeFileManifests(
|
||||
base: FileManifestEntries,
|
||||
local: FileManifestEntries,
|
||||
remote: FileManifestEntries
|
||||
): FileManifestEntries {
|
||||
const proposed: FileManifestEntries = {};
|
||||
for (const id of new Set([
|
||||
...Object.keys(base),
|
||||
...Object.keys(local),
|
||||
...Object.keys(remote)
|
||||
])) {
|
||||
const path =
|
||||
local[id] === base[id]
|
||||
? remote[id]
|
||||
: remote[id] === base[id]
|
||||
? local[id]
|
||||
: remote[id];
|
||||
if (path !== undefined) proposed[id] = path;
|
||||
}
|
||||
return resolvePaths(proposed, remote);
|
||||
}
|
||||
82
frontend/sync-client/src/sync-operations/local-changes.ts
Normal file
82
frontend/sync-client/src/sync-operations/local-changes.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { v4 as uuid } from "uuid";
|
||||
import type { EngineState } from "../persistence/database";
|
||||
import type { VaultSnapshot } from "../services/types/VaultSnapshot";
|
||||
import { arePathAliases } from "../utils/portable-path";
|
||||
|
||||
export type LocalChange = {
|
||||
changeId?: string;
|
||||
/** Identities at notification time, before a remote application can reuse paths. */
|
||||
identities?: Record<string, string>;
|
||||
} & (
|
||||
| { type: "move"; oldPath: string; relativePath: string }
|
||||
| { type: "create"; path: string; documentId?: string }
|
||||
| { type: "delete"; path: string }
|
||||
| { type: "update"; path: string }
|
||||
);
|
||||
|
||||
export function unappliedChanges(
|
||||
state: EngineState,
|
||||
changes: readonly LocalChange[]
|
||||
): LocalChange[] {
|
||||
for (const change of changes) change.changeId ??= uuid();
|
||||
const applied = changes.findIndex(
|
||||
(change) => change.changeId === state.lastAppliedLocalChangeId
|
||||
);
|
||||
return changes.slice(applied + 1);
|
||||
}
|
||||
|
||||
export function applyLocalChange(
|
||||
next: EngineState,
|
||||
change: LocalChange,
|
||||
ignored: (path: string) => boolean,
|
||||
initial?: VaultSnapshot
|
||||
): void {
|
||||
if (change.type === "create") {
|
||||
if (
|
||||
ignored(change.path) ||
|
||||
Object.values(next.local).includes(change.path)
|
||||
)
|
||||
return;
|
||||
const id = (change.documentId ??=
|
||||
(initial &&
|
||||
Object.keys(initial.fileManifest.entries).find(
|
||||
(id) =>
|
||||
!next.local[id] &&
|
||||
arePathAliases(
|
||||
initial.fileManifest.entries[id],
|
||||
change.path
|
||||
)
|
||||
)) ||
|
||||
uuid());
|
||||
next.local[id] = change.path;
|
||||
next.documents[id] ??= {
|
||||
materialized: true,
|
||||
bootstrap: initial?.fileManifest.entries[id] !== undefined
|
||||
};
|
||||
return;
|
||||
}
|
||||
const from = change.type === "move" ? change.oldPath : change.path;
|
||||
change.identities ??= Object.fromEntries(
|
||||
Object.entries(next.local).filter(
|
||||
([id, path]) =>
|
||||
next.documents[id]?.materialized &&
|
||||
(path === from || path.startsWith(from + "/"))
|
||||
)
|
||||
);
|
||||
if (change.type === "update") return;
|
||||
for (const [id, path] of Object.entries(change.identities)) {
|
||||
if (change.type === "delete") {
|
||||
if (ignored(path)) {
|
||||
next.excluded ??= {};
|
||||
if (next.excluded[id] === undefined)
|
||||
next.excluded[id] = next.fileManifest.entries[id] ?? null;
|
||||
}
|
||||
delete next.local[id];
|
||||
} else {
|
||||
const to = change.relativePath + path.slice(from.length);
|
||||
for (const [other, otherPath] of Object.entries(next.local))
|
||||
if (other !== id && otherPath === to) delete next.local[other];
|
||||
next.local[id] = to;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { CursorTracker } from "./cursor-tracker";
|
||||
import { FileChangeNotifier } from "./file-change-notifier";
|
||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||
import { hash } from "../utils/hash";
|
||||
import type { Database, DocumentRecord } from "../persistence/database";
|
||||
import type { FileOperations } from "../file-operations/file-operations";
|
||||
import type { WebSocketManager } from "../services/websocket-manager";
|
||||
import type { DocumentWithCursors } from "../services/types/DocumentWithCursors";
|
||||
|
||||
for (const change of ["version", "identity", "path", "delete"] as const)
|
||||
test(`outgoing cursor is dirty when ${change} changes during its snapshot read`, async () => {
|
||||
// Equal bytes deliberately defeat a freshness check that compares only hashes.
|
||||
const content = new TextEncoder().encode("same bytes");
|
||||
const digest = await hash(content);
|
||||
let current: DocumentRecord | undefined = {
|
||||
documentId: "a",
|
||||
relativePath: "a.md",
|
||||
isDeleted: false,
|
||||
metadata: { parentVersionId: 1, hash: digest }
|
||||
};
|
||||
const sent: { documentsWithCursors: DocumentWithCursors[] }[] = [];
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners(),
|
||||
onRemoteCursorsUpdateReceived: new EventListeners(),
|
||||
updateLocalCursors: (data: {
|
||||
documentsWithCursors: DocumentWithCursors[];
|
||||
}) => sent.push(data)
|
||||
};
|
||||
const files = {
|
||||
read: async () => {
|
||||
if (change === "delete") current = undefined;
|
||||
else if (change === "version")
|
||||
current!.metadata!.parentVersionId = 2;
|
||||
else if (change === "identity")
|
||||
current!.documentId = "replacement";
|
||||
else current!.relativePath = "moved.md";
|
||||
return content;
|
||||
}
|
||||
};
|
||||
const tracker = new CursorTracker(
|
||||
{
|
||||
getDocumentByDocumentId: (id: string) =>
|
||||
current?.documentId === id ? current : undefined,
|
||||
getLatestDocumentByRelativePath: (path: string) =>
|
||||
current?.relativePath === path ? current : undefined
|
||||
} as unknown as Database,
|
||||
websocket as unknown as WebSocketManager,
|
||||
files as unknown as FileOperations,
|
||||
new FileChangeNotifier()
|
||||
);
|
||||
await tracker.sendLocalCursorsToServer({
|
||||
"a.md": [{ start: 4, end: 4 }]
|
||||
});
|
||||
assert.equal(sent.length, 1);
|
||||
assert.equal(sent[0].documentsWithCursors[0].vault_update_id, null);
|
||||
tracker.reset();
|
||||
});
|
||||
152
frontend/sync-client/src/sync-operations/scan.ts
Normal file
152
frontend/sync-client/src/sync-operations/scan.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import { v4 as uuid } from "uuid";
|
||||
import type { FileOperations } from "../file-operations/file-operations";
|
||||
import type { EngineState } from "../persistence/database";
|
||||
import type { VaultSnapshot } from "../services/types/VaultSnapshot";
|
||||
import { EMPTY_HASH, hash } from "../utils/hash";
|
||||
import {
|
||||
arePathAliases,
|
||||
findPathWithSameSpelling
|
||||
} from "../utils/portable-path";
|
||||
|
||||
import {
|
||||
applyLocalChange,
|
||||
unappliedChanges,
|
||||
type LocalChange
|
||||
} from "./local-changes";
|
||||
export type { LocalChange } from "./local-changes";
|
||||
|
||||
interface ScanOptions {
|
||||
next: EngineState;
|
||||
guard: () => void;
|
||||
changes: LocalChange[];
|
||||
files: FileOperations;
|
||||
ignored: (path: string) => boolean;
|
||||
oversized: (size: number) => boolean;
|
||||
commit: (next: EngineState) => Promise<void>;
|
||||
}
|
||||
|
||||
/** Scans are complete or fail. Missing unmaterialized/ignored entries are not deletions. */
|
||||
export async function scanLocalFiles(
|
||||
{
|
||||
next,
|
||||
guard,
|
||||
changes: queuedChanges,
|
||||
files,
|
||||
ignored,
|
||||
oversized,
|
||||
commit
|
||||
}: ScanOptions,
|
||||
initial?: VaultSnapshot
|
||||
): Promise<Set<string>> {
|
||||
// Leave the prefix queued until its resulting state is durably committed.
|
||||
// A persisted change ID makes replay safe after an uncertain save.
|
||||
const changes = [...queuedChanges];
|
||||
const unsyncablePaths = new Set<string>();
|
||||
for (const change of unappliedChanges(next, changes)) {
|
||||
applyLocalChange(next, change, ignored, initial);
|
||||
next.lastAppliedLocalChangeId = change.changeId;
|
||||
}
|
||||
const paths: string[] = [];
|
||||
const reportedPaths = await files.listFilesRecursively();
|
||||
for (const path of reportedPaths) {
|
||||
// Some filesystems may report a different Unicode spelling after a
|
||||
// rename. Keep an already-accessible spelling when possible.
|
||||
const alias = findPathWithSameSpelling(Object.values(next.local), path);
|
||||
paths.push(
|
||||
alias &&
|
||||
!reportedPaths.includes(alias) &&
|
||||
(await files.fs.exists(alias))
|
||||
? alias
|
||||
: path
|
||||
);
|
||||
}
|
||||
const present = new Set(paths);
|
||||
const snapshots = new Map<string, { hash: string }>();
|
||||
for (const path of paths) {
|
||||
if (ignored(path)) {
|
||||
unsyncablePaths.add(path);
|
||||
continue;
|
||||
}
|
||||
const info = await files.fs.stat(path);
|
||||
if (!info || info.kind !== "file")
|
||||
throw new Error(`File changed during scan: ${path}`);
|
||||
if (oversized(info.size)) {
|
||||
unsyncablePaths.add(path);
|
||||
continue;
|
||||
}
|
||||
const snapshot = await files.fs.readSnapshot(path);
|
||||
if (!snapshot) throw new Error(`File changed during scan: ${path}`);
|
||||
if (!oversized(snapshot.content.length)) {
|
||||
snapshots.set(path, { hash: await hash(snapshot.content) });
|
||||
} else {
|
||||
unsyncablePaths.add(path);
|
||||
}
|
||||
}
|
||||
const missing = new Set(
|
||||
Object.keys(next.local).filter(
|
||||
(id) =>
|
||||
next.documents[id]?.materialized &&
|
||||
!ignored(next.local[id]) &&
|
||||
!present.has(next.local[id])
|
||||
)
|
||||
);
|
||||
const knownPaths = new Set(Object.values(next.local));
|
||||
const unknown = [...snapshots.keys()].filter(
|
||||
(path) => !knownPaths.has(path)
|
||||
);
|
||||
for (const path of unknown) {
|
||||
const snapshot = snapshots.get(path)!;
|
||||
const matches = [...missing].filter(
|
||||
(id) => next.documents[id]?.observedHash === snapshot.hash
|
||||
);
|
||||
const sameHash = unknown.filter(
|
||||
(other) => snapshots.get(other)?.hash === snapshot.hash
|
||||
);
|
||||
let id: string | undefined;
|
||||
if (
|
||||
snapshot.hash !== EMPTY_HASH &&
|
||||
matches.length === 1 &&
|
||||
sameHash.length === 1
|
||||
) {
|
||||
id = matches[0];
|
||||
missing.delete(id);
|
||||
} else if (initial) {
|
||||
id = Object.keys(initial.fileManifest.entries).find(
|
||||
(candidate) =>
|
||||
!next.local[candidate] &&
|
||||
arePathAliases(
|
||||
initial.fileManifest.entries[candidate],
|
||||
path
|
||||
)
|
||||
);
|
||||
}
|
||||
id ??= uuid();
|
||||
next.local[id] = path;
|
||||
next.documents[id] ??= {
|
||||
materialized: true,
|
||||
bootstrap: initial?.fileManifest.entries[id] !== undefined
|
||||
};
|
||||
}
|
||||
for (const id of missing) delete next.local[id];
|
||||
for (const [id, path] of Object.entries(next.local)) {
|
||||
if (unsyncablePaths.has(path) || ignored(path)) {
|
||||
next.excluded ??= {};
|
||||
if (next.excluded[id] === undefined)
|
||||
next.excluded[id] = next.fileManifest.entries[id] ?? null;
|
||||
}
|
||||
const snapshot = snapshots.get(path);
|
||||
if (snapshot) {
|
||||
next.documents[id] ??= { materialized: true };
|
||||
if (!next.documents[id].materialized && !next.documents[id].base)
|
||||
next.documents[id].bootstrap = true;
|
||||
next.documents[id].materialized = true;
|
||||
next.documents[id].observedHash = snapshot.hash;
|
||||
}
|
||||
}
|
||||
next.protectedPaths = [...unsyncablePaths];
|
||||
guard();
|
||||
await commit(next);
|
||||
queuedChanges.splice(0, changes.length);
|
||||
guard();
|
||||
return unsyncablePaths;
|
||||
}
|
||||
351
frontend/sync-client/src/sync-operations/syncer.test.ts
Normal file
351
frontend/sync-client/src/sync-operations/syncer.test.ts
Normal file
|
|
@ -0,0 +1,351 @@
|
|||
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
|
||||
import assert from "node:assert";
|
||||
import { afterEach, beforeEach, describe, it, mock } from "node:test";
|
||||
import type { Database } from "../persistence/database";
|
||||
import {
|
||||
DEFAULT_SETTINGS,
|
||||
type Settings,
|
||||
type SyncSettings
|
||||
} from "../persistence/settings";
|
||||
import type { FileOperations } from "../file-operations/file-operations";
|
||||
import type { SyncService } from "../services/sync-service";
|
||||
import type { ServerConfig } from "../services/server-config";
|
||||
import type { WebSocketManager } from "../services/websocket-manager";
|
||||
import type { Logger } from "../tracing/logger";
|
||||
import type { SyncHistory } from "../tracing/sync-history";
|
||||
import type { EventBatch } from "../services/types/EventBatch";
|
||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||
import { Syncer } from "./syncer";
|
||||
import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache";
|
||||
import { toStoredSnapshot } from "./content";
|
||||
import { undiff } from "reconcile-text";
|
||||
|
||||
function createSyncer(overrides: Partial<SyncSettings> = {}): {
|
||||
syncer: Syncer;
|
||||
runCount: () => number;
|
||||
} {
|
||||
const settings = {
|
||||
getSettings: () => ({
|
||||
...DEFAULT_SETTINGS,
|
||||
...overrides
|
||||
})
|
||||
} as unknown as Settings;
|
||||
const database = {
|
||||
state: {}
|
||||
} as unknown as Database;
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteVaultUpdateReceived: new EventListeners<() => unknown>()
|
||||
} as unknown as WebSocketManager;
|
||||
const syncer = new Syncer(
|
||||
"device",
|
||||
{} as Logger,
|
||||
database,
|
||||
settings,
|
||||
{} as SyncService,
|
||||
websocket,
|
||||
{} as FileOperations,
|
||||
{} as ServerConfig,
|
||||
{} as SyncHistory,
|
||||
new FixedSizeDocumentCache(1024)
|
||||
);
|
||||
let runs = 0;
|
||||
const internals = syncer as unknown as {
|
||||
dirty: boolean;
|
||||
run: () => Promise<void>;
|
||||
};
|
||||
internals.run = async (): Promise<void> => {
|
||||
runs++;
|
||||
internals.dirty = false;
|
||||
};
|
||||
|
||||
return { syncer, runCount: () => runs };
|
||||
}
|
||||
|
||||
describe("Syncer interval", () => {
|
||||
beforeEach(() => {
|
||||
mock.timers.enable({ apis: ["setTimeout"] });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
mock.timers.reset();
|
||||
});
|
||||
|
||||
it("does not schedule a timer when syncIntervalMs is unset", async () => {
|
||||
const { syncer, runCount } = createSyncer({
|
||||
syncIntervalMs: undefined
|
||||
});
|
||||
|
||||
syncer.start();
|
||||
await syncer.waitUntilFinished();
|
||||
mock.timers.tick(60_000);
|
||||
|
||||
assert.strictEqual(runCount(), 1);
|
||||
await syncer.stop();
|
||||
});
|
||||
|
||||
it("keeps periodic reconciliation opt-in by default", async () => {
|
||||
assert.strictEqual(DEFAULT_SETTINGS.syncIntervalMs, undefined);
|
||||
const { syncer, runCount } = createSyncer();
|
||||
syncer.start();
|
||||
await syncer.waitUntilFinished();
|
||||
mock.timers.tick(60_000);
|
||||
assert.strictEqual(runCount(), 1);
|
||||
await syncer.stop();
|
||||
});
|
||||
|
||||
it("does not poll when explicitly disabled with zero", async () => {
|
||||
const { syncer, runCount } = createSyncer({ syncIntervalMs: 0 });
|
||||
syncer.start();
|
||||
await syncer.waitUntilFinished();
|
||||
mock.timers.tick(60_000);
|
||||
assert.strictEqual(runCount(), 1);
|
||||
await syncer.stop();
|
||||
});
|
||||
|
||||
it("wakes after the configured sync interval", async () => {
|
||||
const { syncer, runCount } = createSyncer({ syncIntervalMs: 25 });
|
||||
|
||||
syncer.start();
|
||||
await syncer.waitUntilFinished();
|
||||
mock.timers.tick(24);
|
||||
assert.strictEqual(runCount(), 1);
|
||||
|
||||
mock.timers.tick(1);
|
||||
assert.strictEqual(runCount(), 2);
|
||||
await syncer.waitUntilFinished();
|
||||
await syncer.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Syncer per-file state and event delivery", () => {
|
||||
const head = {
|
||||
vaultUpdateId: 1,
|
||||
documentId: "document",
|
||||
updatedDate: "2026-01-01T00:00:00Z",
|
||||
userId: "user",
|
||||
deviceId: "device",
|
||||
contentSize: 4
|
||||
};
|
||||
|
||||
function fixture() {
|
||||
const database = {
|
||||
state: {
|
||||
initialized: true,
|
||||
local: { document: "note.md" },
|
||||
documents: {
|
||||
document: {
|
||||
materialized: true,
|
||||
observedHash: "hash",
|
||||
base: { ...head, hash: "hash" }
|
||||
}
|
||||
},
|
||||
fileManifest: {
|
||||
fileManifestId: 1,
|
||||
entries: { document: "note.md" }
|
||||
},
|
||||
remoteHeads: { document: head },
|
||||
lastSeenUpdateId: 0
|
||||
},
|
||||
getLatestDocumentByRelativePath: (path: string) =>
|
||||
path === "note.md"
|
||||
? {
|
||||
documentId: "document",
|
||||
relativePath: path,
|
||||
isDeleted: false
|
||||
}
|
||||
: undefined
|
||||
} as unknown as Database;
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteVaultUpdateReceived: new EventListeners<
|
||||
(batch: EventBatch) => Promise<void>
|
||||
>()
|
||||
} as unknown as WebSocketManager;
|
||||
let httpEventsCalls = 0;
|
||||
const service = {
|
||||
events: async () => {
|
||||
httpEventsCalls++;
|
||||
return { headEventId: 0, events: [] };
|
||||
}
|
||||
} as unknown as SyncService;
|
||||
const syncer = new Syncer(
|
||||
"device",
|
||||
{} as Logger,
|
||||
database,
|
||||
{
|
||||
getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 })
|
||||
} as Settings,
|
||||
service,
|
||||
websocket,
|
||||
{} as FileOperations,
|
||||
{} as ServerConfig,
|
||||
{} as SyncHistory,
|
||||
new FixedSizeDocumentCache(1024)
|
||||
);
|
||||
(syncer as unknown as { hasScanned: boolean }).hasScanned = true;
|
||||
return {
|
||||
database,
|
||||
websocket,
|
||||
syncer,
|
||||
httpEventsCalls: () => httpEventsCalls
|
||||
};
|
||||
}
|
||||
|
||||
it("reports only a genuinely current path as up to date", () => {
|
||||
const { syncer } = fixture();
|
||||
assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true);
|
||||
assert.strictEqual(syncer.isDocumentUpToDate("missing.md"), false);
|
||||
|
||||
const internals = syncer as unknown as {
|
||||
dirty: boolean;
|
||||
unsyncablePaths: Set<string>;
|
||||
};
|
||||
internals.dirty = true;
|
||||
assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true);
|
||||
internals.unsyncablePaths.add("note.md");
|
||||
assert.strictEqual(syncer.isDocumentUpToDate("note.md"), false);
|
||||
});
|
||||
|
||||
it("consumes a complete WebSocket event batch without fetching it again", async () => {
|
||||
const { websocket, syncer, httpEventsCalls } = fixture();
|
||||
const batch: EventBatch = {
|
||||
headEventId: 1,
|
||||
events: [
|
||||
{
|
||||
eventId: 1,
|
||||
requestId: "00000000-0000-0000-0000-000000000001",
|
||||
type: "fileManifest",
|
||||
fileManifest: { fileManifestId: 1, entries: {} }
|
||||
}
|
||||
]
|
||||
};
|
||||
await websocket.onRemoteVaultUpdateReceived.triggerAsync(batch);
|
||||
|
||||
const selected = await (
|
||||
syncer as unknown as { nextEventBatch: () => Promise<EventBatch> }
|
||||
).nextEventBatch();
|
||||
assert.deepEqual(selected, batch);
|
||||
assert.strictEqual(httpEventsCalls(), 0);
|
||||
});
|
||||
|
||||
it("projects content responses to metadata before persistence", () => {
|
||||
const { syncer } = fixture();
|
||||
const projected = (
|
||||
syncer as unknown as {
|
||||
documentHead: (value: typeof head) => Record<string, unknown>;
|
||||
}
|
||||
).documentHead({ ...head, contentBase64: "secret" } as typeof head);
|
||||
assert.deepEqual(projected, head);
|
||||
assert.strictEqual("contentBase64" in projected, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Syncer diff uploads", () => {
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
function fixture(cacheSize = 1024): {
|
||||
syncer: Syncer;
|
||||
parent: Uint8Array;
|
||||
} {
|
||||
const cache = new FixedSizeDocumentCache(cacheSize);
|
||||
const parent = encoder.encode("one two three\n");
|
||||
const websocket = {
|
||||
onWebSocketStatusChanged: new EventListeners<
|
||||
(connected: boolean) => unknown
|
||||
>(),
|
||||
onRemoteVaultUpdateReceived: new EventListeners<() => unknown>()
|
||||
} as unknown as WebSocketManager;
|
||||
const syncer = new Syncer(
|
||||
"device",
|
||||
{} as Logger,
|
||||
{ state: {} } as Database,
|
||||
{
|
||||
getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 })
|
||||
} as Settings,
|
||||
{
|
||||
getDocumentVersionContent: async () => parent
|
||||
} as unknown as SyncService,
|
||||
websocket,
|
||||
{} as FileOperations,
|
||||
{
|
||||
getConfig: async () => ({ mergeableFileExtensions: ["md"] })
|
||||
} as ServerConfig,
|
||||
{} as SyncHistory,
|
||||
cache
|
||||
);
|
||||
return { syncer, parent };
|
||||
}
|
||||
|
||||
it("sends a diff when the fetched parent is still cached", async () => {
|
||||
const { syncer, parent } = fixture();
|
||||
const internals = syncer as unknown as {
|
||||
remoteSnapshot: (
|
||||
head: {
|
||||
vaultUpdateId: number;
|
||||
documentId: string;
|
||||
},
|
||||
path: string
|
||||
) => Promise<unknown>;
|
||||
pushContent: (
|
||||
path: string,
|
||||
parentVersionId: number,
|
||||
snapshot: Awaited<ReturnType<typeof toStoredSnapshot>>
|
||||
) => Promise<
|
||||
| { type: "Diff"; value: (number | string)[] }
|
||||
| { type: "Snapshot"; value: string }
|
||||
>;
|
||||
};
|
||||
await internals.remoteSnapshot(
|
||||
{ vaultUpdateId: 7, documentId: "document" },
|
||||
"note.md"
|
||||
);
|
||||
const changed = "one changed three\n";
|
||||
const payload = await internals.pushContent(
|
||||
"note.md",
|
||||
7,
|
||||
await toStoredSnapshot({ content: encoder.encode(changed) })
|
||||
);
|
||||
|
||||
assert.equal(payload.type, "Diff");
|
||||
if (payload.type === "Diff") {
|
||||
assert.equal(
|
||||
undiff(new TextDecoder().decode(parent), payload.value),
|
||||
changed
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to a snapshot when the parent cannot fit in the cache", async () => {
|
||||
const { syncer } = fixture(0);
|
||||
const internals = syncer as unknown as {
|
||||
remoteSnapshot: (
|
||||
head: {
|
||||
vaultUpdateId: number;
|
||||
documentId: string;
|
||||
},
|
||||
path: string
|
||||
) => Promise<unknown>;
|
||||
pushContent: (
|
||||
path: string,
|
||||
parentVersionId: number,
|
||||
snapshot: Awaited<ReturnType<typeof toStoredSnapshot>>
|
||||
) => Promise<{ type: string; value: unknown }>;
|
||||
};
|
||||
await internals.remoteSnapshot(
|
||||
{ vaultUpdateId: 7, documentId: "document" },
|
||||
"note.md"
|
||||
);
|
||||
const payload = await internals.pushContent(
|
||||
"note.md",
|
||||
7,
|
||||
await toStoredSnapshot({ content: encoder.encode("changed") })
|
||||
);
|
||||
|
||||
assert.equal(payload.type, "Snapshot");
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -3,6 +3,30 @@ import assert from "node:assert";
|
|||
import { EventListeners } from "./event-listeners";
|
||||
|
||||
describe("EventListeners", () => {
|
||||
it("isolates throwing and rejecting observers and still notifies later listeners", async () => {
|
||||
const failures: unknown[] = [];
|
||||
const listeners = new EventListeners<() => unknown>((error) =>
|
||||
failures.push(error)
|
||||
);
|
||||
const syncError = new Error("sync observer");
|
||||
const asyncError = new Error("async observer");
|
||||
let called = 0;
|
||||
listeners.add(() => {
|
||||
throw syncError;
|
||||
});
|
||||
listeners.add(async () => {
|
||||
throw asyncError;
|
||||
});
|
||||
listeners.add(() => {
|
||||
called++;
|
||||
});
|
||||
assert.doesNotThrow(() => {
|
||||
listeners.trigger();
|
||||
});
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
assert.equal(called, 1);
|
||||
assert.deepEqual(failures, [syncError, asyncError]);
|
||||
});
|
||||
it("should add & remove listeners", () => {
|
||||
const listeners = new EventListeners<() => void>();
|
||||
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
||||
|
|
|
|||
|
|
@ -8,50 +8,65 @@ import { awaitAll } from "../await-all";
|
|||
export class EventListeners<TListener extends (...args: any[]) => any> {
|
||||
private readonly listeners: TListener[] = [];
|
||||
|
||||
public constructor(
|
||||
private readonly onError: (error: unknown) => void = (error) => {
|
||||
console.error("Event observer failed", error);
|
||||
}
|
||||
) {}
|
||||
|
||||
public get count(): number {
|
||||
return this.listeners.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new listener to the collection.
|
||||
*
|
||||
* @param listener The listener callback to add
|
||||
* @returns An unsubscribe function that removes this listener when called
|
||||
*/
|
||||
* Adds a new listener to the collection.
|
||||
*
|
||||
* @param listener The listener callback to add
|
||||
* @returns An unsubscribe function that removes this listener when called
|
||||
*/
|
||||
public add(listener: TListener): () => void {
|
||||
this.listeners.push(listener);
|
||||
return () => this.remove(listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a listener from the collection.
|
||||
*
|
||||
* @param listener The listener callback to remove
|
||||
* @returns true if the listener was found and removed, false otherwise
|
||||
*/
|
||||
* Removes a listener from the collection.
|
||||
*
|
||||
* @param listener The listener callback to remove
|
||||
* @returns true if the listener was found and removed, false otherwise
|
||||
*/
|
||||
public remove(listener: TListener): boolean {
|
||||
return removeFromArray(this.listeners, listener);
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers all listeners synchronously with the provided arguments.
|
||||
* Any returned promises are ignored. Use triggerAsync() to await them.
|
||||
*
|
||||
* @param args The arguments to pass to each listener
|
||||
*/
|
||||
* Triggers all listeners synchronously with the provided arguments.
|
||||
* Observer failures are reported without interrupting delivery or background
|
||||
* work. Returned promises are observed; use triggerAsync() to await them.
|
||||
*
|
||||
* @param args The arguments to pass to each listener
|
||||
*/
|
||||
public trigger(...args: Parameters<TListener>): void {
|
||||
this.listeners.forEach((listener) => {
|
||||
listener(...args);
|
||||
});
|
||||
for (const listener of [...this.listeners]) {
|
||||
try {
|
||||
void Promise.resolve(listener(...args)).catch(
|
||||
(error: unknown) => {
|
||||
this.reportFailure(error);
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
this.reportFailure(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Triggers all listeners and awaits any promises they return.
|
||||
* Synchronous listeners are called immediately, and any async listeners
|
||||
* are awaited in parallel.
|
||||
*
|
||||
* @param args The arguments to pass to each listener
|
||||
*/
|
||||
* Triggers all listeners and awaits any promises they return.
|
||||
* Synchronous listeners are called immediately, and any async listeners
|
||||
* are awaited in parallel.
|
||||
*
|
||||
* @param args The arguments to pass to each listener
|
||||
*/
|
||||
public async triggerAsync(...args: Parameters<TListener>): Promise<void> {
|
||||
await awaitAll(
|
||||
this.listeners
|
||||
|
|
@ -68,4 +83,11 @@ export class EventListeners<TListener extends (...args: any[]) => any> {
|
|||
public clear(): void {
|
||||
this.listeners.length = 0;
|
||||
}
|
||||
private reportFailure(error: unknown): void {
|
||||
try {
|
||||
this.onError(error);
|
||||
} catch {
|
||||
// A diagnostic callback must not break background work either.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue