andras/vault-link
andras/vault-link
Watch
1
0
Fork
You've already forked vault-link
 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 31

.

Browse source
This commit is contained in:
Andras Schmelczer 2026-09-26 20:22:16 +01:00
commit b07aa589ff
20 changed files with 728 additions and 530 deletions
Download patch file Download diff file Expand all files Collapse all files

2
frontend/sync-client/src/consts.ts
View file

@ -3,3 +3,5 @@ export const MAX_LOG_MESSAGE_COUNT = 100000;
export const MAX_HISTORY_ENTRY_COUNT = 5000;
export const SUPPORTED_API_VERSION = 4;
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
export const CURSOR_HEARTBEAT_INTERVAL_MS = 15_000;
export const WEBSOCKET_RECEIVE_TIMEOUT_MS = CURSOR_HEARTBEAT_INTERVAL_MS * 3;

33
frontend/sync-client/src/file-operations/file-operations.ts
View file

@ -1,20 +1,15 @@
import { reconcile } from "reconcile-text";
import type {
FileSnapshot,
FileSystemOperations
} from "./filesystem-operations";
import type {
Database,
EngineState,
StoredSnapshot
} from "../persistence/database";
import { isInternalPath } from "../utils/portable-path";
import { FilePaths } from "./file-paths";
import type { FileSystemOperations } from "./filesystem-operations";
import type { Database, EngineState } from "../persistence/database";
import {
fromStoredSnapshot,
toStoredSnapshot,
mergeContent
} from "../sync-operations/content";
type FileSnapshot,
type StoredSnapshot
} from "../snapshot";
import { isInternalPath } from "../utils/portable-path";
import { FilePaths } from "./file-paths";
import { mergeContent } from "../sync-operations/content";
import { hash } from "../utils/hash";
import { isBinary } from "../utils/is-binary";
import { isFileTypeMergeable } from "../utils/is-file-type-mergeable";
@ -89,7 +84,7 @@ export class FileOperations {
): Promise<void> {
const state = structuredClone(this.applying ?? this.database.state);
if (!state.initialized) return;
if (state.lastSeenUpdateId === undefined) return;
for (const pending of unappliedChanges(state, previous))
applyLocalChange(state, pending, () => false);
@ -118,7 +113,7 @@ export class FileOperations {
};
guard();
const current = structuredClone(this.database.state);
current.initialized = next.initialized;
current.lastSeenUpdateId = next.lastSeenUpdateId;
this.applying = current;
const paths = new Set<string>();
const observeChanges = async (): Promise<void> => {
@ -145,7 +140,7 @@ export class FileOperations {
...Object.keys(next.local)
])) {
await observeChanges();
let from = current.documents[id]?.materialized
let from = current.documents[id]?.observedHash !== undefined
? current.local[id]
: undefined;
let to = next.local[id];
@ -166,7 +161,6 @@ export class FileOperations {
);
(next.excluded ??= {})[id] =
current.fileManifest.entries[id] ?? null;
(next.protectedPaths ??= []).push(from);
continue;
}
}
@ -187,7 +181,7 @@ export class FileOperations {
mutate
);
next.local[id] = to;
from = current.documents[id]?.materialized
from = current.documents[id]?.observedHash !== undefined
? current.local[id]
: undefined;
if (from !== undefined) {
@ -211,7 +205,7 @@ export class FileOperations {
.mergeableFileExtensions;
await mutate(async () => {
current.local[id] = to;
current.documents[id] = { materialized: true };
current.documents[id] = { observedHash: "" };
next.documents[id].observedHash =
await this.updateContent(
to,
@ -222,7 +216,6 @@ export class FileOperations {
});
}
paths.add(to);
next.documents[id].materialized = true;
current.local[id] = to;
current.documents[id] = structuredClone(next.documents[id]);
}

7
frontend/sync-client/src/file-operations/file-paths.ts
View file

@ -29,13 +29,12 @@ export class FilePaths {
for (const child of children) {
const occupant = Object.keys(current.local).find(
(key) =>
current.documents[key]?.materialized &&
current.documents[key]?.observedHash !== undefined &&
current.local[key] === child
);
const info = await this.fs.stat(child);
if (
!info ||
next.protectedPaths?.includes(child) ||
this.isProtectedFile(child, info.size) ||
(obstruction.kind === "directory" &&
(occupant === undefined ||
@ -59,8 +58,8 @@ export class FilePaths {
await this.fs.rename(occupant.path, displaced);
});
if (!current.documents[occupant.id]) {
current.documents[occupant.id] = { materialized: true };
next.documents[occupant.id] = { materialized: true };
current.documents[occupant.id] = { observedHash: "" };
next.documents[occupant.id] = { observedHash: "" };
next.local[occupant.id] = displaced;
} else if (next.local[occupant.id] === occupant.path)
next.local[occupant.id] = displaced;

9
frontend/sync-client/src/file-operations/filesystem-operations.ts
View file

@ -1,11 +1,6 @@
import type { CursorPosition, TextWithCursors } from "reconcile-text";
import type { TextWithCursors } from "reconcile-text";
import type { RelativePath } from "../persistence/database";
export interface FileSnapshot {
content: Uint8Array;
/** Optional editor selections, moved with the file and applied with its bytes. */
cursors?: CursorPosition[];
}
import type { FileSnapshot } from "../snapshot";
/** Filesystem operations act on the current user-owned namespace.
* Paths are vault-relative. Never follow symlinks (including ancestor symlinks).

6
frontend/sync-client/src/index.ts
View file

@ -22,10 +22,8 @@ export { Logger, LogLevel, LogLine } from "./tracing/logger";
export { type SyncSettings, DEFAULT_SETTINGS } from "./persistence/settings";
export { rateLimit } from "./utils/rate-limit";
export type { RelativePath, StoredDatabase } from "./persistence/database";
export type {
FileSystemOperations,
FileSnapshot
} from "./file-operations/filesystem-operations";
export type { FileSystemOperations } from "./file-operations/filesystem-operations";
export type { FileSnapshot } from "./snapshot";
export type { MetadataPersistenceProvider as PersistenceProvider } from "./persistence/metadata-persistence-provider";
export type { CursorSpan } from "./services/types/CursorSpan";
export type { ClientCursors } from "./services/types/ClientCursors";

94
frontend/sync-client/src/persistence/database.test.ts Normal file
View file

@ -0,0 +1,94 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { Database, emptyState, type StoredDatabase } from "./database";
test("old state loads without duplicate flags, full recovery content, or acknowledged heads", async () => {
const head = {
documentId: "clean",
vaultUpdateId: 1,
contentSize: 3,
updatedDate: "2026-09-23",
userId: "user",
deviceId: "device"
};
const legacy = {
...emptyState("vault"),
initialized: true,
lastSeenUpdateId: 2,
protectedPaths: ["ignored.md"],
local: {
clean: "clean.md",
recovering: "recovering.md",
deferred: "deferred.md"
},
documents: {
clean: {
materialized: true,
observedHash: "clean",
base: { ...head, hash: "clean" }
},
recovering: {
materialized: true,
observedHash: "edited",
recoveryBase: { contentBase64: "T0xE", hash: "old" }
},
deferred: { materialized: false, observedHash: undefined }
},
remoteHeads: {
clean: head,
deferred: { ...head, documentId: "deferred" }
}
};
const saves: StoredDatabase[] = [];
const db = new Database(
legacy,
async (next) => {
saves.push(next);
},
"vault",
async () => saves.at(-1)
);
await db.save();
const saved = saves.at(-1);
assert(saved);
assert.deepEqual(saved.documents, {
clean: {
base: { vaultUpdateId: 1, hash: "clean" },
observedHash: "clean"
},
recovering: {
base: { vaultUpdateId: null, hash: "old" },
observedHash: "edited"
},
deferred: {
base: undefined,
observedHash: undefined,
remote: { ...head, documentId: "deferred" }
}
});
assert.equal(saved.lastSeenUpdateId, 2);
for (const field of ["initialized", "remoteHeads", "protectedPaths"])
assert(!(field in saved));
assert(!JSON.stringify(saved).includes("contentBase64"));
});
test("old reset intent leaves the cursor absent after an uncertain save reload", async () => {
const legacy = {
...emptyState("vault"),
initialized: false,
historyRecovery: true,
lastSeenUpdateId: 0
};
const db = new Database(
undefined,
async () => {
throw new Error("interrupted");
},
"vault",
async () => legacy
);
await assert.rejects(db.save(), /interrupted/);
await db.recoverPersistence();
assert.equal(db.state.lastSeenUpdateId, undefined);
assert(!("historyRecovery" in db.state));
});

70
frontend/sync-client/src/persistence/database.ts
View file

@ -1,38 +1,29 @@
import type { CursorPosition } from "reconcile-text";
import { upgradeState } from "./upgrade-state";
import type { StoredSnapshot } from "../snapshot";
import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
import type { FileManifest } from "../services/types/FileManifest";
import type { FileManifestUpdateResponse } from "../services/types/FileManifestUpdateResponse";
import type { PushFileManifest } from "../services/types/PushFileManifest";
import type { PutFileContent } from "../services/types/PutFileContent";
import type { VaultSnapshot } from "../services/types/VaultSnapshot";
import type { FileManifestEntries } from "../types/file-manifest-entries";
export type VaultUpdateId = number;
export type DocumentId = string;
export type RelativePath = string;
export interface StoredSnapshot {
contentBase64: string;
hash: string;
cursors?: CursorPosition[];
}
export interface DocumentMetadata {
parentVersionId: number;
hash: string;
}
export interface DocumentState {
/** Local clean base retained while rebasing after a server restore. */
recoveryBase?: StoredSnapshot;
base?: DocumentVersionWithoutContent & { hash: string };
/** A null version retains only the clean hash after a server history reset. */
base?: { vaultUpdateId: number | null; hash: string };
/** Last observed disk hash; an empty string means a notification awaits scanning. */
observedHash?: string;
materialized: boolean;
/** First contact with existing local content: there is no common ancestor. */
bootstrap?: boolean;
/** Do not resubmit an unchanged payload rejected by the server. */
rejected?: { hash: string; message: string };
/** A remote version still to be incorporated, including excluded downloads. */
remote?: DocumentVersionWithoutContent;
}
export type PendingRequest = { rejection?: string } & (
@ -51,26 +42,19 @@ export type PendingRequest = { rejection?: string } & (
);
export interface EngineState {
/** Persisted reset intent; clear the transport checkpoint before bootstrap. */
historyRecovery?: boolean;
vaultKey: string;
initialized: boolean;
fileManifest: FileManifest;
local: FileManifestEntries;
documents: Record<string, DocumentState>;
lastSeenUpdateId: number;
remoteHeads: Record<string, DocumentVersionWithoutContent>;
bootstrap?: VaultSnapshot;
/** Absent until a vault snapshot is applied; also marks a durable history reset. */
lastSeenUpdateId?: number;
pending?: PendingRequest;
/** A rejected retry does not prove an earlier attempt failed to commit. */
unconfirmed?: PendingRequest[];
/** Commits consumption of the saved notification queue with the identity map. */
lastAppliedLocalChangeId?: string;
/** Physical exclusions, including files with no sync identity. */
protectedPaths?: string[];
/** Original server paths while local files are excluded; null means absent. */
excluded?: Record<string, string | null>;
rejectedManifest?: { entries: FileManifestEntries; message: string };
}
export type StoredDatabase = EngineState;
@ -85,21 +69,17 @@ export interface DocumentRecord {
export function emptyState(vaultKey: string): StoredDatabase {
return {
vaultKey,
initialized: false,
fileManifest: { fileManifestId: 0, entries: {} },
local: {},
documents: {},
remoteHeads: {},
lastSeenUpdateId: 0
documents: {}
};
}
function canRebindEmpty(state: Partial<StoredDatabase> | undefined): boolean {
return (
state !== undefined &&
!state.initialized &&
state.lastSeenUpdateId === undefined &&
!state.pending &&
!state.bootstrap &&
(state.unconfirmed?.length ?? 0) === 0 &&
Object.keys(state.local ?? {}).length === 0
);
@ -114,19 +94,16 @@ export class Database {
vaultKey: string,
private readonly loadData: () => Promise<StoredDatabase | undefined>
) {
const hasState = initial && Object.keys(initial).length > 0;
if (
hasState &&
initial.vaultKey !== vaultKey &&
!canRebindEmpty(initial)
) {
const state =
initial !== undefined && Object.keys(initial).length > 0
? upgradeState(initial as StoredDatabase)
: emptyState(vaultKey);
if (state.vaultKey !== vaultKey && !canRebindEmpty(state)) {
throw new Error(
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
);
}
this.state = hasState
? { ...(initial as StoredDatabase), vaultKey }
: emptyState(vaultKey);
this.state = { ...state, vaultKey };
}
/** Bind an empty database to the configured vault. */
@ -154,7 +131,7 @@ export class Database {
public async recoverPersistence(): Promise<void> {
if (!this.needsReload) return;
const saved = await this.loadData();
if (saved) this.state = saved;
if (saved) this.state = upgradeState(saved);
this.needsReload = false;
}
@ -173,10 +150,13 @@ export class Database {
return {
documentId: id,
relativePath,
metadata: base && {
parentVersionId: base.vaultUpdateId,
hash: base.hash
}
metadata:
base?.vaultUpdateId != null
? {
parentVersionId: base.vaultUpdateId,
hash: base.hash
}
: undefined
};
}

57
frontend/sync-client/src/persistence/upgrade-state.ts Normal file
View file

@ -0,0 +1,57 @@
import type { DocumentState, EngineState } from "./database";
import type { StoredSnapshot } from "../snapshot";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
/** Read the previous redundant state once; all subsequent saves use the smaller format. */
export function upgradeState(state: EngineState): EngineState {
if (!("initialized" in state)) return state;
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Read the previous persisted schema at the migration boundary.
const legacy = state as EngineState & {
initialized: boolean;
historyRecovery?: boolean;
remoteHeads?: Record<string, DocumentVersionWithoutContent>;
documents: Record<
string,
DocumentState & {
materialized?: boolean;
bootstrap?: boolean;
recoveryBase?: StoredSnapshot;
}
>;
};
const next: EngineState = {
vaultKey: state.vaultKey,
fileManifest: state.fileManifest,
local: state.local,
documents: {},
lastSeenUpdateId:
legacy.initialized && legacy.historyRecovery !== true
? state.lastSeenUpdateId
: undefined,
pending: state.pending,
unconfirmed: state.unconfirmed,
lastAppliedLocalChangeId: state.lastAppliedLocalChangeId,
excluded: state.excluded
};
for (const [id, doc] of Object.entries(legacy.documents)) {
const base = doc.recoveryBase
? { vaultUpdateId: null, hash: doc.recoveryBase.hash }
: doc.base && doc.bootstrap !== true
? { vaultUpdateId: doc.base.vaultUpdateId, hash: doc.base.hash }
: undefined;
next.documents[id] = {
base,
observedHash:
doc.materialized === true ? (doc.observedHash ?? "") : undefined
};
}
for (const [id, remote] of Object.entries(legacy.remoteHeads ?? {})) {
const doc = (next.documents[id] ??= {});
if (
doc.observedHash === undefined ||
remote.vaultUpdateId > (doc.base?.vaultUpdateId ?? 0)
)
doc.remote = remote;
}
return next;
}

94
frontend/sync-client/src/services/websocket-manager.test.ts
View file

@ -3,6 +3,7 @@ import { describe, it, beforeEach } from "node:test";
import assert from "node:assert";
import { createServer } from "node:net";
import { WebSocketManager } from "./websocket-manager";
import { WEBSOCKET_RECEIVE_TIMEOUT_MS } from "../consts";
import { Logger } from "../tracing/logger";
import { Settings } from "../persistence/settings";
// eslint-disable-next-line @typescript-eslint/no-require-imports
@ -502,3 +503,96 @@ it("a handshake send failure retires the connection without escaping the open ca
await manager.stop();
}
});
it("reconnects a connection that stops delivering messages", async (context) => {
context.mock.timers.enable({ apis: ["setTimeout"] });
let attempts = 0;
class CountingSocket extends MockWebSocket {
static OPEN = 1;
constructor(url: string) {
super(url);
attempts++;
}
}
const logger = new Logger();
const settings = new Settings(
logger,
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
async () => {}
);
const manager = new WebSocketManager(
"test",
logger,
settings,
CountingSocket as unknown as typeof WebSocket
);
const statuses: boolean[] = [];
manager.onWebSocketStatusChanged.add((status) => statuses.push(status));
try {
manager.start();
context.mock.timers.tick(0);
assert.equal(manager.isWebSocketConnected, true);
assert.equal(attempts, 1);
const silentSocket = (
manager as unknown as { webSocket: MockWebSocket }
).webSocket;
context.mock.timers.tick(WEBSOCKET_RECEIVE_TIMEOUT_MS);
assert.equal(manager.isWebSocketConnected, false);
assert.equal(statuses.at(-1), false);
assert.equal(silentSocket.onmessage, null);
context.mock.timers.tick(1000);
context.mock.timers.tick(0);
assert.equal(attempts, 2);
assert.equal(manager.isWebSocketConnected, true);
} finally {
await manager.stop();
}
});
it("does not reconnect while messages keep arriving before the deadline", async (context) => {
context.mock.timers.enable({ apis: ["setTimeout"] });
let attempts = 0;
class CountingSocket extends MockWebSocket {
static OPEN = 1;
constructor(url: string) {
super(url);
attempts++;
}
}
const logger = new Logger();
const settings = new Settings(
logger,
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
async () => {}
);
const manager = new WebSocketManager(
"test",
logger,
settings,
CountingSocket as unknown as typeof WebSocket
);
let disconnected = 0;
manager.onWebSocketStatusChanged.add((status) => {
if (!status) disconnected++;
});
try {
manager.start();
context.mock.timers.tick(0);
const socket = (manager as unknown as { webSocket: MockWebSocket })
.webSocket;
assert.equal(attempts, 1);
context.mock.timers.tick(10_000);
socket.simulateMessage({ type: "vaultChanged" });
context.mock.timers.tick(10_000);
socket.simulateMessage({ type: "cursorPositions", clients: [] });
context.mock.timers.tick(WEBSOCKET_RECEIVE_TIMEOUT_MS - 20_000);
assert.equal(manager.isWebSocketConnected, true);
assert.equal(disconnected, 0);
assert.equal(attempts, 1);
} finally {
await manager.stop();
}
});

10
frontend/sync-client/src/services/websocket-manager.ts
View file

@ -5,7 +5,7 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage";
import type { CursorPositionFromClient } from "./types/CursorPositionFromClient";
import type { ClientCursors } from "./types/ClientCursors";
import { createPromise } from "../utils/create-promise";
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts";
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S, WEBSOCKET_RECEIVE_TIMEOUT_MS } from "../consts";
import { removeFromArray } from "../utils/remove-from-array";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { awaitAll } from "../utils/await-all";
@ -177,13 +177,15 @@ export class WebSocketManager {
private expectResponse(socket: WebSocket): void {
clearTimeout(this.receiveTimeoutId);
// Cursor heartbeats are acknowledged every 15 seconds, including when
// there are no open editors. An OPEN TCP socket alone proves nothing.
// Cursor heartbeats are acknowledged every CURSOR_HEARTBEAT_INTERVAL_MS,
// including when there are no open editors. An OPEN TCP socket alone
// proves nothing: silence past the heartbeat deadline means a
// half-open connection.
this.receiveTimeoutId = setTimeout(() => {
if (this.webSocket !== socket || this.isStopped) return;
this.logger.warn("WebSocket receive deadline expired");
this.reconnect(socket);
}, 45_000);
}, WEBSOCKET_RECEIVE_TIMEOUT_MS);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Browser timers have no unref method.
this.receiveTimeoutId.unref?.();
}

32
frontend/sync-client/src/snapshot.ts Normal file
View file

@ -0,0 +1,32 @@
import { base64ToBytes, bytesToBase64 } from "byte-base64";
import type { CursorPosition } from "reconcile-text";
import { hash } from "./utils/hash";
export interface FileSnapshot {
content: Uint8Array;
/** Optional editor selections, moved with the file and applied with its bytes. */
cursors?: CursorPosition[];
}
export interface StoredSnapshot {
contentBase64: string;
hash: string;
cursors?: CursorPosition[];
}
export async function toStoredSnapshot(
snapshot: FileSnapshot
): Promise<StoredSnapshot> {
return {
contentBase64: bytesToBase64(snapshot.content),
cursors: snapshot.cursors,
hash: await hash(snapshot.content)
};
}
export function fromStoredSnapshot(snapshot: StoredSnapshot): FileSnapshot {
return {
content: base64ToBytes(snapshot.contentBase64),
cursors: snapshot.cursors
};
}

3
frontend/sync-client/src/sync-client.ts
View file

@ -320,8 +320,7 @@ export class SyncClient {
if (
vaultKey(next) !== vaultKey(old) &&
(this.started ||
this.database.state.initialized ||
this.database.state.bootstrap ||
this.database.state.lastSeenUpdateId !== undefined ||
this.database.state.pending ||
this.syncer.hasLocalChanges ||
Object.keys(this.database.state.local).length)

3
frontend/sync-client/src/sync-operations/content.test.ts
View file

@ -1,6 +1,7 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { mergeContent, fromStoredSnapshot, toStoredSnapshot } from "./content";
import { fromStoredSnapshot, toStoredSnapshot } from "../snapshot";
import { mergeContent } from "./content";
const extensions = ["md"];
const snapshot = async (text: string) =>

23
frontend/sync-client/src/sync-operations/content.ts
View file

@ -1,28 +1,9 @@
import { base64ToBytes, bytesToBase64 } from "byte-base64";
import { base64ToBytes } from "byte-base64";
import { reconcile } from "reconcile-text";
import type { FileSnapshot } from "../file-operations/filesystem-operations";
import type { StoredSnapshot } from "../persistence/database";
import { hash } from "../utils/hash";
import { toStoredSnapshot, type StoredSnapshot } from "../snapshot";
import { isBinary } from "../utils/is-binary";
import { isFileTypeMergeable } from "../utils/is-file-type-mergeable";
export async function toStoredSnapshot(
snapshot: FileSnapshot
): Promise<StoredSnapshot> {
return {
contentBase64: bytesToBase64(snapshot.content),
cursors: snapshot.cursors,
hash: await hash(snapshot.content)
};
}
export function fromStoredSnapshot(snapshot: StoredSnapshot): FileSnapshot {
return {
content: base64ToBytes(snapshot.contentBase64),
cursors: snapshot.cursors
};
}
export async function mergeContent(
path: string,
base: StoredSnapshot | undefined,

6
frontend/sync-client/src/sync-operations/cursor-tracker.ts
View file

@ -10,6 +10,7 @@ import { hash } from "../utils/hash";
import type { FileChangeNotifier } from "./file-change-notifier";
import { Lock } from "../utils/data-structures/locks";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { CURSOR_HEARTBEAT_INTERVAL_MS } from "../consts";
// Cursor positions are updated separately from documents. However, a given cursor position is only
// valid within a certain version of the document it belongs to. This class tracks previous and the latest
@ -60,7 +61,10 @@ export class CursorTracker {
});
};
refresh();
this.cursorTimer = setInterval(refresh, 15_000);
this.cursorTimer = setInterval(
refresh,
CURSOR_HEARTBEAT_INTERVAL_MS
);
this.cursorTimer.unref?.();
});
this.webSocketManager.onRemoteCursorsUpdateReceived.add(

18
frontend/sync-client/src/sync-operations/file-manifest.ts
View file

@ -34,7 +34,7 @@ export function resolvePaths(
const ids = Object.keys(proposed).sort(
(a, b) =>
Number(remote[b] === proposed[b]) -
Number(remote[a] === proposed[a]) ||
Number(remote[a] === proposed[a]) ||
(a < b ? -1 : a > b ? 1 : 0)
);
for (const id of ids.filter(
@ -57,12 +57,16 @@ export function mergeFileManifests(
...Object.keys(local),
...Object.keys(remote)
])) {
const path =
local[id] === base[id]
? remote[id]
: remote[id] === base[id]
? local[id]
: remote[id];
let path: string | undefined;
if (local[id] === base[id]) {
path = remote[id];
} else if (remote[id] === base[id]) {
path = local[id];
} else {
path = remote[id];
}
if (path !== undefined) proposed[id] = path;
}
return resolvePaths(proposed, remote);

7
frontend/sync-client/src/sync-operations/local-changes.ts
View file

@ -48,17 +48,14 @@ export function applyLocalChange(
)) ||
uuid());
next.local[id] = change.path;
next.documents[id] ??= {
materialized: true,
bootstrap: initial?.fileManifest.entries[id] !== undefined
};
next.documents[id] ??= { observedHash: "" };
return;
}
const from = change.type === "move" ? change.oldPath : change.path;
change.identities ??= Object.fromEntries(
Object.entries(next.local).filter(
([id, path]) =>
next.documents[id]?.materialized &&
next.documents[id]?.observedHash !== undefined &&
(path === from || path.startsWith(from + "/"))
)
);

25
frontend/sync-client/src/sync-operations/scan.ts
View file

@ -85,7 +85,7 @@ export async function scanLocalFiles(
const missing = new Set(
Object.keys(next.local).filter(
(id) =>
next.documents[id]?.materialized &&
next.documents[id]?.observedHash !== undefined &&
!ignored(next.local[id]) &&
!present.has(next.local[id])
)
@ -122,10 +122,7 @@ export async function scanLocalFiles(
}
id ??= uuid();
next.local[id] = path;
next.documents[id] ??= {
materialized: true,
bootstrap: initial?.fileManifest.entries[id] !== undefined
};
next.documents[id] ??= { observedHash: "" };
}
for (const id of missing) delete next.local[id];
for (const [id, path] of Object.entries(next.local)) {
@ -136,14 +133,20 @@ export async function scanLocalFiles(
}
const snapshot = snapshots.get(path);
if (snapshot) {
next.documents[id] ??= { materialized: true };
if (!next.documents[id].materialized && !next.documents[id].base)
next.documents[id].bootstrap = true;
next.documents[id].materialized = true;
next.documents[id].observedHash = snapshot.hash;
const doc = (next.documents[id] ??= {});
// The old history no longer contains this base. Preserve any edit
// made during recovery as a new file instead of overwriting it.
if (
doc.base?.vaultUpdateId === null &&
doc.base.hash !== snapshot.hash
) {
const recoveredId = uuid();
next.local[recoveredId] = path;
next.documents[recoveredId] = { observedHash: snapshot.hash };
delete next.local[id];
} else doc.observedHash = snapshot.hash;
}
}
next.protectedPaths = [...unsyncablePaths];
guard();
await commit(next);
queuedChanges.splice(0, changes.length);

7
frontend/sync-client/src/sync-operations/syncer.test.ts
View file

@ -16,7 +16,7 @@ import type { SyncHistory } from "../tracing/sync-history";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { Syncer } from "./syncer";
import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache";
import { toStoredSnapshot } from "./content";
import { toStoredSnapshot } from "../snapshot";
import { undiff } from "reconcile-text";
function createSyncer(overrides: Partial<SyncSettings> = {}): {
@ -131,11 +131,11 @@ describe("Syncer per-file state and event delivery", () => {
function fixture() {
const database = {
state: {
initialized: true,
local: { document: "note.md" },
documents: {
document: {
materialized: true,
observedHash: "hash",
base: { ...head, hash: "hash" }
}
@ -144,7 +144,6 @@ describe("Syncer per-file state and event delivery", () => {
fileManifestId: 1,
entries: { document: "note.md" }
},
remoteHeads: { document: head },
lastSeenUpdateId: 0
},
getLatestDocumentByRelativePath: (path: string) =>

535
frontend/sync-client/src/sync-operations/syncer.ts
View file

@ -6,9 +6,10 @@ import { diff } from "reconcile-text";
import type {
Database,
EngineState,
StoredSnapshot
PendingRequest
} from "../persistence/database";
import { emptyState } from "../persistence/database";
import { toStoredSnapshot, type StoredSnapshot } from "../snapshot";
import type {
FileOperations,
FileWrite
@ -17,7 +18,6 @@ import type { Settings } from "../persistence/settings";
import type { SyncService } from "../services/sync-service";
import type { ServerConfig } from "../services/server-config";
import type { WebSocketManager } from "../services/websocket-manager";
import type { DocumentVersion } from "../services/types/DocumentVersion";
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
import type { EventBatch } from "../services/types/EventBatch";
import type { EventRecord } from "../services/types/EventRecord";
@ -27,7 +27,7 @@ import type { PushContent } from "../services/types/PushContent";
import type { Logger } from "../tracing/logger";
import { EventListeners } from "../utils/data-structures/event-listeners";
import { VaultHistoryStatus } from "../types/vault-history-status";
import { mergeContent, toStoredSnapshot } from "./content";
import { mergeContent } from "./content";
import { scanLocalFiles, type LocalChange } from "./scan";
import {
mergeFileManifests,
@ -83,7 +83,7 @@ export class Syncer {
private readonly localChangeStorage: {
entries: LocalChange[];
save: (entries: LocalChange[]) => Promise<void>;
} = { entries: [], save: async () => { } }
} = { entries: [], save: async () => {} }
) {
this.changes = localChangeStorage.entries;
websocket.onWebSocketStatusChanged.add((connected) => {
@ -104,7 +104,7 @@ export class Syncer {
}
public get isFirstSyncComplete(): boolean {
return this.database.state.initialized;
return this.database.state.lastSeenUpdateId !== undefined;
}
public get hasLocalChanges(): boolean {
@ -123,7 +123,7 @@ export class Syncer {
public isDocumentUpToDate(path: string): boolean {
if (
!this.database.state.initialized ||
this.database.state.lastSeenUpdateId === undefined ||
!this.hasScanned ||
this.pendingPaths.has(path) ||
this.unsyncablePaths.has(path) ||
@ -136,17 +136,16 @@ export class Syncer {
if (!record) return false;
const document = this.database.state.documents[record.documentId];
if (
!document?.materialized ||
document?.observedHash === undefined ||
!document.base ||
document.bootstrap ||
document.observedHash !== document.base.hash ||
this.database.state.fileManifest.entries[record.documentId] !== path
) {
return false;
}
const remote = this.database.state.remoteHeads[record.documentId];
if (remote && remote.vaultUpdateId > document.base.vaultUpdateId)
const remote = document.remote;
if (remote && remote.vaultUpdateId > (document.base.vaultUpdateId ?? 0))
return false;
const { pending } = this.database.state;
@ -159,7 +158,7 @@ export class Syncer {
if (
pending?.type === "fileManifest" &&
pending.request.entries[record.documentId] !==
this.database.state.fileManifest.entries[record.documentId]
this.database.state.fileManifest.entries[record.documentId]
) {
return false;
}
@ -323,15 +322,13 @@ export class Syncer {
try {
await this.reloadLocalState();
if (
this.database.state.historyRecovery ||
(this.database.state.initialized &&
this.service.vaultHistoryStatus === VaultHistoryStatus.MISSING)
this.isFirstSyncComplete &&
this.service.vaultHistoryStatus === VaultHistoryStatus.MISSING
)
await this.recoverServerHistory();
if (!this.isFirstSyncComplete) await this.service.resetHistory();
await this.serverConfig.initialize();
if (!this.database.state.initialized) {
await this.bootstrap();
}
if (!this.isFirstSyncComplete) await this.bootstrap();
if (!this.stopped) this.onReadyForEvents.trigger();
do {
this.dirty = false;
@ -341,31 +338,9 @@ export class Syncer {
await this.finishPending();
}
const batch = await this.service.getEvents(
this.database.state.lastSeenUpdateId
this.database.state.lastSeenUpdateId!
);
await this.incorporateEventBatch(batch);
// Previously excluded content becomes eligible after a setting
// change. Its saved obligation survives advancing the cursor.
if (Object.keys(this.database.state.excluded ?? {}).length)
await this.incorporateFileManifest(
this.database.state.fileManifest
);
else await this.scan();
for (const head of Object.values(
this.database.state.remoteHeads
)) {
const doc = this.database.state.documents[head.documentId];
if (
this.database.state.local[head.documentId] !==
undefined &&
(!doc?.materialized ||
!doc.base ||
doc.bootstrap ||
doc.base.vaultUpdateId < head.vaultUpdateId)
) {
await this.incorporateContent(head);
}
}
if (await this.preparePush()) {
await this.finishPending();
this.dirty = true;
@ -399,52 +374,40 @@ export class Syncer {
this.onServerHistoryChanged.trigger();
this.contentCache.reset();
this.hasScanned = false;
if (!this.database.state.historyRecovery) {
await this.scan();
const guard = this.localGuard();
const previous = this.database.state;
const next = emptyState(previous.vaultKey);
next.historyRecovery = true;
next.lastAppliedLocalChangeId = previous.lastAppliedLocalChangeId;
next.protectedPaths = previous.protectedPaths;
for (const [id, path] of Object.entries(previous.local)) {
const info = await this.files.fs.stat(path);
if (info?.kind !== "file") continue;
const doc = previous.documents[id];
if (
this.settings.isIgnored(path) ||
this.settings.isOversized(info.size)
) {
next.local[id] = path;
next.documents[id] = {
materialized: true,
bootstrap: true
};
(next.excluded ??= {})[id] =
previous.fileManifest.entries[id] ?? null;
continue;
}
const snapshot = await this.files.snapshot(path);
if (!snapshot) throw new LocalChangesDuringReconciliation();
// An unsent edit from the abandoned history is a separate local
// document. It must never be diffed against a reused version ID.
const clean = doc?.base?.hash === snapshot.hash;
const localId = doc?.base && !clean ? uuid() : id;
next.local[localId] = path;
next.documents[localId] = {
materialized: true,
observedHash: snapshot.hash,
recoveryBase: clean ? snapshot : doc?.recoveryBase
};
await this.scan();
const guard = this.localGuard();
const previous = this.database.state;
const next = emptyState(previous.vaultKey);
next.lastAppliedLocalChangeId = previous.lastAppliedLocalChangeId;
for (const [id, path] of Object.entries(previous.local)) {
const info = await this.files.fs.stat(path);
if (info?.kind !== "file") continue;
const doc = previous.documents[id];
if (
this.settings.isIgnored(path) ||
this.settings.isOversized(info.size)
) {
next.local[id] = path;
next.documents[id] = { observedHash: doc?.observedHash ?? "" };
(next.excluded ??= {})[id] =
previous.fileManifest.entries[id] ?? null;
continue;
}
guard();
// Persist the reset BEFORE releasing the old checkpoint. After a
// crash no old request can be retried against the replacement log.
await this.database.commit(next);
const snapshot = await this.files.snapshot(path);
if (!snapshot) throw new LocalChangesDuringReconciliation();
const clean = doc?.base?.hash === snapshot.hash;
const localId = doc?.base && !clean ? uuid() : id;
next.local[localId] = path;
next.documents[localId] = {
observedHash: snapshot.hash,
base: clean
? { vaultUpdateId: null, hash: snapshot.hash }
: undefined
};
}
await this.service.resetHistory();
const next = this.next();
delete next.historyRecovery;
guard();
// An absent cursor durably abandons all old requests and version IDs.
// The next bootstrap clears the transport checkpoint before any request.
await this.database.commit(next);
}
@ -511,20 +474,11 @@ export class Syncer {
}
private async bootstrap(): Promise<void> {
let vaultSnapshot = this.database.state.bootstrap;
if (!vaultSnapshot) {
vaultSnapshot = await this.service.getVaultSnapshot();
validateFileManifest(vaultSnapshot.fileManifest.entries);
await this.database.commit({
...this.next(),
bootstrap: vaultSnapshot
});
}
await this.incorporateFileManifest(
vaultSnapshot.fileManifest,
vaultSnapshot.headEventId,
vaultSnapshot
);
const vaultSnapshot = await this.service.getVaultSnapshot();
await this.reconcile(vaultSnapshot.fileManifest, {
cursor: vaultSnapshot.headEventId,
initial: vaultSnapshot
});
}
private async scan(initial?: VaultSnapshot): Promise<void> {
@ -548,13 +502,15 @@ export class Syncer {
}
private async incorporateEventBatch(batch: EventBatch): Promise<void> {
let cursor = this.database.state.lastSeenUpdateId;
let cursor = this.database.state.lastSeenUpdateId ?? 0;
let manifest = this.database.state.fileManifest;
const heads = { ...this.database.state.remoteHeads };
const heads: Record<string, DocumentVersionWithoutContent> = {};
for (const doc of Object.values(this.database.state.documents))
if (doc.remote) heads[doc.remote.documentId] = doc.remote;
const receipts: EventRecord[] = [];
// Fold every page before touching local files. If fetching is interrupted,
// discard this work and retry from the last committed cursor.
for (; ;) {
for (;;) {
const after = cursor;
for (const event of batch.events) {
if (event.eventId <= cursor) continue;
@ -579,7 +535,7 @@ export class Syncer {
} else if (
!heads[event.document.documentId] ||
heads[event.document.documentId].vaultUpdateId <
event.document.vaultUpdateId
event.document.vaultUpdateId
) {
heads[event.document.documentId] = this.documentHead(
event.document
@ -596,7 +552,6 @@ export class Syncer {
if (end === batch.headEventId) break;
batch = await this.service.getEvents(cursor);
}
if (cursor === this.database.state.lastSeenUpdateId) return;
// Recover our submitted bases before folding later remote edits. A
// permanent retry failure cannot cancel an earlier in-flight attempt.
const next = this.next();
@ -608,13 +563,15 @@ export class Syncer {
if (!pending) continue;
if (event.type === "content" && pending.type === "content") {
const doc = next.documents[pending.documentId];
if (!doc.base || doc.base.vaultUpdateId < event.eventId) {
if (
!doc.base ||
(doc.base.vaultUpdateId ?? 0) < event.eventId
) {
doc.base = {
...event.document,
vaultUpdateId: event.eventId,
hash: pending.snapshot.hash
};
doc.bootstrap = false;
delete doc.rejected;
this.clearRejections(next, pending.documentId);
await this.cacheContent(
event.eventId,
next.local[pending.documentId],
@ -627,7 +584,7 @@ export class Syncer {
) {
if (next.fileManifest.fileManifestId < event.eventId) {
next.fileManifest = event.fileManifest;
delete next.rejectedManifest;
this.clearRejections(next, "manifest");
}
} else
throw new PermanentSyncError(
@ -639,13 +596,7 @@ export class Syncer {
if (acknowledged) await this.database.commit(next);
// Three-way merging compares snapshots. Replaying intermediate remote
// edits can discard a local edit even when the remote ends at its base.
await this.incorporateFileManifest(
manifest,
cursor,
undefined,
false,
heads
);
await this.reconcile(manifest, { cursor, heads });
}
private documentHead(
@ -679,34 +630,30 @@ export class Syncer {
details:
from && to && from !== to
? {
type: SyncType.MOVE,
relativePath: to,
movedFrom: from
}
type: SyncType.MOVE,
relativePath: to,
movedFrom: from
}
: !from && to
? { type: SyncType.CREATE, relativePath: to }
: from && !to
? { type: SyncType.DELETE, relativePath: from }
: {
type: SyncType.UPDATE,
relativePath: to ?? from
}
? { type: SyncType.CREATE, relativePath: to }
: from && !to
? { type: SyncType.DELETE, relativePath: from }
: {
type: SyncType.UPDATE,
relativePath: to ?? from
}
});
}
}
private async remoteSnapshot(
head: DocumentVersionWithoutContent,
path: string,
supplied?: DocumentVersion
head: Pick<
DocumentVersionWithoutContent,
"documentId" | "vaultUpdateId"
>,
path: string
): Promise<StoredSnapshot> {
const content =
supplied !== undefined
? base64ToBytes(supplied.contentBase64)
: await this.service.getDocumentVersionContent({
documentId: head.documentId,
vaultUpdateId: head.vaultUpdateId
});
const content = await this.service.getDocumentVersionContent(head);
await this.cacheContent(head.vaultUpdateId, path, content);
return toStoredSnapshot({ content });
}
@ -714,12 +661,9 @@ export class Syncer {
private async contentWrite(
next: EngineState,
head: DocumentVersionWithoutContent,
path: string,
supplied?: DocumentVersion
path: string
): Promise<FileWrite | undefined> {
const document = (next.documents[head.documentId] ??= {
materialized: false
});
const document = (next.documents[head.documentId] ??= {});
if (
next.excluded?.[head.documentId] !== undefined ||
this.settings.isIgnored(path) ||
@ -730,7 +674,7 @@ export class Syncer {
const currentPath = this.database.state.local[head.documentId];
const local =
currentPath &&
(await this.files.fs.stat(currentPath))?.kind === "file"
(await this.files.fs.stat(currentPath))?.kind === "file"
? await this.files.snapshot(currentPath)
: undefined;
if (
@ -739,16 +683,27 @@ export class Syncer {
) {
return undefined;
}
if (
document.base?.vaultUpdateId === null &&
local &&
local.hash !== document.base.hash
)
throw new LocalChangesDuringReconciliation();
const base =
document.recoveryBase ??
(local !== undefined &&
document.base !== undefined &&
document.bootstrap !== true
? await this.remoteSnapshot(document.base, path)
: undefined);
local && document.base
? document.base.vaultUpdateId === null
? local
: await this.remoteSnapshot(
{
documentId: head.documentId,
vaultUpdateId: document.base.vaultUpdateId
},
path
)
: undefined;
// Cache the new head after the older merge base so it remains the most
// recently used version and can back the follow-up local diff.
const remote = await this.remoteSnapshot(head, path, supplied);
const remote = await this.remoteSnapshot(head, path);
let result = remote;
if (local !== undefined) {
result = await mergeContent(
@ -759,74 +714,30 @@ export class Syncer {
(await this.serverConfig.getConfig()).mergeableFileExtensions
);
}
document.base = { ...this.documentHead(head), hash: remote.hash };
delete document.recoveryBase;
document.bootstrap = false;
document.materialized = true;
document.base = {
vaultUpdateId: head.vaultUpdateId,
hash: remote.hash
};
delete document.remote;
document.observedHash = result.hash;
return local?.contentBase64 === result.contentBase64
? undefined
: { expected: local, replacement: result };
}
private async incorporateContent(
head: DocumentVersionWithoutContent,
cursor?: number,
supplied?: DocumentVersion,
clearPending = false
): Promise<void> {
await this.scan();
const guard = this.localGuard();
const next = this.next();
if (
!next.remoteHeads[head.documentId] ||
next.remoteHeads[head.documentId].vaultUpdateId < head.vaultUpdateId
) {
next.remoteHeads[head.documentId] = this.documentHead(head);
}
if (cursor !== undefined) {
next.lastSeenUpdateId = cursor;
}
if (clearPending) {
delete next.pending;
}
const path = next.local[head.documentId];
const document = next.documents[head.documentId];
const wasMaterialized = document?.materialized;
const writes: Record<string, FileWrite> = {};
if (
path !== undefined &&
(!document?.base ||
!document.materialized ||
document.bootstrap ||
document.base.vaultUpdateId < head.vaultUpdateId)
) {
const write = await this.contentWrite(next, head, path, supplied);
if (write) {
writes[head.documentId] = write;
}
}
await this.files.apply(next, writes, guard);
if (path !== undefined && writes[head.documentId]) {
this.history.addHistoryEntry({
status: SyncStatus.SUCCESS,
message: "Remote content applied",
details: {
type: wasMaterialized ? SyncType.UPDATE : SyncType.CREATE,
relativePath: path
},
author: head.userId,
timestamp: new Date(head.updatedDate)
});
}
}
private async incorporateFileManifest(
private async reconcile(
remote: FileManifest,
cursor?: number,
initial?: VaultSnapshot,
clearPending = false,
heads?: EngineState["remoteHeads"]
{
cursor,
initial,
clearPending = false,
heads
}: {
cursor?: number;
initial?: VaultSnapshot;
clearPending?: boolean;
heads?: Record<string, DocumentVersionWithoutContent>;
} = {}
): Promise<void> {
validateFileManifest(remote.entries);
await this.scan(initial);
@ -852,7 +763,8 @@ export class Syncer {
const previousManifest = next.fileManifest.entries;
next.local = mergeFileManifests(base, next.local, remote.entries);
next.fileManifest = remote;
if (heads) next.remoteHeads = heads;
for (const head of Object.values(heads ?? {}))
(next.documents[head.documentId] ??= {}).remote = head;
for (const [id, oldPath] of Object.entries(before)) {
const path = next.local[id];
if (
@ -877,41 +789,43 @@ export class Syncer {
next.local,
remote.entries,
Object.keys(next.excluded ?? {}),
next.protectedPaths
[...this.unsyncablePaths]
);
if (initial) {
next.initialized = true;
delete next.bootstrap;
for (const head of initial.documents) {
next.remoteHeads[head.documentId] = this.documentHead(head);
}
for (const head of initial.documents)
(next.documents[head.documentId] ??= {}).remote =
this.documentHead(head);
}
const writes: Record<string, FileWrite> = {};
for (const [id, path] of Object.entries(next.local)) {
next.documents[id] ??= { materialized: false };
if (before[id] === undefined)
next.documents[id].materialized = false;
const doc = (next.documents[id] ??= {});
if (before[id] === undefined) delete doc.observedHash;
if (
doc.observedHash !== undefined &&
doc.remote &&
doc.remote.vaultUpdateId <= (doc.base?.vaultUpdateId ?? 0)
)
delete doc.remote;
if (
next.excluded?.[id] === undefined &&
!this.settings.isIgnored(path) &&
remote.entries[id] !== undefined &&
(remote.entries[id] !== undefined ||
doc.remote !== undefined) &&
(!this.database.state.local[id] ||
!next.documents[id].base ||
next.documents[id].bootstrap ||
!next.documents[id].materialized ||
(next.remoteHeads[id]?.vaultUpdateId ?? 0) >
(next.documents[id].base?.vaultUpdateId ?? 0))
!doc.base ||
doc.observedHash === undefined ||
(doc.remote?.vaultUpdateId ?? 0) >
(doc.base.vaultUpdateId ?? 0))
) {
const knownHead =
next.remoteHeads[id] ??
initial?.documents.find((doc) => doc.documentId === id);
const head = knownHead ?? (await this.service.getDocumentMetadata(id));
const knownHead = doc.remote;
const head =
knownHead ?? (await this.service.getDocumentMetadata(id));
if (!head) {
throw new PermanentSyncError(
`File manifest references missing content: ${id}`
);
}
next.remoteHeads[id] = this.documentHead(head);
doc.remote = this.documentHead(head);
const write = await this.contentWrite(next, head, path);
if (write) {
writes[id] = write;
@ -932,7 +846,7 @@ export class Syncer {
this.database.state.local,
this.database.state.fileManifest.entries,
Object.keys(this.database.state.excluded ?? {}),
this.database.state.protectedPaths
[...this.unsyncablePaths]
);
if (!sameFileManifest(canonical, this.database.state.local)) {
await this.files.apply(
@ -960,11 +874,11 @@ export class Syncer {
if (
this.settings.isIgnored(path) ||
this.database.state.excluded?.[id] !== undefined ||
!this.database.state.documents[id]?.materialized ||
this.database.state.documents[id]?.observedHash === undefined ||
this.unsyncablePaths.has(path) ||
this.settings.isOversized(
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime.
this.database.state.remoteHeads[id]?.contentSize ?? 0
this.database.state.documents[id]?.remote?.contentSize ?? 0
)
) {
continue;
@ -978,10 +892,15 @@ export class Syncer {
) {
continue;
}
const { base, rejected } = this.database.state.documents[id];
const { base } = this.database.state.documents[id];
if (
base?.hash === snapshot.hash ||
rejected?.hash === snapshot.hash
this.rejections.some(
(request) =>
request.type === "content" &&
request.documentId === id &&
request.snapshot.hash === snapshot.hash
)
) {
continue;
}
@ -997,7 +916,7 @@ export class Syncer {
parentVersionId: base?.vaultUpdateId ?? null,
content: await this.pushContent(
path,
base?.vaultUpdateId,
base?.vaultUpdateId ?? undefined,
snapshot
)
}
@ -1013,11 +932,11 @@ export class Syncer {
entries,
this.database.state.fileManifest.entries
) &&
(!this.database.state.rejectedManifest ||
!sameFileManifest(
entries,
this.database.state.rejectedManifest.entries
))
!this.rejections.some(
(request) =>
request.type === "fileManifest" &&
sameFileManifest(entries, request.request.entries)
)
) {
guard();
await this.database.commit({
@ -1058,7 +977,7 @@ export class Syncer {
this.database.state.fileManifest.entries,
Object.keys(this.database.state.excluded ?? {}),
[
...(this.database.state.protectedPaths ?? []),
...this.unsyncablePaths,
...Object.keys(this.database.state.excluded ?? {}).flatMap(
(id) =>
this.database.state.local[id]
@ -1069,50 +988,59 @@ export class Syncer {
);
}
private get rejections(): PendingRequest[] {
return (this.database.state.unconfirmed ?? []).filter(
(request) => request.rejection !== undefined
);
}
private get rejectionMessage(): string | undefined {
const { state } = this.database;
for (const [id, path] of Object.entries(state.local ?? {})) {
const doc = state.documents[id];
if (
doc?.rejected &&
doc.observedHash === doc.rejected.hash &&
doc.base?.hash !== doc.rejected.hash &&
!this.settings.isIgnored(path) &&
state.excluded?.[id] === undefined
for (const request of this.rejections) {
if (request.type === "content") {
const id = request.documentId;
const path = state.local[id];
const doc = state.documents[id];
if (
path !== undefined &&
doc?.observedHash === request.snapshot.hash &&
doc.base?.hash !== request.snapshot.hash &&
!this.settings.isIgnored(path) &&
state.excluded?.[id] === undefined
)
return request.rejection;
} else if (
sameFileManifest(
this.manifestToSubmit(),
request.request.entries
) &&
!sameFileManifest(
state.fileManifest.entries,
request.request.entries
)
)
return doc.rejected.message;
return request.rejection;
}
if (
state.rejectedManifest &&
sameFileManifest(
this.manifestToSubmit(),
state.rejectedManifest.entries
) &&
!sameFileManifest(
state.fileManifest.entries,
state.rejectedManifest.entries
}
private clearRejections(state: EngineState, id?: string): void {
for (const request of state.unconfirmed ?? [])
if (
id === undefined ||
(request.type === "content"
? request.documentId
: "manifest") === id
)
)
return state.rejectedManifest.message;
return undefined;
delete request.rejection;
}
/** An explicit reset or settings change permits fresh attempts after rejection. */
public async retryRejectedRequests(): Promise<void> {
await this.database.recoverPersistence();
const next = structuredClone(this.database.state);
let changed = false;
if (next.rejectedManifest) {
delete next.rejectedManifest;
changed = true;
}
for (const doc of Object.values(next.documents)) {
if (doc.rejected) {
delete doc.rejected;
changed = true;
}
}
if (changed) await this.database.commit(next);
if (!this.rejections.length) return;
const next = this.next();
this.clearRejections(next);
await this.database.commit(next);
}
private async finishPending(): Promise<void> {
@ -1144,16 +1072,6 @@ export class Syncer {
if (pending.rejection && !pending.response) {
const next = this.next();
if (pending.type === "content") {
next.documents[pending.documentId].rejected = {
hash: pending.snapshot.hash,
message: pending.rejection
};
} else
next.rejectedManifest = {
entries: pending.request.entries,
message: pending.rejection
};
(next.unconfirmed ??= []).push(pending);
delete next.pending;
await this.database.commit(next);
@ -1165,7 +1083,7 @@ export class Syncer {
relativePath:
pending.type === "content"
? (next.local[pending.documentId] ??
pending.documentId)
pending.documentId)
: "File manifest"
}
});
@ -1175,15 +1093,10 @@ export class Syncer {
if (pending.type === "fileManifest") {
const response = pending.response!;
if (response.type === "StaleBase") {
await this.incorporateFileManifest(
response,
undefined,
undefined,
true
);
await this.reconcile(response, { clearPending: true });
} else {
const next = this.next();
delete next.rejectedManifest;
this.clearRejections(next, "manifest");
next.fileManifest = {
fileManifestId: response.fileManifestId,
entries: pending.request.entries
@ -1206,15 +1119,15 @@ export class Syncer {
details:
after === undefined
? {
type: SyncType.DELETE,
relativePath: before
}
type: SyncType.DELETE,
relativePath: before
}
: before === undefined
? {
? {
type: SyncType.CREATE,
relativePath: after
}
: {
: {
type: SyncType.MOVE,
relativePath: after,
movedFrom: before
@ -1225,25 +1138,19 @@ export class Syncer {
} else {
const response = pending.response!;
if (response.type === "StaleBase") {
await this.incorporateContent(
response,
undefined,
undefined,
true
);
await this.reconcile(this.database.state.fileManifest, {
clearPending: true,
heads: { [pending.documentId]: this.documentHead(response) }
});
} else {
const next = this.next();
const doc = (next.documents[pending.documentId] ??= {
materialized: false
});
const doc = (next.documents[pending.documentId] ??= {});
doc.base = {
...this.documentHead(response),
vaultUpdateId: response.vaultUpdateId,
hash: pending.snapshot.hash
};
doc.bootstrap = false;
delete doc.rejected;
next.remoteHeads[pending.documentId] =
this.documentHead(response);
this.clearRejections(next, pending.documentId);
delete doc.remote;
delete next.pending;
await this.database.commit(next);
const path = next.local[pending.documentId];