.
This commit is contained in:
parent
25f64ec690
commit
ff1f474324
93 changed files with 3271 additions and 2003 deletions
|
|
@ -5,7 +5,8 @@ import unusedImports from "eslint-plugin-unused-imports";
|
||||||
export default [
|
export default [
|
||||||
{
|
{
|
||||||
ignores: [
|
ignores: [
|
||||||
"sync-client/src/services/types.ts",
|
// Generated by ts-rs in sync-server; lint the Rust definitions instead.
|
||||||
|
"sync-client/src/services/types/**",
|
||||||
"**/dist/",
|
"**/dist/",
|
||||||
"**/*.mjs",
|
"**/*.mjs",
|
||||||
"**/*.js"
|
"**/*.js"
|
||||||
|
|
@ -90,5 +91,15 @@ export default [
|
||||||
tsconfigRootDir: import.meta.dirname
|
tsconfigRootDir: import.meta.dirname
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}),
|
||||||
|
{
|
||||||
|
files: ["sync-client/**/*.ts"],
|
||||||
|
rules: {
|
||||||
|
curly: ["error", "all"],
|
||||||
|
"padding-line-between-statements": [
|
||||||
|
"error",
|
||||||
|
{ blankLine: "always", prev: "block-like", next: "*" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
# Sync client (API v4)
|
# Sync client (API v4)
|
||||||
|
|
||||||
The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket
|
The sync coordinator uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket
|
||||||
`vaultChanged` hints wake it; complete disk scans and the durable server event log are the
|
`vaultChanged` hints wake it; complete disk scans and the durable server event log are the
|
||||||
sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a
|
sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a
|
||||||
positive interval to repair missed notifications. Unset or `0` disables polling.
|
positive interval to repair missed notifications. Unset or `0` disables polling.
|
||||||
|
|
@ -20,6 +20,46 @@ resuming starts a fresh session. Connection checks can run while sync is disable
|
||||||
WebSocket shutdown retires callbacks immediately and drains message handlers
|
WebSocket shutdown retires callbacks immediately and drains message handlers
|
||||||
already running; it does not wait for the transport's close handshake.
|
already running; it does not wait for the transport's close handshake.
|
||||||
|
|
||||||
|
## Code layout
|
||||||
|
|
||||||
|
Start with `src/sync-operations/syncer.ts` for the sync workflow: resolve the saved
|
||||||
|
upload, fetch the complete remote history, apply it, and prepare the next upload.
|
||||||
|
It coordinates the following components:
|
||||||
|
|
||||||
|
| Component | Responsibility |
|
||||||
|
| -------------------------------------- | --------------------------------------------------------------------------------------------- |
|
||||||
|
| `sync-scheduler.ts` | Serialize attempts, coalesce notifications, stop, and schedule polling or retries. |
|
||||||
|
| `local-change-queue.ts` | Persist create/delete/move notifications and invalidate plans when identities change. |
|
||||||
|
| `scan.ts` | Inventory disk files, infer identities, and commit the scan before consuming notifications. |
|
||||||
|
| `remote-changes.ts` | Fetch and validate every event page before returning the final remote state. |
|
||||||
|
| `content-sync.ts` | Fetch merge bases, plan content writes, encode uploads, and maintain the optional diff cache. |
|
||||||
|
| `upload-queue.ts` | Persist exact requests, handle acknowledgements, and retain rejected requests for retry. |
|
||||||
|
| `file-operations/` | Apply paths and content to disk, incorporating notifications between files. |
|
||||||
|
| `persistence/database.ts` | Define `StoredDatabase` and atomically commit or reload it. |
|
||||||
|
| `persistence/client-metadata-store.ts` | Serialize database, settings, checkpoint, and notification saves into the shared store. |
|
||||||
|
|
||||||
|
`requestSync()` schedules work; `synchronize()` runs an attempt; `refreshLocalFiles()`
|
||||||
|
flushes notifications and calls `scanLocalFiles()`. `initializeFromServerSnapshot()`
|
||||||
|
applies the initial server snapshot, while `applyRemoteChanges()` merges and applies
|
||||||
|
an observed remote state. These operations keep the same persisted database format.
|
||||||
|
|
||||||
|
The scheduler's `SyncPhase` enum makes its transitions explicit:
|
||||||
|
|
||||||
|
| Event | Transition |
|
||||||
|
| ----------------------------------- | ------------------------------------------------------------------- |
|
||||||
|
| Start or explicit request | Idle, waiting, or blocked → run requested |
|
||||||
|
| Begin a pass | Run requested → running |
|
||||||
|
| Notification during a pass | Running → run requested; repeated notifications coalesce |
|
||||||
|
| Successful completion | Running → idle; optionally schedule the polling timer |
|
||||||
|
| Transient failure | Running → waiting to retry |
|
||||||
|
| Authentication or permanent failure | Running → blocked until an explicit request |
|
||||||
|
| Stop | Running → stopping → stopped; idle timers are cancelled immediately |
|
||||||
|
|
||||||
|
From `frontend/`, run `npm run lint -w sync-client`, `npm run check -w sync-client`,
|
||||||
|
and `npm run test -w sync-client`. `npm run test:harness` covers interrupted saves,
|
||||||
|
file application, and recovery. Protocol types under `src/services/types/` are
|
||||||
|
generated from Rust and excluded from ESLint; handwritten TypeScript requires braces.
|
||||||
|
|
||||||
## State and policy
|
## State and policy
|
||||||
|
|
||||||
The persisted state contains the incorporated server file manifest, the current
|
The persisted state contains the incorporated server file manifest, the current
|
||||||
|
|
@ -127,7 +167,7 @@ Power loss may lose bytes or rename identity and may cause a merge to run again.
|
||||||
Metadata is the one atomic boundary: persistence `save` replaces a complete value
|
Metadata is the one atomic boundary: persistence `save` replaces a complete value
|
||||||
and an interrupted/failed save leaves an old or new complete value, never a torn
|
and an interrupted/failed save leaves an old or new complete value, never a torn
|
||||||
object. A failed save is reloaded before retrying. No ordering between metadata and
|
object. A failed save is reloaded before retrying. No ordering between metadata and
|
||||||
user-file durability is required. Settings and engine saves are serialized.
|
user-file durability is required. Settings and database saves are serialized.
|
||||||
Keep metadata outside the user-file namespace or in `.vault-link-sync`, which is
|
Keep metadata outside the user-file namespace or in `.vault-link-sync`, which is
|
||||||
excluded from scans. Changing a client's vault once it holds identities, offline
|
excluded from scans. Changing a client's vault once it holds identities, offline
|
||||||
notifications, pending requests or an incorporated event cursor requires a separate state store.
|
notifications, pending requests or an incorporated event cursor requires a separate state store.
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,9 @@
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "webpack watch --mode development",
|
"dev": "webpack watch --mode development",
|
||||||
"build": "webpack --mode production",
|
"build": "webpack --mode production",
|
||||||
"test": "tsx --test 'src/**/*.test.ts'"
|
"test": "tsx --test 'src/**/*.test.ts'",
|
||||||
|
"lint": "eslint . --max-warnings 0",
|
||||||
|
"check": "tsc --noEmit"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"byte-base64": "^1.1.0",
|
"byte-base64": "^1.1.0",
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import type { FileSystemOperations } from "./filesystem-operations";
|
||||||
import type {
|
import type {
|
||||||
Database,
|
Database,
|
||||||
DocumentId,
|
DocumentId,
|
||||||
EngineState,
|
StoredDatabase,
|
||||||
RelativePath
|
RelativePath
|
||||||
} from "../persistence/database";
|
} from "../persistence/database";
|
||||||
import {
|
import {
|
||||||
|
|
@ -21,7 +21,7 @@ import { decodeText } from "../utils/decode-text";
|
||||||
import type { ServerConfig } from "../services/server-config";
|
import type { ServerConfig } from "../services/server-config";
|
||||||
import {
|
import {
|
||||||
applyLocalChange,
|
applyLocalChange,
|
||||||
unappliedChanges,
|
getUnappliedChanges,
|
||||||
type LocalChange
|
type LocalChange
|
||||||
} from "../sync-operations/local-changes";
|
} from "../sync-operations/local-changes";
|
||||||
import { LocalChangesDuringReconciliation } from "../errors/errors";
|
import { LocalChangesDuringReconciliation } from "../errors/errors";
|
||||||
|
|
@ -33,9 +33,18 @@ export interface FileWrite {
|
||||||
|
|
||||||
type Mutation = (action: () => Promise<void>) => Promise<void>;
|
type Mutation = (action: () => Promise<void>) => Promise<void>;
|
||||||
|
|
||||||
|
interface FileApplicationContext {
|
||||||
|
current: StoredDatabase;
|
||||||
|
next: StoredDatabase;
|
||||||
|
writes: Partial<Record<DocumentId, FileWrite>>;
|
||||||
|
paths: Set<RelativePath>;
|
||||||
|
guard: () => void;
|
||||||
|
mutate: Mutation;
|
||||||
|
}
|
||||||
|
|
||||||
/** Apply files first, then save metadata. Repeating a content merge is idempotent: reconcile(p, a, a) == a */
|
/** Apply files first, then save metadata. Repeating a content merge is idempotent: reconcile(p, a, a) == a */
|
||||||
export class FileOperations {
|
export class FileOperations {
|
||||||
private applying?: EngineState;
|
private applying?: StoredDatabase;
|
||||||
private readonly paths: FilePaths;
|
private readonly paths: FilePaths;
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
|
|
@ -60,7 +69,7 @@ export class FileOperations {
|
||||||
this.paths = new FilePaths(fs, isProtectedFile);
|
this.paths = new FilePaths(fs, isProtectedFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async snapshot(
|
public async readSnapshot(
|
||||||
path: RelativePath
|
path: RelativePath
|
||||||
): Promise<HashedSnapshot | undefined> {
|
): Promise<HashedSnapshot | undefined> {
|
||||||
const snapshot = await this.fs.read(path);
|
const snapshot = await this.fs.read(path);
|
||||||
|
|
@ -79,16 +88,19 @@ export class FileOperations {
|
||||||
): void {
|
): void {
|
||||||
const state = structuredClone(this.applying ?? this.database.state);
|
const state = structuredClone(this.applying ?? this.database.state);
|
||||||
|
|
||||||
if (state.lastSeenUpdateId === undefined) return;
|
if (state.lastSeenUpdateId === undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
for (const pending of unappliedChanges(state, previous))
|
for (const pending of getUnappliedChanges(state, previous)) {
|
||||||
applyLocalChange(state, pending, () => false);
|
applyLocalChange(state, pending, () => false);
|
||||||
|
}
|
||||||
|
|
||||||
applyLocalChange(state, change, () => false);
|
applyLocalChange(state, change, () => false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async apply(
|
public async applyChanges(
|
||||||
next: EngineState,
|
next: StoredDatabase,
|
||||||
writes: Partial<Record<DocumentId, FileWrite>> = {},
|
writes: Partial<Record<DocumentId, FileWrite>> = {},
|
||||||
validatePlan: () => void = () => undefined
|
validatePlan: () => void = () => undefined
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
|
@ -96,16 +108,22 @@ export class FileOperations {
|
||||||
const changes = this.localChanges.entries().length;
|
const changes = this.localChanges.entries().length;
|
||||||
let started = false;
|
let started = false;
|
||||||
const guard = (): void => {
|
const guard = (): void => {
|
||||||
if (started) return;
|
if (started) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
validatePlan();
|
validatePlan();
|
||||||
if (this.localChanges.entries().length > changes)
|
if (this.localChanges.entries().length > changes) {
|
||||||
throw new LocalChangesDuringReconciliation();
|
throw new LocalChangesDuringReconciliation();
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const mutate: Mutation = async (action) => {
|
const mutate: Mutation = async (action) => {
|
||||||
guard();
|
guard();
|
||||||
started = true;
|
started = true;
|
||||||
await action();
|
await action();
|
||||||
};
|
};
|
||||||
|
|
||||||
guard();
|
guard();
|
||||||
const current = structuredClone(this.database.state);
|
const current = structuredClone(this.database.state);
|
||||||
current.lastSeenUpdateId = next.lastSeenUpdateId;
|
current.lastSeenUpdateId = next.lastSeenUpdateId;
|
||||||
|
|
@ -114,7 +132,7 @@ export class FileOperations {
|
||||||
const observeChanges = async (): Promise<void> => {
|
const observeChanges = async (): Promise<void> => {
|
||||||
await this.localChanges.flush();
|
await this.localChanges.flush();
|
||||||
guard();
|
guard();
|
||||||
for (const change of unappliedChanges(
|
for (const change of getUnappliedChanges(
|
||||||
next,
|
next,
|
||||||
this.localChanges.entries()
|
this.localChanges.entries()
|
||||||
)) {
|
)) {
|
||||||
|
|
@ -130,45 +148,77 @@ export class FileOperations {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (const id of new Set([
|
for (const id of new Set([
|
||||||
...Object.keys(current.local),
|
...Object.keys(current.local),
|
||||||
...Object.keys(next.local)
|
...Object.keys(next.local)
|
||||||
])) {
|
])) {
|
||||||
await observeChanges();
|
await observeChanges();
|
||||||
|
await this.applyDocument(id, {
|
||||||
|
current,
|
||||||
|
next,
|
||||||
|
writes,
|
||||||
|
paths,
|
||||||
|
guard,
|
||||||
|
mutate
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await observeChanges();
|
||||||
|
await this.database.commit(next);
|
||||||
|
} finally {
|
||||||
|
this.applying = undefined;
|
||||||
|
this.filesApplied([...paths]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async applyDocument(
|
||||||
|
id: DocumentId,
|
||||||
|
context: FileApplicationContext
|
||||||
|
): Promise<void> {
|
||||||
|
const { current, next, writes, paths, guard, mutate } = context;
|
||||||
let from =
|
let from =
|
||||||
current.documents[id]?.observedHash !== undefined
|
current.documents[id]?.observedHash !== undefined
|
||||||
? current.local[id]
|
? current.local[id]
|
||||||
: undefined;
|
: undefined;
|
||||||
let to = next.local[id];
|
let to = next.local[id];
|
||||||
const write = writes[id];
|
const write = writes[id];
|
||||||
if (!write && (from === undefined || from === to)) continue;
|
if (!write && (from === undefined || from === to)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (from !== undefined) {
|
if (from !== undefined) {
|
||||||
paths.add(from);
|
paths.add(from);
|
||||||
const info = await this.fs.stat(from);
|
const info = await this.fs.stat(from);
|
||||||
guard();
|
guard();
|
||||||
if (info?.kind !== "file") {
|
if (info?.kind !== "file") {
|
||||||
delete next.local[id];
|
Reflect.deleteProperty(next.local, id);
|
||||||
continue;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.isProtectedFile(from, info.size)) {
|
if (this.isProtectedFile(from, info.size)) {
|
||||||
next.local[id] = from;
|
next.local[id] = from;
|
||||||
next.documents[id] = structuredClone(
|
next.documents[id] = structuredClone(
|
||||||
current.documents[id]
|
current.documents[id] ?? {}
|
||||||
);
|
);
|
||||||
(next.excluded ??= {})[id] =
|
(next.excluded ??= {})[id] =
|
||||||
current.fileManifest.entries[id] ?? null;
|
current.fileManifest.entries[id] ?? null;
|
||||||
continue;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (to === undefined) {
|
if (to === undefined) {
|
||||||
if (from !== undefined) {
|
if (from !== undefined) {
|
||||||
await mutate(() => this.fs.deleteFile(from!));
|
const source = from;
|
||||||
delete current.local[id];
|
await mutate(async () => this.fs.deleteFile(source));
|
||||||
|
Reflect.deleteProperty(current.local, id);
|
||||||
await this.paths.removeEmptyParents(from);
|
await this.paths.removeEmptyParents(from);
|
||||||
}
|
}
|
||||||
continue;
|
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (from !== to) {
|
if (from !== to) {
|
||||||
to = await this.paths.prepareDestination(
|
to = await this.paths.prepareDestination(
|
||||||
next,
|
next,
|
||||||
|
|
@ -183,44 +233,43 @@ export class FileOperations {
|
||||||
? current.local[id]
|
? current.local[id]
|
||||||
: undefined;
|
: undefined;
|
||||||
if (from !== undefined) {
|
if (from !== undefined) {
|
||||||
|
const source = from;
|
||||||
|
const destination = to;
|
||||||
await mutate(async () => {
|
await mutate(async () => {
|
||||||
current.local[id] = to;
|
current.local[id] = destination;
|
||||||
await this.fs.rename(from!, to);
|
await this.fs.rename(source, destination);
|
||||||
});
|
});
|
||||||
await this.paths.removeEmptyParents(from);
|
await this.paths.removeEmptyParents(from);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (write) {
|
if (write) {
|
||||||
const snapshot =
|
const snapshot =
|
||||||
from === undefined ? undefined : await this.fs.read(to);
|
from === undefined ? undefined : await this.fs.read(to);
|
||||||
if (from !== undefined && !snapshot) {
|
if (from !== undefined && !snapshot) {
|
||||||
delete next.local[id];
|
Reflect.deleteProperty(next.local, id);
|
||||||
continue;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const extensions = (await this.serverConfig.getConfig())
|
const extensions = (await this.serverConfig.getConfig())
|
||||||
.mergeableFileExtensions;
|
.mergeableFileExtensions;
|
||||||
|
const destination = to;
|
||||||
await mutate(async () => {
|
await mutate(async () => {
|
||||||
current.local[id] = to;
|
current.local[id] = destination;
|
||||||
current.documents[id] = { observedHash: "" };
|
current.documents[id] = { observedHash: "" };
|
||||||
next.documents[id].observedHash =
|
(next.documents[id] ??= {}).observedHash =
|
||||||
await this.updateContent(
|
await this.updateContent(
|
||||||
to,
|
destination,
|
||||||
snapshot,
|
snapshot,
|
||||||
write,
|
write,
|
||||||
extensions
|
extensions
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
paths.add(to);
|
paths.add(to);
|
||||||
current.local[id] = to;
|
current.local[id] = to;
|
||||||
current.documents[id] = structuredClone(next.documents[id]);
|
current.documents[id] = structuredClone(next.documents[id] ?? {});
|
||||||
}
|
|
||||||
await observeChanges();
|
|
||||||
await this.database.commit(next);
|
|
||||||
} finally {
|
|
||||||
this.applying = undefined;
|
|
||||||
this.filesApplied([...paths]);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async updateContent(
|
private async updateContent(
|
||||||
|
|
@ -245,6 +294,7 @@ export class FileOperations {
|
||||||
);
|
);
|
||||||
return hash(new TextEncoder().encode(text));
|
return hash(new TextEncoder().encode(text));
|
||||||
}
|
}
|
||||||
|
|
||||||
const result =
|
const result =
|
||||||
current && base
|
current && base
|
||||||
? await mergeContent(
|
? await mergeContent(
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,22 @@ import { v4 as uuid } from "uuid";
|
||||||
import type { FileSystemOperations } from "./filesystem-operations";
|
import type { FileSystemOperations } from "./filesystem-operations";
|
||||||
import type {
|
import type {
|
||||||
DocumentId,
|
DocumentId,
|
||||||
EngineState,
|
StoredDatabase,
|
||||||
RelativePath
|
RelativePath
|
||||||
} from "../persistence/database";
|
} from "../persistence/database";
|
||||||
import { allocatePortablePath, isInternalPath } from "../utils/portable-path";
|
import { allocatePortablePath, isInternalPath } from "../utils/portable-path";
|
||||||
|
|
||||||
|
interface PathObstruction {
|
||||||
|
path: RelativePath;
|
||||||
|
kind: "file" | "directory";
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Occupant {
|
||||||
|
id: DocumentId;
|
||||||
|
path: RelativePath;
|
||||||
|
}
|
||||||
|
type Mutation = (action: () => Promise<void>) => Promise<void>;
|
||||||
|
|
||||||
/** Resolve occupied rename destinations without discarding another file. */
|
/** Resolve occupied rename destinations without discarding another file. */
|
||||||
export class FilePaths {
|
export class FilePaths {
|
||||||
public constructor(
|
public constructor(
|
||||||
|
|
@ -18,16 +29,67 @@ export class FilePaths {
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public async prepareDestination(
|
public async prepareDestination(
|
||||||
next: EngineState,
|
next: StoredDatabase,
|
||||||
current: EngineState,
|
current: StoredDatabase,
|
||||||
id: DocumentId,
|
id: DocumentId,
|
||||||
path: RelativePath,
|
path: RelativePath,
|
||||||
mutate: (action: () => Promise<void>) => Promise<void>
|
mutate: (action: () => Promise<void>) => Promise<void>
|
||||||
): Promise<RelativePath> {
|
): Promise<RelativePath> {
|
||||||
try {
|
try {
|
||||||
for (;;) {
|
for (;;) {
|
||||||
const obstruction = await this.obstruction(path);
|
const obstruction = await this.findObstruction(path);
|
||||||
if (!obstruction) break;
|
if (!obstruction) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const occupants = await this.findMovableOccupants(
|
||||||
|
obstruction,
|
||||||
|
current,
|
||||||
|
next
|
||||||
|
);
|
||||||
|
if (occupants === undefined) {
|
||||||
|
path = await this.allocateConflictPath(next, id, path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.relocateOccupants(occupants, current, next, mutate);
|
||||||
|
if (obstruction.kind === "directory") {
|
||||||
|
await mutate(async () =>
|
||||||
|
this.fs.deleteDirectory(obstruction.path)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.createParentDirectories(path);
|
||||||
|
} catch (error) {
|
||||||
|
return this.recoverDestination(next, id, path, error);
|
||||||
|
}
|
||||||
|
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async removeEmptyParents(path: RelativePath): Promise<void> {
|
||||||
|
let parent = path.split("/").slice(0, -1).join("/");
|
||||||
|
while (parent && !isInternalPath(parent)) {
|
||||||
|
if ((await this.fs.listFilesRecursively(parent)).length) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.fs.deleteDirectory(parent);
|
||||||
|
} catch {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
parent = parent.split("/").slice(0, -1).join("/");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async findMovableOccupants(
|
||||||
|
obstruction: PathObstruction,
|
||||||
|
current: StoredDatabase,
|
||||||
|
next: StoredDatabase
|
||||||
|
): Promise<Occupant[] | undefined> {
|
||||||
const children =
|
const children =
|
||||||
obstruction.kind === "file"
|
obstruction.kind === "file"
|
||||||
? [obstruction.path]
|
? [obstruction.path]
|
||||||
|
|
@ -36,29 +98,36 @@ export class FilePaths {
|
||||||
for (const child of children) {
|
for (const child of children) {
|
||||||
const occupant = Object.keys(current.local).find(
|
const occupant = Object.keys(current.local).find(
|
||||||
(key) =>
|
(key) =>
|
||||||
current.documents[key]?.observedHash !==
|
current.documents[key]?.observedHash !== undefined &&
|
||||||
undefined && current.local[key] === child
|
current.local[key] === child
|
||||||
);
|
);
|
||||||
const info = await this.fs.stat(child);
|
const info = await this.fs.stat(child);
|
||||||
if (
|
if (
|
||||||
!info ||
|
!info ||
|
||||||
this.isProtectedFile(child, info.size) ||
|
this.isProtectedFile(child, info.size) ||
|
||||||
(obstruction.kind === "directory" &&
|
(obstruction.kind === "directory" &&
|
||||||
(occupant === undefined ||
|
(occupant === undefined || next.local[occupant] === child))
|
||||||
next.local[occupant] === child))
|
) {
|
||||||
)
|
return undefined;
|
||||||
break;
|
}
|
||||||
|
|
||||||
occupants.push({ id: occupant ?? uuid(), path: child });
|
occupants.push({ id: occupant ?? uuid(), path: child });
|
||||||
}
|
}
|
||||||
if (occupants.length !== children.length) {
|
|
||||||
path = await this.conflictPath(next, id, path);
|
return occupants;
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async relocateOccupants(
|
||||||
|
occupants: Occupant[],
|
||||||
|
current: StoredDatabase,
|
||||||
|
next: StoredDatabase,
|
||||||
|
mutate: Mutation
|
||||||
|
): Promise<void> {
|
||||||
for (const occupant of occupants) {
|
for (const occupant of occupants) {
|
||||||
const displaced = await this.conflictPath(
|
const displaced = await this.allocateConflictPath(
|
||||||
next,
|
next,
|
||||||
occupant.id,
|
occupant.id,
|
||||||
occupant.path.split("/").at(-1)!
|
occupant.path.split("/").at(-1) ?? occupant.path
|
||||||
);
|
);
|
||||||
await mutate(async () => {
|
await mutate(async () => {
|
||||||
current.local[occupant.id] = displaced;
|
current.local[occupant.id] = displaced;
|
||||||
|
|
@ -68,16 +137,18 @@ export class FilePaths {
|
||||||
current.documents[occupant.id] = { observedHash: "" };
|
current.documents[occupant.id] = { observedHash: "" };
|
||||||
next.documents[occupant.id] = { observedHash: "" };
|
next.documents[occupant.id] = { observedHash: "" };
|
||||||
next.local[occupant.id] = displaced;
|
next.local[occupant.id] = displaced;
|
||||||
} else if (next.local[occupant.id] === occupant.path)
|
} else if (next.local[occupant.id] === occupant.path) {
|
||||||
next.local[occupant.id] = displaced;
|
next.local[occupant.id] = displaced;
|
||||||
}
|
}
|
||||||
if (obstruction.kind === "directory")
|
|
||||||
await mutate(() =>
|
|
||||||
this.fs.deleteDirectory(obstruction.path)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
await this.ensureParents(path);
|
}
|
||||||
} catch (error) {
|
|
||||||
|
private async recoverDestination(
|
||||||
|
next: StoredDatabase,
|
||||||
|
id: DocumentId,
|
||||||
|
path: RelativePath,
|
||||||
|
error: unknown
|
||||||
|
): Promise<RelativePath> {
|
||||||
const code =
|
const code =
|
||||||
typeof error === "object" && error !== null && "code" in error
|
typeof error === "object" && error !== null && "code" in error
|
||||||
? error.code
|
? error.code
|
||||||
|
|
@ -91,34 +162,36 @@ export class FilePaths {
|
||||||
"EACCES",
|
"EACCES",
|
||||||
"EPERM"
|
"EPERM"
|
||||||
].includes(String(code))
|
].includes(String(code))
|
||||||
)
|
) {
|
||||||
throw error;
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
const extension =
|
const extension =
|
||||||
/\.[^.]*$/u.exec(path.split("/").at(-1) ?? "")?.[0] ?? "";
|
/\.[^.]*$/u.exec(path.split("/").at(-1) ?? "")?.[0] ?? "";
|
||||||
path = await this.conflictPath(
|
path = await this.allocateConflictPath(
|
||||||
next,
|
next,
|
||||||
id,
|
id,
|
||||||
`Recovered ${id}${extension}`
|
`Recovered ${id}${extension}`
|
||||||
);
|
);
|
||||||
await this.ensureParents(path);
|
await this.createParentDirectories(path);
|
||||||
}
|
|
||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async obstruction(
|
private async findObstruction(
|
||||||
path: RelativePath
|
path: RelativePath
|
||||||
): Promise<{ path: RelativePath; kind: "file" | "directory" } | undefined> {
|
): Promise<PathObstruction | undefined> {
|
||||||
const parts = path.split("/");
|
const parts = path.split("/");
|
||||||
for (let i = 0; i < parts.length; i++) {
|
for (let i = 0; i < parts.length; i++) {
|
||||||
const prefix = parts.slice(0, i + 1).join("/");
|
const prefix = parts.slice(0, i + 1).join("/");
|
||||||
const info = await this.fs.stat(prefix);
|
const info = await this.fs.stat(prefix);
|
||||||
if (info && (info.kind === "file" || i === parts.length - 1))
|
if (info && (info.kind === "file" || i === parts.length - 1)) {
|
||||||
return { path: prefix, kind: info.kind };
|
return { path: prefix, kind: info.kind };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async conflictPath(
|
private async allocateConflictPath(
|
||||||
next: EngineState,
|
next: StoredDatabase,
|
||||||
id: DocumentId,
|
id: DocumentId,
|
||||||
path: RelativePath
|
path: RelativePath
|
||||||
): Promise<RelativePath> {
|
): Promise<RelativePath> {
|
||||||
|
|
@ -131,21 +204,10 @@ export class FilePaths {
|
||||||
return allocatePortablePath(path, id, [...occupied, ...files]);
|
return allocatePortablePath(path, id, [...occupied, ...files]);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async ensureParents(path: RelativePath): Promise<void> {
|
private async createParentDirectories(path: RelativePath): Promise<void> {
|
||||||
const parent = path.split("/").slice(0, -1).join("/");
|
const parent = path.split("/").slice(0, -1).join("/");
|
||||||
if (parent) await this.fs.createDirectory(parent);
|
if (parent) {
|
||||||
}
|
await this.fs.createDirectory(parent);
|
||||||
|
|
||||||
public async removeEmptyParents(path: RelativePath): Promise<void> {
|
|
||||||
let parent = path.split("/").slice(0, -1).join("/");
|
|
||||||
while (parent && !isInternalPath(parent)) {
|
|
||||||
if ((await this.fs.listFilesRecursively(parent)).length) break;
|
|
||||||
try {
|
|
||||||
await this.fs.deleteDirectory(parent);
|
|
||||||
} catch {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
parent = parent.split("/").slice(0, -1).join("/");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
import type { StoredClient } from "../sync-client";
|
||||||
|
import type { StoredDatabase } from "./database";
|
||||||
|
import type { MetadataPersistenceProvider } from "./metadata-persistence-provider";
|
||||||
|
import { Lock } from "../utils/data-structures/locks";
|
||||||
|
|
||||||
|
/** Serialize partial updates to the shared settings, database and notification store. */
|
||||||
|
export class ClientMetadataStore {
|
||||||
|
private readonly lock = new Lock();
|
||||||
|
private reloadBeforeSave = false;
|
||||||
|
|
||||||
|
private constructor(
|
||||||
|
public stored: StoredClient,
|
||||||
|
private readonly persistence: MetadataPersistenceProvider<StoredClient>
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public static async load(
|
||||||
|
persistence: MetadataPersistenceProvider<StoredClient>
|
||||||
|
): Promise<ClientMetadataStore> {
|
||||||
|
return new ClientMetadataStore(
|
||||||
|
(await persistence.load()) ?? {},
|
||||||
|
persistence
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async save(update: StoredClient): Promise<void> {
|
||||||
|
await this.lock.withLock(async () => {
|
||||||
|
if (this.reloadBeforeSave) {
|
||||||
|
await this.reload();
|
||||||
|
}
|
||||||
|
|
||||||
|
const next = { ...this.stored, ...structuredClone(update) };
|
||||||
|
try {
|
||||||
|
await this.persistence.save(next);
|
||||||
|
this.stored = next;
|
||||||
|
} catch (error) {
|
||||||
|
// A failed save may have committed. Reload before another update
|
||||||
|
// so settings or notifications cannot roll the database back.
|
||||||
|
this.reloadBeforeSave = true;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public async reloadDatabase(): Promise<StoredDatabase | undefined> {
|
||||||
|
return this.lock.withLock(async () => {
|
||||||
|
await this.reload();
|
||||||
|
return this.stored.database;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async reload(): Promise<void> {
|
||||||
|
this.stored = (await this.persistence.load()) ?? {};
|
||||||
|
this.reloadBeforeSave = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { test } from "node:test";
|
import { test } from "node:test";
|
||||||
import { Database, emptyState } from "./database";
|
import { Database, createEmptyDatabase } from "./database";
|
||||||
|
|
||||||
test("current state keeps absent optional fields absent across save and reload", async () => {
|
test("current state keeps absent optional fields absent across save and reload", async () => {
|
||||||
const initial = emptyState("vault");
|
const initial = createEmptyDatabase("vault");
|
||||||
initial.local = { local: "local.md", remote: "remote.md" };
|
initial.local = { local: "local.md", remote: "remote.md" };
|
||||||
initial.documents = {
|
initial.documents = {
|
||||||
local: { observedHash: "local" },
|
local: { observedHash: "local" },
|
||||||
|
|
@ -28,7 +28,7 @@ test("current state keeps absent optional fields absent across save and reload",
|
||||||
});
|
});
|
||||||
|
|
||||||
test("current reset state is restored after an uncertain save", async () => {
|
test("current reset state is restored after an uncertain save", async () => {
|
||||||
const saved = emptyState("vault");
|
const saved = createEmptyDatabase("vault");
|
||||||
saved.local = { note: "note.md" };
|
saved.local = { note: "note.md" };
|
||||||
saved.documents.note = {
|
saved.documents.note = {
|
||||||
base: { vaultUpdateId: null, hash: "clean" },
|
base: { vaultUpdateId: null, hash: "clean" },
|
||||||
|
|
|
||||||
|
|
@ -22,13 +22,17 @@ export interface DocumentState {
|
||||||
|
|
||||||
export type { PendingRequest } from "./requests";
|
export type { PendingRequest } from "./requests";
|
||||||
|
|
||||||
export interface EngineState {
|
export interface StoredDatabase {
|
||||||
vaultKey: string;
|
vaultKey: string;
|
||||||
|
/** Last incorporated server manifest; the base for the next three-way path merge. */
|
||||||
fileManifest: FileManifest;
|
fileManifest: FileManifest;
|
||||||
|
/** Current local document IDs and paths, including files awaiting upload or download. */
|
||||||
local: FileManifestEntries;
|
local: FileManifestEntries;
|
||||||
|
/** Per-document merge bases and observations; content bytes are not stored here. */
|
||||||
documents: Record<DocumentId, DocumentState>;
|
documents: Record<DocumentId, DocumentState>;
|
||||||
/** Absent until a vault snapshot is applied; also marks a durable history reset. */
|
/** Absent until a vault snapshot is applied; also marks a durable history reset. */
|
||||||
lastSeenUpdateId?: VaultUpdateId;
|
lastSeenUpdateId?: VaultUpdateId;
|
||||||
|
/** Exact request to resolve before observing a newer remote base. */
|
||||||
pending?: PendingRequest;
|
pending?: PendingRequest;
|
||||||
/** Commits consumption of the saved notification queue with the identity map. */
|
/** Commits consumption of the saved notification queue with the identity map. */
|
||||||
lastAppliedLocalChangeId?: string;
|
lastAppliedLocalChangeId?: string;
|
||||||
|
|
@ -36,9 +40,7 @@ export interface EngineState {
|
||||||
excluded?: Record<DocumentId, RelativePath | null>;
|
excluded?: Record<DocumentId, RelativePath | null>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type StoredDatabase = EngineState;
|
export function createEmptyDatabase(vaultKey: string): StoredDatabase {
|
||||||
|
|
||||||
export function emptyState(vaultKey: string): StoredDatabase {
|
|
||||||
return {
|
return {
|
||||||
vaultKey,
|
vaultKey,
|
||||||
fileManifest: { fileManifestId: 0, entries: {} },
|
fileManifest: { fileManifestId: 0, entries: {} },
|
||||||
|
|
@ -47,7 +49,7 @@ export function emptyState(vaultKey: string): StoredDatabase {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function canRebindEmpty(state: StoredDatabase): boolean {
|
function canBindToAnotherVault(state: StoredDatabase): boolean {
|
||||||
return (
|
return (
|
||||||
state.lastSeenUpdateId === undefined &&
|
state.lastSeenUpdateId === undefined &&
|
||||||
!state.pending &&
|
!state.pending &&
|
||||||
|
|
@ -66,32 +68,44 @@ export class Database {
|
||||||
) {
|
) {
|
||||||
const state =
|
const state =
|
||||||
initial === undefined
|
initial === undefined
|
||||||
? emptyState(vaultKey)
|
? createEmptyDatabase(vaultKey)
|
||||||
: structuredClone(initial);
|
: structuredClone(initial);
|
||||||
if (state.vaultKey !== vaultKey && !canRebindEmpty(state)) {
|
if (state.vaultKey !== vaultKey && !canBindToAnotherVault(state)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
|
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.state = { ...state, vaultKey };
|
this.state = { ...state, vaultKey };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public get length(): number {
|
||||||
|
return Object.keys(this.state.local).length;
|
||||||
|
}
|
||||||
|
|
||||||
/** Bind an empty database to the configured vault. */
|
/** Bind an empty database to the configured vault. */
|
||||||
public async bindVault(vaultKey: string): Promise<void> {
|
public async bindVault(vaultKey: string): Promise<void> {
|
||||||
if (this.state.vaultKey === vaultKey) return;
|
if (this.state.vaultKey === vaultKey) {
|
||||||
if (!canRebindEmpty(this.state))
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!canBindToAnotherVault(this.state)) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
|
"Incompatible vault identity. Preserve it for recovery and initialize a separate state store."
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
await this.commit({ ...this.state, vaultKey });
|
await this.commit({ ...this.state, vaultKey });
|
||||||
}
|
}
|
||||||
|
|
||||||
public async commit(next: StoredDatabase): Promise<void> {
|
public async commit(next: StoredDatabase): Promise<void> {
|
||||||
const snapshot = structuredClone(next);
|
const snapshot = structuredClone(next);
|
||||||
if (this.needsReload)
|
if (this.needsReload) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Reload saved state before retrying an uncertain save"
|
"Reload saved state before retrying an uncertain save"
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
this.needsReload = true;
|
this.needsReload = true;
|
||||||
await this.saveData(snapshot);
|
await this.saveData(snapshot);
|
||||||
this.state = snapshot;
|
this.state = snapshot;
|
||||||
|
|
@ -99,20 +113,26 @@ export class Database {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async recoverPersistence(): Promise<void> {
|
public async recoverPersistence(): Promise<void> {
|
||||||
if (!this.needsReload) return;
|
if (!this.needsReload) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const saved = await this.loadData();
|
const saved = await this.loadData();
|
||||||
if (saved) this.state = structuredClone(saved);
|
if (saved) {
|
||||||
|
this.state = structuredClone(saved);
|
||||||
|
}
|
||||||
|
|
||||||
this.needsReload = false;
|
this.needsReload = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public cloneState(): StoredDatabase {
|
||||||
|
return structuredClone(this.state);
|
||||||
|
}
|
||||||
|
|
||||||
public async save(): Promise<void> {
|
public async save(): Promise<void> {
|
||||||
await this.commit(this.state);
|
await this.commit(this.state);
|
||||||
}
|
}
|
||||||
|
|
||||||
public get length(): number {
|
|
||||||
return Object.keys(this.state.local).length;
|
|
||||||
}
|
|
||||||
|
|
||||||
public findDocumentId(path: RelativePath): DocumentId | undefined {
|
public findDocumentId(path: RelativePath): DocumentId | undefined {
|
||||||
return Object.keys(this.state.local).find(
|
return Object.keys(this.state.local).find(
|
||||||
(id) => this.state.local[id] === path
|
(id) => this.state.local[id] === path
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ describe("Settings logging", () => {
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ token: "loaded-secret" },
|
{ token: "loaded-secret" },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
await settings.setSettings({ token: "updated-secret" });
|
await settings.setSettings({ token: "updated-secret" });
|
||||||
|
|
||||||
|
|
@ -43,7 +43,7 @@ it("owns settings arrays instead of sharing them with callers and listeners", as
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
new Logger(),
|
new Logger(),
|
||||||
{ ignorePatterns: original },
|
{ ignorePatterns: original },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
original.push("external mutation");
|
original.push("external mutation");
|
||||||
assert.deepEqual(settings.getSettings().ignorePatterns, ["original"]);
|
assert.deepEqual(settings.getSettings().ignorePatterns, ["original"]);
|
||||||
|
|
@ -67,7 +67,9 @@ it("file eligibility follows successfully saved settings", async () => {
|
||||||
maxFileSizeMB: 1
|
maxFileSizeMB: 1
|
||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
if (fail) throw new Error("save failed");
|
if (fail) {
|
||||||
|
throw new Error("save failed");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
assert(settings.isIgnored("private/note.md"));
|
assert(settings.isIgnored("private/note.md"));
|
||||||
|
|
|
||||||
|
|
@ -112,17 +112,20 @@ export class Settings {
|
||||||
await this.saveData(next);
|
await this.saveData(next);
|
||||||
this.settings = next;
|
this.settings = next;
|
||||||
|
|
||||||
if (update.ignorePatterns !== undefined)
|
if (update.ignorePatterns !== undefined) {
|
||||||
this.ignorePatterns = globsToRegexes(
|
this.ignorePatterns = globsToRegexes(
|
||||||
next.ignorePatterns,
|
next.ignorePatterns,
|
||||||
this.logger
|
this.logger
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
return [this.getSettings(), oldSettings] as const;
|
return [this.getSettings(), oldSettings] as const;
|
||||||
});
|
});
|
||||||
|
|
||||||
// Listeners can themselves update settings. Never await them while
|
// Listeners can themselves update settings. Never await them while
|
||||||
// holding a lock needed by the operation they invoke.
|
// holding a lock needed by the operation they invoke.
|
||||||
if (notify) await this.onSettingsChanged.triggerAsync(...change);
|
if (notify) {
|
||||||
|
await this.onSettingsChanged.triggerAsync(...change);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { awaitAll } from "../utils/await-all";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { test } from "node:test";
|
import { test } from "node:test";
|
||||||
import { ServerConfig } from "./server-config";
|
import { ServerConfig } from "./server-config";
|
||||||
|
|
@ -12,28 +13,31 @@ const config = {
|
||||||
mergeableFileExtensions: ["md"],
|
mergeableFileExtensions: ["md"],
|
||||||
serverVersion: "test"
|
serverVersion: "test"
|
||||||
};
|
};
|
||||||
const make = (fetch: typeof globalThis.fetch) =>
|
const make = (fetch: typeof globalThis.fetch): ServerConfig =>
|
||||||
new ServerConfig(
|
new ServerConfig(
|
||||||
new SyncService(
|
new SyncService(
|
||||||
"device",
|
"device",
|
||||||
new Settings(
|
new Settings(
|
||||||
new Logger(),
|
new Logger(),
|
||||||
{ remoteUri: "http://test" },
|
{ remoteUri: "http://test" },
|
||||||
async () => {}
|
async () => undefined
|
||||||
),
|
),
|
||||||
fetch,
|
fetch,
|
||||||
{ get: () => undefined, save: async () => {} }
|
{ get: () => undefined, save: async () => undefined }
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
test("configuration shares one validated request and retries failures", async () => {
|
test("configuration shares one validated request and retries failures", async () => {
|
||||||
let calls = 0;
|
let calls = 0;
|
||||||
const server = make(async () => {
|
const server = make(async () => {
|
||||||
if (++calls === 1) throw new Error("offline");
|
if (++calls === 1) {
|
||||||
|
throw new Error("offline");
|
||||||
|
}
|
||||||
|
|
||||||
return Response.json(config);
|
return Response.json(config);
|
||||||
});
|
});
|
||||||
await assert.rejects(server.getConfig(), /offline/);
|
await assert.rejects(server.getConfig(), /offline/);
|
||||||
const [a, b] = await Promise.all([server.getConfig(), server.getConfig()]);
|
const [a, b] = await awaitAll([server.getConfig(), server.getConfig()]);
|
||||||
assert.equal(a, b);
|
assert.equal(a, b);
|
||||||
assert.equal(calls, 2);
|
assert.equal(calls, 2);
|
||||||
await server.getConfig();
|
await server.getConfig();
|
||||||
|
|
@ -74,6 +78,7 @@ for (const fail of [false, true]) {
|
||||||
);
|
);
|
||||||
await old;
|
await old;
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.equal(await server.getConfig(), current);
|
assert.equal(await server.getConfig(), current);
|
||||||
assert.equal(calls, 2);
|
assert.equal(calls, 2);
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -59,12 +59,16 @@ export class ServerConfig {
|
||||||
return config;
|
return config;
|
||||||
})
|
})
|
||||||
.catch((error: unknown) => {
|
.catch((error: unknown) => {
|
||||||
if (this.config === pending) this.config = undefined;
|
if (this.config === pending) {
|
||||||
|
this.config = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
throw error;
|
throw error;
|
||||||
});
|
});
|
||||||
|
|
||||||
this.config = pending;
|
this.config = pending;
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.config;
|
return this.config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import { Logger } from "../tracing/logger";
|
||||||
import { SyncResetError } from "../errors/errors";
|
import { SyncResetError } from "../errors/errors";
|
||||||
import { SyncService } from "./sync-service";
|
import { SyncService } from "./sync-service";
|
||||||
|
|
||||||
const settings = () =>
|
const settings = (): Settings =>
|
||||||
new Settings(
|
new Settings(
|
||||||
new Logger(),
|
new Logger(),
|
||||||
{
|
{
|
||||||
|
|
@ -13,18 +13,26 @@ const settings = () =>
|
||||||
vaultName: "test",
|
vaultName: "test",
|
||||||
requestTimeoutMs: 1000
|
requestTimeoutMs: 1000
|
||||||
},
|
},
|
||||||
async () => { }
|
async () => undefined
|
||||||
);
|
);
|
||||||
|
|
||||||
test("paused sync sends no requests, while connection checks still work", async () => {
|
test("paused sync sends no requests, while connection checks still work", async (): Promise<void> => {
|
||||||
let requests = 0;
|
let requests = 0;
|
||||||
const service = new SyncService("device", settings(), async () => {
|
const service = new SyncService(
|
||||||
|
"device",
|
||||||
|
settings(),
|
||||||
|
async () => {
|
||||||
requests++;
|
requests++;
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ headEventId: 0, endEventId: 0, events: [] },
|
{ headEventId: 0, endEventId: 0, events: [] },
|
||||||
{ headers: { "X-Vault-Link-History": "0:empty" } }
|
{ headers: { "X-Vault-Link-History": "0:empty" } }
|
||||||
);
|
);
|
||||||
}, { get: () => undefined, save: async () => { } });
|
},
|
||||||
|
{
|
||||||
|
get: (): undefined => undefined,
|
||||||
|
save: async (): Promise<void> => undefined
|
||||||
|
}
|
||||||
|
);
|
||||||
await assert.rejects(service.getEvents(0), SyncResetError);
|
await assert.rejects(service.getEvents(0), SyncResetError);
|
||||||
assert.equal(requests, 0);
|
assert.equal(requests, 0);
|
||||||
await service.ping();
|
await service.ping();
|
||||||
|
|
@ -34,7 +42,7 @@ test("paused sync sends no requests, while connection checks still work", async
|
||||||
assert.equal(requests, 2);
|
assert.equal(requests, 2);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("pause aborts requests and fences late responses from an injected fetch", async () => {
|
test("pause aborts requests and fences late responses from an injected fetch", async (): Promise<void> => {
|
||||||
const response = Promise.withResolvers<Response>();
|
const response = Promise.withResolvers<Response>();
|
||||||
const signals: AbortSignal[] = [];
|
const signals: AbortSignal[] = [];
|
||||||
const saved: (string | undefined)[] = [];
|
const saved: (string | undefined)[] = [];
|
||||||
|
|
@ -42,7 +50,8 @@ test("pause aborts requests and fences late responses from an injected fetch", a
|
||||||
"device",
|
"device",
|
||||||
settings(),
|
settings(),
|
||||||
async (_, init) => {
|
async (_, init) => {
|
||||||
signals.push(init!.signal!);
|
assert.ok(init?.signal);
|
||||||
|
signals.push(init.signal);
|
||||||
return signals.length === 1
|
return signals.length === 1
|
||||||
? response.promise
|
? response.promise
|
||||||
: Response.json(
|
: Response.json(
|
||||||
|
|
@ -51,8 +60,8 @@ test("pause aborts requests and fences late responses from an injected fetch", a
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
get: () => undefined,
|
get: (): undefined => undefined,
|
||||||
save: async (value) => {
|
save: async (value): Promise<void> => {
|
||||||
saved.push(value);
|
saved.push(value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -61,10 +70,10 @@ test("pause aborts requests and fences late responses from an injected fetch", a
|
||||||
const first = service.getEvents(0);
|
const first = service.getEvents(0);
|
||||||
service.pause();
|
service.pause();
|
||||||
await assert.rejects(first, SyncResetError);
|
await assert.rejects(first, SyncResetError);
|
||||||
assert.equal(signals[0].aborted, true);
|
assert.equal(signals[0]?.aborted, true);
|
||||||
service.resume();
|
service.resume();
|
||||||
await service.getEvents(0);
|
await service.getEvents(0);
|
||||||
assert.equal(signals[1].aborted, false);
|
assert.equal(signals[1]?.aborted, false);
|
||||||
response.resolve(
|
response.resolve(
|
||||||
Response.json(
|
Response.json(
|
||||||
{},
|
{},
|
||||||
|
|
@ -76,25 +85,28 @@ test("pause aborts requests and fences late responses from an injected fetch", a
|
||||||
});
|
});
|
||||||
|
|
||||||
for (const binary of [false, true]) {
|
for (const binary of [false, true]) {
|
||||||
test(`pause interrupts a stalled ${binary ? "binary" : "JSON"} response body`, async () => {
|
test(`pause interrupts a stalled ${binary ? "binary" : "JSON"} response body`, async (): Promise<void> => {
|
||||||
const headersSaved = Promise.withResolvers<void>();
|
const headersSaved = Promise.withResolvers<undefined>();
|
||||||
let body!: ReadableStreamDefaultController<Uint8Array>;
|
const bodyReady =
|
||||||
|
Promise.withResolvers<
|
||||||
|
ReadableStreamDefaultController<Uint8Array>
|
||||||
|
>();
|
||||||
const service = new SyncService(
|
const service = new SyncService(
|
||||||
"device",
|
"device",
|
||||||
settings(),
|
settings(),
|
||||||
async () =>
|
async () =>
|
||||||
new Response(
|
new Response(
|
||||||
new ReadableStream({
|
new ReadableStream<Uint8Array>({
|
||||||
start(controller) {
|
start(controller): void {
|
||||||
body = controller;
|
bodyReady.resolve(controller);
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
{ headers: { "X-Vault-Link-History": "1:head" } }
|
{ headers: { "X-Vault-Link-History": "1:head" } }
|
||||||
),
|
),
|
||||||
{
|
{
|
||||||
get: () => undefined,
|
get: (): undefined => undefined,
|
||||||
save: async () => {
|
save: async (): Promise<void> => {
|
||||||
headersSaved.resolve();
|
headersSaved.resolve(undefined);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
@ -109,16 +121,17 @@ for (const binary of [false, true]) {
|
||||||
await new Promise((resolve) => setImmediate(resolve));
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
service.pause();
|
service.pause();
|
||||||
await assert.rejects(request, SyncResetError);
|
await assert.rejects(request, SyncResetError);
|
||||||
|
const body = await bodyReady.promise;
|
||||||
body.enqueue(new TextEncoder().encode("{}"));
|
body.enqueue(new TextEncoder().encode("{}"));
|
||||||
body.close();
|
body.close();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
test("pause waits for a checkpoint save already in progress", async () => {
|
test("pause waits for a checkpoint save already in progress", async (): Promise<void> => {
|
||||||
const saving = Promise.withResolvers<void>();
|
const saving = Promise.withResolvers<undefined>();
|
||||||
const release = Promise.withResolvers<void>();
|
const release = Promise.withResolvers<undefined>();
|
||||||
let settled = false;
|
let settled = false;
|
||||||
let checkpoint: string | undefined;
|
let checkpoint: string | undefined = undefined;
|
||||||
const service = new SyncService(
|
const service = new SyncService(
|
||||||
"device",
|
"device",
|
||||||
settings(),
|
settings(),
|
||||||
|
|
@ -130,9 +143,9 @@ test("pause waits for a checkpoint save already in progress", async () => {
|
||||||
}
|
}
|
||||||
),
|
),
|
||||||
{
|
{
|
||||||
get: () => checkpoint,
|
get: (): string | undefined => checkpoint,
|
||||||
save: async (value) => {
|
save: async (value): Promise<void> => {
|
||||||
saving.resolve();
|
saving.resolve(undefined);
|
||||||
await release.promise;
|
await release.promise;
|
||||||
checkpoint = value;
|
checkpoint = value;
|
||||||
}
|
}
|
||||||
|
|
@ -146,13 +159,13 @@ test("pause waits for a checkpoint save already in progress", async () => {
|
||||||
service.pause();
|
service.pause();
|
||||||
await new Promise((resolve) => setImmediate(resolve));
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
assert.equal(settled, false);
|
assert.equal(settled, false);
|
||||||
release.resolve();
|
release.resolve(undefined);
|
||||||
await assert.rejects(request, SyncResetError);
|
await assert.rejects(request, SyncResetError);
|
||||||
assert.equal(checkpoint, "1:head");
|
assert.equal(checkpoint, "1:head");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("history saves only new checkpoints, including a changed token at the same event ID", async () => {
|
test("history saves only new checkpoints, including a changed token at the same event ID", async (): Promise<void> => {
|
||||||
let checkpoint: string | undefined;
|
let checkpoint: string | undefined = undefined;
|
||||||
let responseCheckpoint = "1:first";
|
let responseCheckpoint = "1:first";
|
||||||
const saved: (string | undefined)[] = [];
|
const saved: (string | undefined)[] = [];
|
||||||
const service = new SyncService(
|
const service = new SyncService(
|
||||||
|
|
@ -164,8 +177,8 @@ test("history saves only new checkpoints, including a changed token at the same
|
||||||
{ headers: { "X-Vault-Link-History": responseCheckpoint } }
|
{ headers: { "X-Vault-Link-History": responseCheckpoint } }
|
||||||
),
|
),
|
||||||
{
|
{
|
||||||
get: () => checkpoint,
|
get: (): string | undefined => checkpoint,
|
||||||
save: async (value) => {
|
save: async (value): Promise<void> => {
|
||||||
checkpoint = value;
|
checkpoint = value;
|
||||||
saved.push(value);
|
saved.push(value);
|
||||||
}
|
}
|
||||||
|
|
@ -183,6 +196,7 @@ test("history saves only new checkpoints, including a changed token at the same
|
||||||
responseCheckpoint = value;
|
responseCheckpoint = value;
|
||||||
await service.getEvents(0);
|
await service.getEvents(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.deepEqual(saved, ["1:first", "2:next", "2:replacement"]);
|
assert.deepEqual(saved, ["1:first", "2:next", "2:replacement"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -195,7 +209,8 @@ for (const checkpoint of [undefined, ""]) {
|
||||||
Response.json(
|
Response.json(
|
||||||
{ headEventId: 0, endEventId: 0, events: [] },
|
{ headEventId: 0, endEventId: 0, events: [] },
|
||||||
{
|
{
|
||||||
headers: checkpoint === undefined
|
headers:
|
||||||
|
checkpoint === undefined
|
||||||
? {}
|
? {}
|
||||||
: { "X-Vault-Link-History": checkpoint }
|
: { "X-Vault-Link-History": checkpoint }
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,17 +17,19 @@ import {
|
||||||
} from "../errors/errors";
|
} from "../errors/errors";
|
||||||
|
|
||||||
export class SyncService {
|
export class SyncService {
|
||||||
|
private readonly fetchImplementation: typeof fetch;
|
||||||
private session = new AbortController();
|
private session = new AbortController();
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
private readonly deviceId: string,
|
private readonly deviceId: string,
|
||||||
private readonly settings: Settings,
|
private readonly settings: Settings,
|
||||||
private readonly fetchImplementation: typeof fetch = globalThis.fetch,
|
fetchImplementation: typeof fetch | undefined,
|
||||||
private readonly history: {
|
private readonly history: {
|
||||||
get: () => string | undefined;
|
get: () => string | undefined;
|
||||||
save: (checkpoint: string | undefined) => Promise<void>;
|
save: (checkpoint: string | undefined) => Promise<void>;
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
|
this.fetchImplementation = fetchImplementation ?? globalThis.fetch;
|
||||||
this.pause();
|
this.pause();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -92,16 +94,22 @@ export class SyncService {
|
||||||
|
|
||||||
private async recordHistory(response: Response): Promise<void> {
|
private async recordHistory(response: Response): Promise<void> {
|
||||||
const checkpoint = response.headers.get("x-vault-link-history");
|
const checkpoint = response.headers.get("x-vault-link-history");
|
||||||
if (checkpoint === null || checkpoint === "")
|
if (checkpoint === null || checkpoint === "") {
|
||||||
throw new PermanentSyncError("Missing server history checkpoint");
|
throw new PermanentSyncError("Missing server history checkpoint");
|
||||||
|
}
|
||||||
|
|
||||||
const previous = this.history.get();
|
const previous = this.history.get();
|
||||||
if (checkpoint === previous) return;
|
if (checkpoint === previous) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
previous === undefined ||
|
previous === undefined ||
|
||||||
Number(checkpoint.split(":")[0]) >= Number(previous.split(":")[0])
|
Number(checkpoint.split(":")[0]) >= Number(previous.split(":")[0])
|
||||||
)
|
) {
|
||||||
await this.history.save(checkpoint);
|
await this.history.save(checkpoint);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async request<T>(
|
private async request<T>(
|
||||||
path: string,
|
path: string,
|
||||||
|
|
@ -143,10 +151,11 @@ export class SyncService {
|
||||||
);
|
);
|
||||||
|
|
||||||
signal.throwIfAborted();
|
signal.throwIfAborted();
|
||||||
if (response.headers.get("x-vault-link-history-mismatch") === "1")
|
if (response.headers.get("x-vault-link-history-mismatch") === "1") {
|
||||||
throw new ServerHistoryChangedError(
|
throw new ServerHistoryChangedError(
|
||||||
"Server history changed; recovering local work"
|
"Server history changed; recovering local work"
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
this.throwForErrorResponse(
|
this.throwForErrorResponse(
|
||||||
|
|
@ -156,7 +165,9 @@ export class SyncService {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Finish metadata saves even if the transport is paused meanwhile.
|
// Finish metadata saves even if the transport is paused meanwhile.
|
||||||
if (!independent) await this.recordHistory(response);
|
if (!independent) {
|
||||||
|
await this.recordHistory(response);
|
||||||
|
}
|
||||||
|
|
||||||
return abortable(signal, async () => decode(response));
|
return abortable(signal, async () => decode(response));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ class MockWebSocket {
|
||||||
|
|
||||||
public sentMessages: string[] = [];
|
public sentMessages: string[] = [];
|
||||||
|
|
||||||
public constructor(public url: string) {
|
public constructor(public url: string | URL) {
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
if (this.readyState === WebSocket.CONNECTING) {
|
if (this.readyState === WebSocket.CONNECTING) {
|
||||||
this.readyState = WebSocket.OPEN;
|
this.readyState = WebSocket.OPEN;
|
||||||
|
|
@ -54,6 +54,7 @@ class MockWebSocket {
|
||||||
if (this.readyState !== WebSocket.OPEN) {
|
if (this.readyState !== WebSocket.OPEN) {
|
||||||
throw new Error("WebSocket is not open");
|
throw new Error("WebSocket is not open");
|
||||||
}
|
}
|
||||||
|
|
||||||
this.sentMessages.push(data);
|
this.sentMessages.push(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -100,6 +101,7 @@ describe("WebSocketManager", () => {
|
||||||
const noop = (): void => {
|
const noop = (): void => {
|
||||||
// Intentionally empty for mock
|
// Intentionally empty for mock
|
||||||
};
|
};
|
||||||
|
|
||||||
mockLogger = {
|
mockLogger = {
|
||||||
info: createMockFn(noop),
|
info: createMockFn(noop),
|
||||||
warn: createMockFn(noop),
|
warn: createMockFn(noop),
|
||||||
|
|
@ -344,9 +346,15 @@ it(
|
||||||
server.listen(0, "127.0.0.1", resolve)
|
server.listen(0, "127.0.0.1", resolve)
|
||||||
);
|
);
|
||||||
const address = server.address();
|
const address = server.address();
|
||||||
assert(address && typeof address !== "string");
|
assert(address !== null && typeof address !== "string");
|
||||||
await new Promise<void>((resolve, reject) =>
|
await new Promise<void>((resolve, reject) =>
|
||||||
server.close((error) => (error ? reject(error) : resolve()))
|
server.close((error) => {
|
||||||
|
if (error !== undefined) {
|
||||||
|
reject(error);
|
||||||
|
} else {
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
})
|
||||||
);
|
);
|
||||||
const logger = new Logger();
|
const logger = new Logger();
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
|
|
@ -355,7 +363,7 @@ it(
|
||||||
remoteUri: `http://127.0.0.1:${address.port}`,
|
remoteUri: `http://127.0.0.1:${address.port}`,
|
||||||
webSocketRetryIntervalMs: 10
|
webSocketRetryIntervalMs: 10
|
||||||
},
|
},
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
const manager = new WebSocketManager(
|
const manager = new WebSocketManager(
|
||||||
"test",
|
"test",
|
||||||
|
|
@ -364,9 +372,11 @@ it(
|
||||||
WebSocket
|
WebSocket
|
||||||
);
|
);
|
||||||
let closes = 0;
|
let closes = 0;
|
||||||
const disconnected = Promise.withResolvers<void>();
|
const disconnected = Promise.withResolvers<undefined>();
|
||||||
manager.onWebSocketStatusChanged.add((connected) => {
|
manager.onWebSocketStatusChanged.add((connected) => {
|
||||||
if (!connected && ++closes >= 2) disconnected.resolve();
|
if (!connected && ++closes >= 2) {
|
||||||
|
disconnected.resolve(undefined);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
manager.start();
|
manager.start();
|
||||||
|
|
@ -384,10 +394,12 @@ for (const failedAttempt of [1, 2]) {
|
||||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||||
let attempts = 0;
|
let attempts = 0;
|
||||||
class FailingSocket extends MockWebSocket {
|
class FailingSocket extends MockWebSocket {
|
||||||
static OPEN = 1;
|
public static OPEN = 1;
|
||||||
constructor(url: string) {
|
public constructor(url: string) {
|
||||||
if (++attempts === failedAttempt)
|
if (++attempts === failedAttempt) {
|
||||||
throw new Error("socket allocation failed");
|
throw new Error("socket allocation failed");
|
||||||
|
}
|
||||||
|
|
||||||
super(url);
|
super(url);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -395,7 +407,7 @@ for (const failedAttempt of [1, 2]) {
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
const manager = new WebSocketManager(
|
const manager = new WebSocketManager(
|
||||||
"test",
|
"test",
|
||||||
|
|
@ -404,7 +416,9 @@ for (const failedAttempt of [1, 2]) {
|
||||||
FailingSocket as unknown as typeof WebSocket
|
FailingSocket as unknown as typeof WebSocket
|
||||||
);
|
);
|
||||||
try {
|
try {
|
||||||
assert.doesNotThrow(() => manager.start());
|
assert.doesNotThrow(() => {
|
||||||
|
manager.start();
|
||||||
|
});
|
||||||
context.mock.timers.tick(0);
|
context.mock.timers.tick(0);
|
||||||
if (failedAttempt === 2) {
|
if (failedAttempt === 2) {
|
||||||
const socket = (
|
const socket = (
|
||||||
|
|
@ -412,7 +426,10 @@ for (const failedAttempt of [1, 2]) {
|
||||||
).webSocket;
|
).webSocket;
|
||||||
socket.close();
|
socket.close();
|
||||||
}
|
}
|
||||||
assert.doesNotThrow(() => context.mock.timers.tick(1000));
|
|
||||||
|
assert.doesNotThrow(() => {
|
||||||
|
context.mock.timers.tick(1000);
|
||||||
|
});
|
||||||
context.mock.timers.tick(1000);
|
context.mock.timers.tick(1000);
|
||||||
assert.equal(attempts, failedAttempt + 1);
|
assert.equal(attempts, failedAttempt + 1);
|
||||||
assert.equal(manager.isWebSocketConnected, true);
|
assert.equal(manager.isWebSocketConnected, true);
|
||||||
|
|
@ -426,16 +443,18 @@ for (const closeBehavior of ["throws", "stays open"]) {
|
||||||
it(`stop retires a socket even when close ${closeBehavior}`, async (context) => {
|
it(`stop retires a socket even when close ${closeBehavior}`, async (context) => {
|
||||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||||
class BrokenCloseSocket extends MockWebSocket {
|
class BrokenCloseSocket extends MockWebSocket {
|
||||||
static OPEN = 1;
|
public static OPEN = 1;
|
||||||
close() {
|
public close(): void {
|
||||||
if (closeBehavior === "throws") throw new Error("close failed");
|
if (closeBehavior === "throws") {
|
||||||
|
throw new Error("close failed");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const logger = new Logger();
|
const logger = new Logger();
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ remoteUri: "http://test" },
|
{ remoteUri: "http://test" },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
const manager = new WebSocketManager(
|
const manager = new WebSocketManager(
|
||||||
"test",
|
"test",
|
||||||
|
|
@ -481,20 +500,20 @@ it("stop retires the socket immediately but drains messages already being handle
|
||||||
new Settings(
|
new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ remoteUri: "http://test", token: "test-token" },
|
{ remoteUri: "http://test", token: "test-token" },
|
||||||
async () => {}
|
async () => undefined
|
||||||
),
|
),
|
||||||
MockWebSocket as unknown as typeof WebSocket
|
MockWebSocket as unknown as typeof WebSocket
|
||||||
);
|
);
|
||||||
const release = Promise.withResolvers<void>();
|
const release = Promise.withResolvers<undefined>();
|
||||||
context.after(async () => {
|
context.after(async () => {
|
||||||
release.resolve();
|
release.resolve(undefined);
|
||||||
await manager.stop();
|
await manager.stop();
|
||||||
});
|
});
|
||||||
manager.start();
|
manager.start();
|
||||||
context.mock.timers.tick(0);
|
context.mock.timers.tick(0);
|
||||||
const socket = (manager as unknown as { webSocket: MockWebSocket })
|
const socket = (manager as unknown as { webSocket: MockWebSocket })
|
||||||
.webSocket;
|
.webSocket;
|
||||||
assert.deepEqual(JSON.parse(socket.sentMessages[0]), {
|
assert.deepEqual(JSON.parse(socket.sentMessages[0] ?? ""), {
|
||||||
type: "handshake",
|
type: "handshake",
|
||||||
token: "test-token",
|
token: "test-token",
|
||||||
deviceId: "device"
|
deviceId: "device"
|
||||||
|
|
@ -509,7 +528,7 @@ it("stop retires the socket immediately but drains messages already being handle
|
||||||
assert.equal(socket.onmessage, null);
|
assert.equal(socket.onmessage, null);
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
assert(!finished);
|
assert(!finished);
|
||||||
release.resolve();
|
release.resolve(undefined);
|
||||||
await stopping;
|
await stopping;
|
||||||
assert(finished);
|
assert(finished);
|
||||||
});
|
});
|
||||||
|
|
@ -517,8 +536,8 @@ it("stop retires the socket immediately but drains messages already being handle
|
||||||
it("a handshake send failure retires the connection without escaping the open callback", async (context) => {
|
it("a handshake send failure retires the connection without escaping the open callback", async (context) => {
|
||||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||||
class BrokenSendSocket extends MockWebSocket {
|
class BrokenSendSocket extends MockWebSocket {
|
||||||
static OPEN = 1;
|
public static OPEN = 1;
|
||||||
send() {
|
public send(): void {
|
||||||
throw new Error("transport closed during open");
|
throw new Error("transport closed during open");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -526,7 +545,7 @@ it("a handshake send failure retires the connection without escaping the open ca
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ remoteUri: "http://test" },
|
{ remoteUri: "http://test" },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
const manager = new WebSocketManager(
|
const manager = new WebSocketManager(
|
||||||
"test",
|
"test",
|
||||||
|
|
@ -538,7 +557,9 @@ it("a handshake send failure retires the connection without escaping the open ca
|
||||||
manager.onWebSocketStatusChanged.add((status) => statuses.push(status));
|
manager.onWebSocketStatusChanged.add((status) => statuses.push(status));
|
||||||
try {
|
try {
|
||||||
manager.start();
|
manager.start();
|
||||||
assert.doesNotThrow(() => context.mock.timers.tick(0));
|
assert.doesNotThrow(() => {
|
||||||
|
context.mock.timers.tick(0);
|
||||||
|
});
|
||||||
assert.equal(manager.isWebSocketConnected, false);
|
assert.equal(manager.isWebSocketConnected, false);
|
||||||
assert.equal(statuses.at(-1), false);
|
assert.equal(statuses.at(-1), false);
|
||||||
assert(
|
assert(
|
||||||
|
|
@ -554,8 +575,8 @@ it("reconnects a connection that stops delivering messages", async (context) =>
|
||||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||||
let attempts = 0;
|
let attempts = 0;
|
||||||
class CountingSocket extends MockWebSocket {
|
class CountingSocket extends MockWebSocket {
|
||||||
static OPEN = 1;
|
public static OPEN = 1;
|
||||||
constructor(url: string) {
|
public constructor(url: string) {
|
||||||
super(url);
|
super(url);
|
||||||
attempts++;
|
attempts++;
|
||||||
}
|
}
|
||||||
|
|
@ -564,7 +585,7 @@ it("reconnects a connection that stops delivering messages", async (context) =>
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
const manager = new WebSocketManager(
|
const manager = new WebSocketManager(
|
||||||
"test",
|
"test",
|
||||||
|
|
@ -601,8 +622,8 @@ it("does not reconnect while messages keep arriving before the deadline", async
|
||||||
context.mock.timers.enable({ apis: ["setTimeout"] });
|
context.mock.timers.enable({ apis: ["setTimeout"] });
|
||||||
let attempts = 0;
|
let attempts = 0;
|
||||||
class CountingSocket extends MockWebSocket {
|
class CountingSocket extends MockWebSocket {
|
||||||
static OPEN = 1;
|
public static OPEN = 1;
|
||||||
constructor(url: string) {
|
public constructor(url: string) {
|
||||||
super(url);
|
super(url);
|
||||||
attempts++;
|
attempts++;
|
||||||
}
|
}
|
||||||
|
|
@ -611,7 +632,7 @@ it("does not reconnect while messages keep arriving before the deadline", async
|
||||||
const settings = new Settings(
|
const settings = new Settings(
|
||||||
logger,
|
logger,
|
||||||
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
{ remoteUri: "http://test", webSocketRetryIntervalMs: 1000 },
|
||||||
async () => {}
|
async () => undefined
|
||||||
);
|
);
|
||||||
const manager = new WebSocketManager(
|
const manager = new WebSocketManager(
|
||||||
"test",
|
"test",
|
||||||
|
|
@ -621,7 +642,9 @@ it("does not reconnect while messages keep arriving before the deadline", async
|
||||||
);
|
);
|
||||||
let disconnected = 0;
|
let disconnected = 0;
|
||||||
manager.onWebSocketStatusChanged.add((status) => {
|
manager.onWebSocketStatusChanged.add((status) => {
|
||||||
if (!status) disconnected++;
|
if (!status) {
|
||||||
|
disconnected++;
|
||||||
|
}
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
manager.start();
|
manager.start();
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,10 @@ export class WebSocketManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
public start(): void {
|
public start(): void {
|
||||||
if (!this.isStopped) return;
|
if (!this.isStopped) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.isStopped = false;
|
this.isStopped = false;
|
||||||
this.initializeWebSocket();
|
this.initializeWebSocket();
|
||||||
}
|
}
|
||||||
|
|
@ -106,7 +109,10 @@ export class WebSocketManager {
|
||||||
// proves nothing: silence past the heartbeat deadline means a
|
// proves nothing: silence past the heartbeat deadline means a
|
||||||
// half-open connection.
|
// half-open connection.
|
||||||
this.receiveTimeoutId = setTimeout(() => {
|
this.receiveTimeoutId = setTimeout(() => {
|
||||||
if (this.webSocket !== socket || this.isStopped) return;
|
if (this.webSocket !== socket || this.isStopped) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.logger.warn("WebSocket receive deadline expired");
|
this.logger.warn("WebSocket receive deadline expired");
|
||||||
this.reconnect(socket);
|
this.reconnect(socket);
|
||||||
}, WEBSOCKET_RECEIVE_TIMEOUT_MS);
|
}, WEBSOCKET_RECEIVE_TIMEOUT_MS);
|
||||||
|
|
@ -117,7 +123,10 @@ export class WebSocketManager {
|
||||||
private retireSocket(): void {
|
private retireSocket(): void {
|
||||||
clearTimeout(this.receiveTimeoutId);
|
clearTimeout(this.receiveTimeoutId);
|
||||||
const socket = this.webSocket;
|
const socket = this.webSocket;
|
||||||
if (!socket) return;
|
if (!socket) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.webSocket = undefined;
|
this.webSocket = undefined;
|
||||||
socket.onopen = null;
|
socket.onopen = null;
|
||||||
socket.onclose = null;
|
socket.onclose = null;
|
||||||
|
|
@ -125,6 +134,7 @@ export class WebSocketManager {
|
||||||
socket.onerror = (): void => {
|
socket.onerror = (): void => {
|
||||||
/* Retired transports may still emit errors. */
|
/* Retired transports may still emit errors. */
|
||||||
};
|
};
|
||||||
|
|
||||||
this.onWebSocketStatusChanged.trigger(false);
|
this.onWebSocketStatusChanged.trigger(false);
|
||||||
try {
|
try {
|
||||||
socket.close(1000, "WebSocket retired");
|
socket.close(1000, "WebSocket retired");
|
||||||
|
|
@ -134,13 +144,19 @@ export class WebSocketManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
private reconnect(socket: WebSocket): void {
|
private reconnect(socket: WebSocket): void {
|
||||||
if (this.webSocket !== socket) return;
|
if (this.webSocket !== socket) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.retireSocket();
|
this.retireSocket();
|
||||||
this.scheduleReconnect();
|
this.scheduleReconnect();
|
||||||
}
|
}
|
||||||
|
|
||||||
private scheduleReconnect(): void {
|
private scheduleReconnect(): void {
|
||||||
if (this.isStopped) return;
|
if (this.isStopped) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
clearTimeout(this.reconnectTimeoutId);
|
clearTimeout(this.reconnectTimeoutId);
|
||||||
this.reconnectTimeoutId = setTimeout(() => {
|
this.reconnectTimeoutId = setTimeout(() => {
|
||||||
this.reconnectTimeoutId = undefined;
|
this.reconnectTimeoutId = undefined;
|
||||||
|
|
@ -149,7 +165,10 @@ export class WebSocketManager {
|
||||||
}
|
}
|
||||||
|
|
||||||
private initializeWebSocket(): void {
|
private initializeWebSocket(): void {
|
||||||
if (this.isStopped) return;
|
if (this.isStopped) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
this.openWebSocket();
|
this.openWebSocket();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -183,7 +202,10 @@ export class WebSocketManager {
|
||||||
};
|
};
|
||||||
|
|
||||||
socket.onopen = (): void => {
|
socket.onopen = (): void => {
|
||||||
if (this.webSocket !== socket || this.isStopped) return;
|
if (this.webSocket !== socket || this.isStopped) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const handshake: WebSocketClientMessage = {
|
const handshake: WebSocketClientMessage = {
|
||||||
type: "handshake",
|
type: "handshake",
|
||||||
|
|
@ -194,8 +216,9 @@ export class WebSocketManager {
|
||||||
this.logger.info("WebSocket connection opened");
|
this.logger.info("WebSocket connection opened");
|
||||||
this.onWebSocketStatusChanged.trigger(true);
|
this.onWebSocketStatusChanged.trigger(true);
|
||||||
// An observer may stop or replace the connection during delivery.
|
// An observer may stop or replace the connection during delivery.
|
||||||
if (this.webSocket !== socket)
|
if (this.webSocket !== socket) {
|
||||||
this.onWebSocketStatusChanged.trigger(false);
|
this.onWebSocketStatusChanged.trigger(false);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.warn(
|
this.logger.warn(
|
||||||
`WebSocket handshake failed: ${String(error)}`
|
`WebSocket handshake failed: ${String(error)}`
|
||||||
|
|
@ -205,15 +228,22 @@ export class WebSocketManager {
|
||||||
};
|
};
|
||||||
|
|
||||||
socket.onmessage = (event): void => {
|
socket.onmessage = (event): void => {
|
||||||
if (this.webSocket !== socket || this.isStopped) return;
|
if (this.webSocket !== socket || this.isStopped) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion
|
||||||
const message = JSON.parse(
|
const message = JSON.parse(
|
||||||
event.data
|
event.data
|
||||||
) as WebSocketServerMessage;
|
) as WebSocketServerMessage;
|
||||||
|
|
||||||
if (["vaultChanged", "cursorPositions"].includes(message.type))
|
if (
|
||||||
|
["vaultChanged", "cursorPositions"].includes(message.type)
|
||||||
|
) {
|
||||||
this.expectResponse(socket);
|
this.expectResponse(socket);
|
||||||
|
}
|
||||||
|
|
||||||
// Track the message handling promise
|
// Track the message handling promise
|
||||||
const messageHandlingPromise = this.handleWebSocketMessage(
|
const messageHandlingPromise = this.handleWebSocketMessage(
|
||||||
message
|
message
|
||||||
|
|
|
||||||
41
frontend/sync-client/src/sync-client.test.ts
Normal file
41
frontend/sync-client/src/sync-client.test.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
/* eslint-disable @typescript-eslint/no-unsafe-type-assertion -- No filesystem operation is reached before initialization. */
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "node:test";
|
||||||
|
import { SyncClient } from "./sync-client";
|
||||||
|
import type { FileSystemOperations } from "./file-operations/filesystem-operations";
|
||||||
|
import { awaitAll } from "./utils/await-all";
|
||||||
|
|
||||||
|
test("destroy during startup metadata work cannot restart the client", async () => {
|
||||||
|
const saving = Promise.withResolvers<undefined>();
|
||||||
|
const release = Promise.withResolvers<undefined>();
|
||||||
|
const client = await SyncClient.create({
|
||||||
|
fs: {} as FileSystemOperations,
|
||||||
|
persistence: {
|
||||||
|
load: async () => ({
|
||||||
|
settings: {
|
||||||
|
isSyncEnabled: true,
|
||||||
|
enableTelemetry: false,
|
||||||
|
remoteUri: "http://test"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
save: async () => {
|
||||||
|
saving.resolve(undefined);
|
||||||
|
await release.promise;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
fetch: async () => {
|
||||||
|
assert.fail("a destroyed client must not start network requests");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const notification = client.syncLocallyCreatedFile("note.md");
|
||||||
|
await saving.promise;
|
||||||
|
const starting = client.start();
|
||||||
|
// Let start acquire the lifecycle lock and begin draining notification saves.
|
||||||
|
await Promise.resolve();
|
||||||
|
const destroying = client.destroy();
|
||||||
|
release.resolve(undefined);
|
||||||
|
await awaitAll([notification, starting, destroying]);
|
||||||
|
|
||||||
|
await assert.rejects(client.start(), /destroyed/u);
|
||||||
|
});
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
import type { NetworkConnectionStatus } from "./types/network-connection-status";
|
||||||
|
import { ClientMetadataStore } from "./persistence/client-metadata-store";
|
||||||
import * as Sentry from "@sentry/browser";
|
import * as Sentry from "@sentry/browser";
|
||||||
import { setUpTelemetry } from "./utils/set-up-telemetry";
|
import { setUpTelemetry } from "./utils/set-up-telemetry";
|
||||||
import type { MetadataPersistenceProvider } from "./persistence/metadata-persistence-provider";
|
import type { MetadataPersistenceProvider } from "./persistence/metadata-persistence-provider";
|
||||||
|
|
@ -28,7 +30,7 @@ import { isInternalPath } from "./utils/portable-path";
|
||||||
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
||||||
import { Lock } from "./utils/data-structures/locks";
|
import { Lock } from "./utils/data-structures/locks";
|
||||||
import {
|
import {
|
||||||
unappliedChanges,
|
getUnappliedChanges,
|
||||||
type LocalChange
|
type LocalChange
|
||||||
} from "./sync-operations/local-changes";
|
} from "./sync-operations/local-changes";
|
||||||
|
|
||||||
|
|
@ -40,17 +42,22 @@ export type StoredClient = Partial<{
|
||||||
historyCheckpoint: { vaultKey: string; checkpoint?: string };
|
historyCheckpoint: { vaultKey: string; checkpoint?: string };
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
const vaultKey = (settings: SyncSettings): string =>
|
const getVaultKey = (settings: SyncSettings): string =>
|
||||||
JSON.stringify([
|
JSON.stringify([
|
||||||
settings.remoteUri.replace(/\/$/u, ""),
|
settings.remoteUri.replace(/\/$/u, ""),
|
||||||
settings.vaultName
|
settings.vaultName
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
enum ClientPhase {
|
||||||
|
Created = "created",
|
||||||
|
Started = "started",
|
||||||
|
Destroyed = "destroyed"
|
||||||
|
}
|
||||||
|
|
||||||
export class SyncClient {
|
export class SyncClient {
|
||||||
private readonly lifecycle = new Lock();
|
private readonly lifecycle = new Lock();
|
||||||
private destroying?: Promise<void>;
|
private destroying?: Promise<void>;
|
||||||
private started = false;
|
private phase = ClientPhase.Created;
|
||||||
private destroyed = false;
|
|
||||||
private unloadTelemetry?: () => void;
|
private unloadTelemetry?: () => void;
|
||||||
private readonly unsubscribeLog: () => void;
|
private readonly unsubscribeLog: () => void;
|
||||||
private constructor(
|
private constructor(
|
||||||
|
|
@ -74,18 +81,47 @@ export class SyncClient {
|
||||||
});
|
});
|
||||||
syncer.onReadyForEvents.add(() => {
|
syncer.onReadyForEvents.add(() => {
|
||||||
if (
|
if (
|
||||||
this.started &&
|
this.phase === ClientPhase.Started &&
|
||||||
!this.destroyed &&
|
|
||||||
settings.getSettings().isSyncEnabled
|
settings.getSettings().isSyncEnabled
|
||||||
)
|
) {
|
||||||
webSocketManager.start();
|
webSocketManager.start();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
this.unsubscribeLog = logger.onLogEmitted.add((log) => {
|
this.unsubscribeLog = logger.onLogEmitted.add((log) => {
|
||||||
if (log.level === LogLevel.ERROR && Sentry.isInitialized())
|
if (log.level === LogLevel.ERROR && Sentry.isInitialized()) {
|
||||||
Sentry.captureMessage(log.message);
|
Sentry.captureMessage(log.message);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public get documentCount(): number {
|
||||||
|
return this.database.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get isWebSocketConnected(): boolean {
|
||||||
|
return this.webSocketManager.isWebSocketConnected;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get onSyncHistoryUpdated(): SyncHistory["onHistoryUpdated"] {
|
||||||
|
return this.history.onHistoryUpdated;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get onSettingsChanged(): Settings["onSettingsChanged"] {
|
||||||
|
return this.settings.onSettingsChanged;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get onRemainingOperationsCountChanged(): Syncer["onRemainingOperationsCountChanged"] {
|
||||||
|
return this.syncer.onRemainingOperationsCountChanged;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get onWebSocketStatusChanged(): WebSocketManager["onWebSocketStatusChanged"] {
|
||||||
|
return this.webSocketManager.onWebSocketStatusChanged;
|
||||||
|
}
|
||||||
|
|
||||||
|
public get onRemoteCursorsUpdated(): CursorTracker["onRemoteCursorsUpdated"] {
|
||||||
|
return this.cursorTracker.onRemoteCursorsUpdated;
|
||||||
|
}
|
||||||
|
|
||||||
public static async create({
|
public static async create({
|
||||||
fs,
|
fs,
|
||||||
persistence,
|
persistence,
|
||||||
|
|
@ -98,65 +134,43 @@ export class SyncClient {
|
||||||
webSocket?: typeof globalThis.WebSocket;
|
webSocket?: typeof globalThis.WebSocket;
|
||||||
}): Promise<SyncClient> {
|
}): Promise<SyncClient> {
|
||||||
const logger = new Logger();
|
const logger = new Logger();
|
||||||
let stored = (await persistence.load()) ?? {};
|
const metadata = await ClientMetadataStore.load(persistence);
|
||||||
const persistenceLock = new Lock();
|
const { stored } = metadata;
|
||||||
let reloadBeforeSave = false;
|
|
||||||
// Settings and engine state share a store. Serialize complete metadata
|
|
||||||
// replacements so one cannot overwrite a newer save from the other.
|
|
||||||
const save = async (update: StoredClient): Promise<void> =>
|
const save = async (update: StoredClient): Promise<void> =>
|
||||||
persistenceLock.withLock(async () => {
|
metadata.save(update);
|
||||||
if (reloadBeforeSave) {
|
|
||||||
stored = (await persistence.load()) ?? {};
|
|
||||||
reloadBeforeSave = false;
|
|
||||||
}
|
|
||||||
const next = { ...stored, ...structuredClone(update) };
|
|
||||||
try {
|
|
||||||
await persistence.save(next);
|
|
||||||
stored = next;
|
|
||||||
} catch (error) {
|
|
||||||
// A failed save may have committed. Do not let a queued
|
|
||||||
// settings/notification save roll back that engine state.
|
|
||||||
reloadBeforeSave = true;
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
const settings = new Settings(logger, stored.settings, async (data) =>
|
const settings = new Settings(logger, stored.settings, async (data) =>
|
||||||
save({ settings: data })
|
save({ settings: data })
|
||||||
);
|
);
|
||||||
const database = new Database(
|
const database = new Database(
|
||||||
stored.database,
|
stored.database,
|
||||||
async (data) => save({ database: data }),
|
async (data) => save({ database: data }),
|
||||||
vaultKey(settings.getSettings()),
|
getVaultKey(settings.getSettings()),
|
||||||
async () =>
|
async () => metadata.reloadDatabase()
|
||||||
persistenceLock.withLock(async () => {
|
|
||||||
stored = (await persistence.load()) ?? {};
|
|
||||||
reloadBeforeSave = false;
|
|
||||||
return stored.database;
|
|
||||||
})
|
|
||||||
);
|
);
|
||||||
const localChanges = unappliedChanges(
|
const localChanges = getUnappliedChanges(
|
||||||
database.state,
|
database.state,
|
||||||
stored.localChanges ?? []
|
stored.localChanges ?? []
|
||||||
);
|
);
|
||||||
if (
|
if (
|
||||||
localChanges.length &&
|
localChanges.length &&
|
||||||
stored.localChangesVaultKey !== vaultKey(settings.getSettings())
|
stored.localChangesVaultKey !== getVaultKey(settings.getSettings())
|
||||||
) {
|
) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Offline notifications belong to another vault; use a separate state store"
|
"Offline notifications belong to another vault; use a separate state store"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const deviceId = createClientId();
|
const deviceId = createClientId();
|
||||||
const service = new SyncService(deviceId, settings, fetch, {
|
const service = new SyncService(deviceId, settings, fetch, {
|
||||||
get: () =>
|
get: (): string | undefined =>
|
||||||
stored.historyCheckpoint?.vaultKey ===
|
metadata.stored.historyCheckpoint?.vaultKey ===
|
||||||
vaultKey(settings.getSettings())
|
getVaultKey(settings.getSettings())
|
||||||
? stored.historyCheckpoint.checkpoint
|
? metadata.stored.historyCheckpoint.checkpoint
|
||||||
: undefined,
|
: undefined,
|
||||||
save: async (checkpoint) =>
|
save: async (checkpoint): Promise<void> =>
|
||||||
save({
|
save({
|
||||||
historyCheckpoint: {
|
historyCheckpoint: {
|
||||||
vaultKey: vaultKey(settings.getSettings()),
|
vaultKey: getVaultKey(settings.getSettings()),
|
||||||
checkpoint
|
checkpoint
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -168,11 +182,15 @@ export class SyncClient {
|
||||||
database,
|
database,
|
||||||
serverConfig,
|
serverConfig,
|
||||||
(paths) => {
|
(paths) => {
|
||||||
for (const path of paths) notifier.trigger(path);
|
for (const path of paths) {
|
||||||
|
notifier.trigger(path);
|
||||||
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
entries: () => localChanges,
|
entries: (): LocalChange[] => localChanges,
|
||||||
flush: async () => syncer.flushLocalChanges()
|
// Invoked after all components have been constructed.
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-use-before-define
|
||||||
|
flush: async (): Promise<void> => syncer.flushLocalChanges()
|
||||||
},
|
},
|
||||||
(path, size) =>
|
(path, size) =>
|
||||||
settings.isIgnored(path) || settings.isOversized(size)
|
settings.isIgnored(path) || settings.isOversized(size)
|
||||||
|
|
@ -202,7 +220,9 @@ export class SyncClient {
|
||||||
save: async (entries) =>
|
save: async (entries) =>
|
||||||
save({
|
save({
|
||||||
localChanges: entries,
|
localChanges: entries,
|
||||||
localChangesVaultKey: vaultKey(settings.getSettings())
|
localChangesVaultKey: getVaultKey(
|
||||||
|
settings.getSettings()
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
@ -223,39 +243,7 @@ export class SyncClient {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private check(): void {
|
public getHistoryEntries(): SyncHistory["entries"] {
|
||||||
if (this.destroyed) throw new Error("SyncClient has been destroyed");
|
|
||||||
}
|
|
||||||
|
|
||||||
public get documentCount(): number {
|
|
||||||
return this.database.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
public get isWebSocketConnected(): boolean {
|
|
||||||
return this.webSocketManager.isWebSocketConnected;
|
|
||||||
}
|
|
||||||
|
|
||||||
public get onSyncHistoryUpdated() {
|
|
||||||
return this.history.onHistoryUpdated;
|
|
||||||
}
|
|
||||||
|
|
||||||
public get onSettingsChanged() {
|
|
||||||
return this.settings.onSettingsChanged;
|
|
||||||
}
|
|
||||||
|
|
||||||
public get onRemainingOperationsCountChanged() {
|
|
||||||
return this.syncer.onRemainingOperationsCountChanged;
|
|
||||||
}
|
|
||||||
|
|
||||||
public get onWebSocketStatusChanged() {
|
|
||||||
return this.webSocketManager.onWebSocketStatusChanged;
|
|
||||||
}
|
|
||||||
|
|
||||||
public get onRemoteCursorsUpdated() {
|
|
||||||
return this.cursorTracker.onRemoteCursorsUpdated;
|
|
||||||
}
|
|
||||||
|
|
||||||
public getHistoryEntries() {
|
|
||||||
return this.history.entries;
|
return this.history.entries;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -265,43 +253,31 @@ export class SyncClient {
|
||||||
|
|
||||||
public async start(): Promise<void> {
|
public async start(): Promise<void> {
|
||||||
await this.lifecycle.withLock(async () => {
|
await this.lifecycle.withLock(async () => {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
if (this.started)
|
if (this.phase === ClientPhase.Started) {
|
||||||
throw new Error("SyncClient has already been started");
|
throw new Error("SyncClient has already been started");
|
||||||
|
}
|
||||||
|
|
||||||
await this.syncer.reloadLocalState();
|
await this.syncer.reloadLocalState();
|
||||||
await this.database.bindVault(
|
await this.database.bindVault(
|
||||||
vaultKey(this.settings.getSettings())
|
getVaultKey(this.settings.getSettings())
|
||||||
);
|
);
|
||||||
this.started = true;
|
// destroy() can interrupt the metadata work above before acquiring
|
||||||
|
// the lifecycle lock. Its terminal state must not be overwritten.
|
||||||
|
if (this.phase === ClientPhase.Destroyed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.phase = ClientPhase.Started;
|
||||||
this.configureTelemetry();
|
this.configureTelemetry();
|
||||||
await this.resume();
|
await this.resume();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async resume(): Promise<void> {
|
|
||||||
if (
|
|
||||||
!this.started ||
|
|
||||||
this.destroyed ||
|
|
||||||
!this.settings.getSettings().isSyncEnabled
|
|
||||||
)
|
|
||||||
return;
|
|
||||||
await this.database.bindVault(vaultKey(this.settings.getSettings()));
|
|
||||||
this.service.resume();
|
|
||||||
this.syncer.start();
|
|
||||||
await this.syncer.waitUntilFinished();
|
|
||||||
}
|
|
||||||
|
|
||||||
private async pause(): Promise<void> {
|
|
||||||
const stopped = this.syncer.stop();
|
|
||||||
this.service.pause();
|
|
||||||
await this.webSocketManager.stop();
|
|
||||||
await stopped;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Restart transports; never discard pending requests. */
|
/** Restart transports; never discard pending requests. */
|
||||||
public async reset(): Promise<void> {
|
public async reset(): Promise<void> {
|
||||||
await this.lifecycle.withLock(async () => {
|
await this.lifecycle.withLock(async () => {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
await this.pause();
|
await this.pause();
|
||||||
this.serverConfig.reset();
|
this.serverConfig.reset();
|
||||||
this.cursorTracker.reset();
|
this.cursorTracker.reset();
|
||||||
|
|
@ -313,28 +289,32 @@ export class SyncClient {
|
||||||
public async setSettings(change: Partial<SyncSettings>): Promise<void> {
|
public async setSettings(change: Partial<SyncSettings>): Promise<void> {
|
||||||
const update = structuredClone(change);
|
const update = structuredClone(change);
|
||||||
const notification = await this.lifecycle.withLock(async () => {
|
const notification = await this.lifecycle.withLock(async () => {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
const old = this.settings.getSettings();
|
const old = this.settings.getSettings();
|
||||||
const next = { ...old, ...update };
|
const next = { ...old, ...update };
|
||||||
if (
|
if (
|
||||||
vaultKey(next) !== vaultKey(old) &&
|
getVaultKey(next) !== getVaultKey(old) &&
|
||||||
(this.started ||
|
(this.phase === ClientPhase.Started ||
|
||||||
this.database.state.lastSeenUpdateId !== undefined ||
|
this.database.state.lastSeenUpdateId !== undefined ||
|
||||||
this.database.state.pending ||
|
this.database.state.pending ||
|
||||||
this.syncer.hasLocalChanges ||
|
this.syncer.hasLocalChanges ||
|
||||||
Object.keys(this.database.state.local).length)
|
Object.keys(this.database.state.local).length)
|
||||||
)
|
) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"Changing vaults requires a separate state store and client; existing sync state must not be reset"
|
"Changing vaults requires a separate state store and client; existing sync state must not be reset"
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (this.started) await this.pause();
|
if (this.phase === ClientPhase.Started) {
|
||||||
|
await this.pause();
|
||||||
|
}
|
||||||
|
|
||||||
await this.database.recoverPersistence();
|
await this.database.recoverPersistence();
|
||||||
await this.settings.setSettings(update, false);
|
await this.settings.setSettings(update, false);
|
||||||
await this.syncer.retryRejectedRequests();
|
await this.syncer.retryRejectedRequests();
|
||||||
this.contentCache.resize(next.diffCacheSizeMB * 1024 * 1024);
|
this.contentCache.resize(next.diffCacheSizeMB * 1024 * 1024);
|
||||||
await this.database.bindVault(vaultKey(next));
|
await this.database.bindVault(getVaultKey(next));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.serverConfig.reset();
|
this.serverConfig.reset();
|
||||||
this.contentCache.resize(
|
this.contentCache.resize(
|
||||||
|
|
@ -348,6 +328,7 @@ export class SyncClient {
|
||||||
});
|
});
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.serverConfig.reset();
|
this.serverConfig.reset();
|
||||||
this.configureTelemetry();
|
this.configureTelemetry();
|
||||||
await this.resume();
|
await this.resume();
|
||||||
|
|
@ -368,7 +349,7 @@ export class SyncClient {
|
||||||
await this.setSettings({ ...DEFAULT_SETTINGS, ...stored?.settings });
|
await this.setSettings({ ...DEFAULT_SETTINGS, ...stored?.settings });
|
||||||
}
|
}
|
||||||
|
|
||||||
public async checkConnection() {
|
public async checkConnection(): Promise<NetworkConnectionStatus> {
|
||||||
const result = await this.serverConfig.checkConnection();
|
const result = await this.serverConfig.checkConnection();
|
||||||
return {
|
return {
|
||||||
isSuccessful: result.isSuccessful,
|
isSuccessful: result.isSuccessful,
|
||||||
|
|
@ -379,15 +360,21 @@ export class SyncClient {
|
||||||
|
|
||||||
/** Notifications enqueue work. Use waitUntilFinished() to await convergence. */
|
/** Notifications enqueue work. Use waitUntilFinished() to await convergence. */
|
||||||
public async syncLocallyCreatedFile(path: RelativePath): Promise<void> {
|
public async syncLocallyCreatedFile(path: RelativePath): Promise<void> {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
if (isInternalPath(path)) return;
|
if (isInternalPath(path)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.fileChanges.trigger(path);
|
this.fileChanges.trigger(path);
|
||||||
await this.syncer.syncLocallyCreatedFile(path);
|
await this.syncer.syncLocallyCreatedFile(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async syncLocallyDeletedFile(path: RelativePath): Promise<void> {
|
public async syncLocallyDeletedFile(path: RelativePath): Promise<void> {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
if (isInternalPath(path)) return;
|
if (isInternalPath(path)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.fileChanges.trigger(path);
|
this.fileChanges.trigger(path);
|
||||||
await this.syncer.syncLocallyDeletedFile(path);
|
await this.syncer.syncLocallyDeletedFile(path);
|
||||||
}
|
}
|
||||||
|
|
@ -396,12 +383,16 @@ export class SyncClient {
|
||||||
oldPath?: RelativePath;
|
oldPath?: RelativePath;
|
||||||
relativePath: RelativePath;
|
relativePath: RelativePath;
|
||||||
}): Promise<void> {
|
}): Promise<void> {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
if (
|
if (
|
||||||
isInternalPath(change.relativePath) ||
|
isInternalPath(change.relativePath) ||
|
||||||
(change.oldPath && isInternalPath(change.oldPath))
|
(change.oldPath !== undefined &&
|
||||||
)
|
change.oldPath !== "" &&
|
||||||
|
isInternalPath(change.oldPath))
|
||||||
|
) {
|
||||||
return;
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
this.fileChanges.trigger(change.relativePath);
|
this.fileChanges.trigger(change.relativePath);
|
||||||
await this.syncer.syncLocallyUpdatedFile(change);
|
await this.syncer.syncLocallyUpdatedFile(change);
|
||||||
}
|
}
|
||||||
|
|
@ -409,39 +400,32 @@ export class SyncClient {
|
||||||
public async updateLocalCursors(
|
public async updateLocalCursors(
|
||||||
cursors: Record<RelativePath, CursorSpan[]>
|
cursors: Record<RelativePath, CursorSpan[]>
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
await this.cursorTracker.sendLocalCursorsToServer(cursors);
|
await this.cursorTracker.sendLocalCursorsToServer(cursors);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getDocumentSyncingStatus(path: RelativePath): DocumentSyncStatus {
|
public getDocumentSyncingStatus(path: RelativePath): DocumentSyncStatus {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
if (!this.settings.getSettings().isSyncEnabled)
|
if (!this.settings.getSettings().isSyncEnabled) {
|
||||||
return DocumentSyncStatus.SYNCING_IS_DISABLED;
|
return DocumentSyncStatus.SYNCING_IS_DISABLED;
|
||||||
|
}
|
||||||
|
|
||||||
return this.syncer.isDocumentUpToDate(path)
|
return this.syncer.isDocumentUpToDate(path)
|
||||||
? DocumentSyncStatus.UP_TO_DATE
|
? DocumentSyncStatus.UP_TO_DATE
|
||||||
: DocumentSyncStatus.SYNCING;
|
: DocumentSyncStatus.SYNCING;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async waitUntilFinished(): Promise<void> {
|
public async waitUntilFinished(): Promise<void> {
|
||||||
this.check();
|
this.assertNotDestroyed();
|
||||||
await this.syncer.waitUntilFinished();
|
await this.syncer.waitUntilFinished();
|
||||||
}
|
}
|
||||||
|
|
||||||
private configureTelemetry(): void {
|
public async destroy(): Promise<void> {
|
||||||
if (
|
if (this.destroying) {
|
||||||
this.settings.getSettings().enableTelemetry &&
|
return this.destroying;
|
||||||
typeof window !== "undefined"
|
|
||||||
) {
|
|
||||||
this.unloadTelemetry ??= setUpTelemetry();
|
|
||||||
} else {
|
|
||||||
this.unloadTelemetry?.();
|
|
||||||
this.unloadTelemetry = undefined;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public async destroy(): Promise<void> {
|
this.phase = ClientPhase.Destroyed;
|
||||||
if (this.destroying) return this.destroying;
|
|
||||||
this.destroyed = true;
|
|
||||||
// Interrupt a start/reset awaiting network progress before waiting for
|
// Interrupt a start/reset awaiting network progress before waiting for
|
||||||
// its lifecycle operation to finish.
|
// its lifecycle operation to finish.
|
||||||
this.service.pause();
|
this.service.pause();
|
||||||
|
|
@ -459,4 +443,43 @@ export class SyncClient {
|
||||||
});
|
});
|
||||||
return this.destroying;
|
return this.destroying;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private assertNotDestroyed(): void {
|
||||||
|
if (this.phase === ClientPhase.Destroyed) {
|
||||||
|
throw new Error("SyncClient has been destroyed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resume(): Promise<void> {
|
||||||
|
if (
|
||||||
|
this.phase !== ClientPhase.Started ||
|
||||||
|
!this.settings.getSettings().isSyncEnabled
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.database.bindVault(getVaultKey(this.settings.getSettings()));
|
||||||
|
this.service.resume();
|
||||||
|
this.syncer.start();
|
||||||
|
await this.syncer.waitUntilFinished();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async pause(): Promise<void> {
|
||||||
|
const stopped = this.syncer.stop();
|
||||||
|
this.service.pause();
|
||||||
|
await this.webSocketManager.stop();
|
||||||
|
await stopped;
|
||||||
|
}
|
||||||
|
|
||||||
|
private configureTelemetry(): void {
|
||||||
|
if (
|
||||||
|
this.settings.getSettings().enableTelemetry &&
|
||||||
|
typeof window !== "undefined"
|
||||||
|
) {
|
||||||
|
this.unloadTelemetry ??= setUpTelemetry();
|
||||||
|
} else {
|
||||||
|
this.unloadTelemetry?.();
|
||||||
|
this.unloadTelemetry = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
245
frontend/sync-client/src/sync-operations/content-sync.ts
Normal file
245
frontend/sync-client/src/sync-operations/content-sync.ts
Normal file
|
|
@ -0,0 +1,245 @@
|
||||||
|
import { base64ToBytes, bytesToBase64 } from "byte-base64";
|
||||||
|
import { diff, undiff } from "reconcile-text";
|
||||||
|
import type {
|
||||||
|
Database,
|
||||||
|
StoredDatabase,
|
||||||
|
PendingRequest,
|
||||||
|
ContentHead,
|
||||||
|
DocumentId,
|
||||||
|
RelativePath,
|
||||||
|
VaultUpdateId
|
||||||
|
} from "../persistence/database";
|
||||||
|
import { hashSnapshot, type HashedSnapshot } from "../snapshot";
|
||||||
|
import type {
|
||||||
|
FileOperations,
|
||||||
|
FileWrite
|
||||||
|
} from "../file-operations/file-operations";
|
||||||
|
import type { Settings } from "../persistence/settings";
|
||||||
|
import type { SyncService } from "../services/sync-service";
|
||||||
|
import type { ServerConfig } from "../services/server-config";
|
||||||
|
import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent";
|
||||||
|
import type { PushContent } from "../services/types/PushContent";
|
||||||
|
import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache";
|
||||||
|
import { isBinary } from "../utils/is-binary";
|
||||||
|
import { isFileTypeMergeable } from "../utils/is-file-type-mergeable";
|
||||||
|
import { decodeText } from "../utils/decode-text";
|
||||||
|
import {
|
||||||
|
LocalChangesDuringReconciliation,
|
||||||
|
PermanentSyncError
|
||||||
|
} from "../errors/errors";
|
||||||
|
import { mergeContent } from "./content";
|
||||||
|
|
||||||
|
/** Store only the remote version metadata, never response content or cursors. */
|
||||||
|
export function selectContentHead(version: ContentHead): ContentHead {
|
||||||
|
return {
|
||||||
|
vaultUpdateId: version.vaultUpdateId,
|
||||||
|
contentSize: version.contentSize
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fetch and merge content, and encode uploads using the optional diff cache. */
|
||||||
|
export class ContentSync {
|
||||||
|
public constructor(
|
||||||
|
private readonly database: Database,
|
||||||
|
private readonly settings: Settings,
|
||||||
|
private readonly service: SyncService,
|
||||||
|
private readonly files: FileOperations,
|
||||||
|
private readonly serverConfig: ServerConfig,
|
||||||
|
private readonly contentCache: FixedSizeDocumentCache
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public async createUploadPayload(
|
||||||
|
path: RelativePath,
|
||||||
|
parentVersionId: VaultUpdateId | undefined,
|
||||||
|
snapshot: HashedSnapshot
|
||||||
|
): Promise<PushContent> {
|
||||||
|
const { content } = snapshot;
|
||||||
|
const parent =
|
||||||
|
parentVersionId === undefined
|
||||||
|
? undefined
|
||||||
|
: this.contentCache.get(parentVersionId);
|
||||||
|
if (
|
||||||
|
parent !== undefined &&
|
||||||
|
!isBinary(content) &&
|
||||||
|
!isBinary(parent) &&
|
||||||
|
isFileTypeMergeable(
|
||||||
|
path,
|
||||||
|
(await this.serverConfig.getConfig()).mergeableFileExtensions
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
const value = diff(decodeText(parent), decodeText(content));
|
||||||
|
// Match the server's work bound; large edit sets remain syncable.
|
||||||
|
if (value.length <= 10_000) {
|
||||||
|
return { type: "Diff", value };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { type: "Snapshot", value: bytesToBase64(content) };
|
||||||
|
}
|
||||||
|
|
||||||
|
public async fetchSnapshot(
|
||||||
|
head: Pick<
|
||||||
|
DocumentVersionWithoutContent,
|
||||||
|
"documentId" | "vaultUpdateId"
|
||||||
|
>,
|
||||||
|
path: RelativePath
|
||||||
|
): Promise<HashedSnapshot> {
|
||||||
|
const content = await this.service.getDocumentVersionContent(head);
|
||||||
|
await this.cacheContent(head.vaultUpdateId, path, content);
|
||||||
|
return hashSnapshot({ content });
|
||||||
|
}
|
||||||
|
|
||||||
|
public async planDownload(
|
||||||
|
next: StoredDatabase,
|
||||||
|
id: DocumentId,
|
||||||
|
path: RelativePath
|
||||||
|
): Promise<FileWrite | undefined> {
|
||||||
|
const document = (next.documents[id] ??= {});
|
||||||
|
const currentPath = this.database.state.local[id];
|
||||||
|
if (currentPath === undefined) {
|
||||||
|
delete document.observedHash;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
document.observedHash !== undefined &&
|
||||||
|
document.remote &&
|
||||||
|
document.remote.vaultUpdateId <= (document.base?.vaultUpdateId ?? 0)
|
||||||
|
) {
|
||||||
|
delete document.remote;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
next.excluded?.[id] !== undefined ||
|
||||||
|
this.settings.isIgnored(path) ||
|
||||||
|
(next.fileManifest.entries[id] === undefined && !document.remote) ||
|
||||||
|
(currentPath !== undefined &&
|
||||||
|
currentPath !== "" &&
|
||||||
|
document.base &&
|
||||||
|
document.observedHash !== undefined &&
|
||||||
|
!document.remote)
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const head =
|
||||||
|
document.remote ?? (await this.service.getDocumentMetadata(id));
|
||||||
|
// The transport is injectable; keep the runtime check for malformed responses.
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, @typescript-eslint/strict-boolean-expressions
|
||||||
|
if (!head) {
|
||||||
|
throw new PermanentSyncError(
|
||||||
|
`File manifest references missing content: ${id}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
document.remote = selectContentHead(head);
|
||||||
|
if (this.settings.isOversized(head.contentSize)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const local =
|
||||||
|
currentPath !== undefined &&
|
||||||
|
currentPath !== "" &&
|
||||||
|
(await this.files.fs.stat(currentPath))?.kind === "file"
|
||||||
|
? await this.files.readSnapshot(currentPath)
|
||||||
|
: undefined;
|
||||||
|
if (
|
||||||
|
local !== undefined &&
|
||||||
|
this.settings.isOversized(local.content.length)
|
||||||
|
) {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
document.base?.vaultUpdateId === null &&
|
||||||
|
local &&
|
||||||
|
local.hash !== document.base.hash
|
||||||
|
) {
|
||||||
|
throw new LocalChangesDuringReconciliation();
|
||||||
|
}
|
||||||
|
|
||||||
|
const base =
|
||||||
|
local && document.base
|
||||||
|
? document.base.vaultUpdateId === null
|
||||||
|
? local
|
||||||
|
: await this.fetchSnapshot(
|
||||||
|
{
|
||||||
|
documentId: id,
|
||||||
|
vaultUpdateId: document.base.vaultUpdateId
|
||||||
|
},
|
||||||
|
path
|
||||||
|
)
|
||||||
|
: undefined;
|
||||||
|
// Cache the new head after the older merge base so it remains the most
|
||||||
|
// recently used version and can back the follow-up local diff.
|
||||||
|
const remote = await this.fetchSnapshot(
|
||||||
|
{ documentId: id, vaultUpdateId: head.vaultUpdateId },
|
||||||
|
path
|
||||||
|
);
|
||||||
|
let result = remote;
|
||||||
|
if (local !== undefined) {
|
||||||
|
result = await mergeContent(
|
||||||
|
path,
|
||||||
|
base,
|
||||||
|
local,
|
||||||
|
remote,
|
||||||
|
(await this.serverConfig.getConfig()).mergeableFileExtensions
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
document.base = {
|
||||||
|
vaultUpdateId: head.vaultUpdateId,
|
||||||
|
hash: remote.hash
|
||||||
|
};
|
||||||
|
delete document.remote;
|
||||||
|
document.observedHash = result.hash;
|
||||||
|
return local?.hash === result.hash
|
||||||
|
? undefined
|
||||||
|
: { expected: local, replacement: result };
|
||||||
|
}
|
||||||
|
|
||||||
|
public async cacheAcceptedUpload(
|
||||||
|
pending: Extract<PendingRequest, { type: "content" }>,
|
||||||
|
version: VaultUpdateId
|
||||||
|
): Promise<void> {
|
||||||
|
const { content, parentVersionId } = pending.request;
|
||||||
|
let bytes: Uint8Array | undefined = undefined;
|
||||||
|
if (content.type === "Snapshot") {
|
||||||
|
bytes = base64ToBytes(content.value);
|
||||||
|
} else {
|
||||||
|
const parent =
|
||||||
|
parentVersionId === null
|
||||||
|
? undefined
|
||||||
|
: this.contentCache.get(parentVersionId);
|
||||||
|
// A restart may have evicted the parent. Future uploads can use snapshots.
|
||||||
|
if (!parent) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const text = decodeText(parent);
|
||||||
|
bytes = new TextEncoder().encode(undiff(text, content.value));
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.cacheContent(
|
||||||
|
version,
|
||||||
|
this.database.state.local[pending.documentId],
|
||||||
|
bytes
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async cacheContent(
|
||||||
|
updateId: VaultUpdateId,
|
||||||
|
path: RelativePath | undefined,
|
||||||
|
content: Uint8Array
|
||||||
|
): Promise<void> {
|
||||||
|
if (
|
||||||
|
path !== undefined &&
|
||||||
|
!isBinary(content) &&
|
||||||
|
isFileTypeMergeable(
|
||||||
|
path,
|
||||||
|
(await this.serverConfig.getConfig()).mergeableFileExtensions
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
this.contentCache.put(updateId, content);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { describe, it } from "node:test";
|
import { describe, it } from "node:test";
|
||||||
import { hashSnapshot } from "../snapshot";
|
import { hashSnapshot, type HashedSnapshot } from "../snapshot";
|
||||||
import { mergeContent } from "./content";
|
import { mergeContent } from "./content";
|
||||||
|
|
||||||
const extensions = ["md"];
|
const extensions = ["md"];
|
||||||
const snapshot = async (text: string) =>
|
const createSnapshot = async (text: string): Promise<HashedSnapshot> =>
|
||||||
hashSnapshot({ content: new TextEncoder().encode(text) });
|
hashSnapshot({ content: new TextEncoder().encode(text) });
|
||||||
|
|
||||||
describe("content reconciliation without file timestamps", () => {
|
describe("content reconciliation without file timestamps", () => {
|
||||||
|
|
@ -21,8 +21,8 @@ describe("content reconciliation without file timestamps", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps local binary edits when the server content is unchanged", async () => {
|
it("keeps local binary edits when the server content is unchanged", async () => {
|
||||||
const base = await snapshot("\0base");
|
const base = await createSnapshot("\0base");
|
||||||
const local = await snapshot("\0local");
|
const local = await createSnapshot("\0local");
|
||||||
const merged = await mergeContent(
|
const merged = await mergeContent(
|
||||||
"file.bin",
|
"file.bin",
|
||||||
base,
|
base,
|
||||||
|
|
@ -34,8 +34,8 @@ describe("content reconciliation without file timestamps", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it("incorporates server binary edits when local content is unchanged", async () => {
|
it("incorporates server binary edits when local content is unchanged", async () => {
|
||||||
const base = await snapshot("\0base");
|
const base = await createSnapshot("\0base");
|
||||||
const remote = await snapshot("\0remote");
|
const remote = await createSnapshot("\0remote");
|
||||||
const merged = await mergeContent(
|
const merged = await mergeContent(
|
||||||
"file.bin",
|
"file.bin",
|
||||||
base,
|
base,
|
||||||
|
|
@ -51,10 +51,10 @@ describe("content reconciliation without file timestamps", () => {
|
||||||
["file.bin", "\0"],
|
["file.bin", "\0"],
|
||||||
["file.md", "\0"],
|
["file.md", "\0"],
|
||||||
["file.json", ""]
|
["file.json", ""]
|
||||||
]) {
|
] as const) {
|
||||||
const base = await snapshot(`${prefix}base`);
|
const base = await createSnapshot(`${prefix}base`);
|
||||||
const local = await snapshot(`${prefix}local`);
|
const local = await createSnapshot(`${prefix}local`);
|
||||||
const remote = await snapshot(`${prefix}remote`);
|
const remote = await createSnapshot(`${prefix}remote`);
|
||||||
const merged = await mergeContent(
|
const merged = await mergeContent(
|
||||||
path,
|
path,
|
||||||
base,
|
base,
|
||||||
|
|
@ -67,8 +67,8 @@ describe("content reconciliation without file timestamps", () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps the server for differing binary content without a common base", async () => {
|
it("keeps the server for differing binary content without a common base", async () => {
|
||||||
const local = await snapshot("\0local");
|
const local = await createSnapshot("\0local");
|
||||||
const remote = await snapshot("\0remote");
|
const remote = await createSnapshot("\0remote");
|
||||||
const merged = await mergeContent(
|
const merged = await mergeContent(
|
||||||
"file.bin",
|
"file.bin",
|
||||||
undefined,
|
undefined,
|
||||||
|
|
@ -82,9 +82,13 @@ describe("content reconciliation without file timestamps", () => {
|
||||||
it("still merges independent text edits from both sides", async () => {
|
it("still merges independent text edits from both sides", async () => {
|
||||||
const merged = await mergeContent(
|
const merged = await mergeContent(
|
||||||
"note.md",
|
"note.md",
|
||||||
await snapshot("First paragraph.\n\nLast paragraph.\n"),
|
await createSnapshot("First paragraph.\n\nLast paragraph.\n"),
|
||||||
await snapshot("Updated first paragraph.\n\nLast paragraph.\n"),
|
await createSnapshot(
|
||||||
await snapshot("First paragraph.\n\nUpdated last paragraph.\n"),
|
"Updated first paragraph.\n\nLast paragraph.\n"
|
||||||
|
),
|
||||||
|
await createSnapshot(
|
||||||
|
"First paragraph.\n\nUpdated last paragraph.\n"
|
||||||
|
),
|
||||||
extensions
|
extensions
|
||||||
);
|
);
|
||||||
assert.equal(
|
assert.equal(
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue