simplify 2

This commit is contained in:
Andras Schmelczer 2026-09-22 08:00:25 +01:00
commit 82929d7479
30 changed files with 503 additions and 1126 deletions

View file

@ -76,8 +76,9 @@ old base are preserved as separate local documents, with conflict names if neede
Interrupted recovery saves are replayable. Backups must be restored with the server Interrupted recovery saves are replayable. Backups must be restored with the server
stopped. stopped.
WebSocket messages wake HTTP replay; missed notifications are recovered by replay WebSocket `vaultChanged` hints wake HTTP replay. The server periodically compares
and transport liveness checks. Authentication and permanent errors remain visible; its history checkpoint to detect missed broadcasts; transport liveness checks
recover disconnected clients. Authentication and permanent errors remain visible;
transient errors retry according to client settings. Observer callback failures transient errors retry according to client settings. Observer callback failures
are isolated from the background sync loop. are isolated from the background sync loop.

View file

@ -62,7 +62,6 @@ vaultlink \
| `--ignore-pattern <pattern>` | - | Glob pattern to ignore (repeatable) | | `--ignore-pattern <pattern>` | - | Glob pattern to ignore (repeatable) |
| `--websocket-retry-interval-ms <ms>` | `3500` | WebSocket reconnection interval | | `--websocket-retry-interval-ms <ms>` | `3500` | WebSocket reconnection interval |
| `--log-level <level>` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR | | `--log-level <level>` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
| `--line-endings <mode>` | `auto` | Line ending style: auto, lf, crlf |
| `-q, --quiet` | - | Suppress startup banner for non-interactive use | | `-q, --quiet` | - | Suppress startup banner for non-interactive use |
| `-h, --help` | - | Show help | | `-h, --help` | - | Show help |
| `-V, --version` | - | Show version | | `-V, --version` | - | Show version |
@ -96,12 +95,7 @@ vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \
--log-level DEBUG --quiet --log-level DEBUG --quiet
``` ```
Force LF line endings (useful for cross-platform vaults): File bytes, including line endings, are preserved on every platform.
```bash
vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \
--line-endings lf
```
## Docker Deployment ## Docker Deployment

View file

@ -2,9 +2,6 @@ import { Command, Option } from "commander";
import packageJson from "../package.json"; import packageJson from "../package.json";
import { LogLevel } from "sync-client"; import { LogLevel } from "sync-client";
export const LINE_ENDING_MODES = ["auto", "lf", "crlf"] as const;
export type LineEndingMode = (typeof LINE_ENDING_MODES)[number];
interface CliArgs { interface CliArgs {
remoteUri: string; remoteUri: string;
token: string; token: string;
@ -17,7 +14,6 @@ interface CliArgs {
health?: string; health?: string;
enableTelemetry?: boolean; enableTelemetry?: boolean;
quiet: boolean; quiet: boolean;
lineEndings: LineEndingMode;
} }
const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"]; const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"];
@ -131,15 +127,6 @@ export function parseArgs(argv: string[]): CliArgs {
"[OPTIONAL] Suppress startup banner for non-interactive use" "[OPTIONAL] Suppress startup banner for non-interactive use"
).env("VAULTLINK_QUIET") ).env("VAULTLINK_QUIET")
) )
.addOption(
new Option(
"--line-endings <mode>",
"[OPTIONAL] Line ending style: auto (platform default), lf, crlf"
)
.default("auto")
.choices([...LINE_ENDING_MODES])
.env("VAULTLINK_LINE_ENDINGS")
)
.addHelpText( .addHelpText(
"after", "after",
` `
@ -173,7 +160,6 @@ Environment variables:
const health = opts.health as string | undefined; const health = opts.health as string | undefined;
const enableTelemetry = opts.enableTelemetry as boolean | undefined; const enableTelemetry = opts.enableTelemetry as boolean | undefined;
const quiet = (opts.quiet as boolean | undefined) ?? false; const quiet = (opts.quiet as boolean | undefined) ?? false;
const lineEndingsStr = (opts.lineEndings as string | undefined) ?? "auto";
/* eslint-enable @typescript-eslint/no-unsafe-type-assertion */ /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */
const requiredLocalPath = requireOption(localPath, "localPath"); const requiredLocalPath = requireOption(localPath, "localPath");
@ -203,15 +189,6 @@ Environment variables:
} }
const logLevel = logLevelUpper; const logLevel = logLevelUpper;
const isLineEndingMode = (value: string): value is LineEndingMode =>
(LINE_ENDING_MODES as readonly string[]).includes(value);
if (!isLineEndingMode(lineEndingsStr)) {
throw new Error(
`Invalid line endings mode '${lineEndingsStr}'. Valid values are: ${LINE_ENDING_MODES.join(", ")}`
);
}
const lineEndings = lineEndingsStr;
return { return {
localPath: requiredLocalPath, localPath: requiredLocalPath,
remoteUri: requiredRemoteUri, remoteUri: requiredRemoteUri,
@ -223,7 +200,6 @@ Environment variables:
logLevel, logLevel,
health, health,
enableTelemetry, enableTelemetry,
quiet, quiet
lineEndings
}; };
} }

View file

@ -1,8 +1,7 @@
import * as path from "path"; import * as path from "path";
import * as fs from "fs/promises"; import * as fs from "fs/promises";
import * as fsSync from "fs"; import * as fsSync from "fs";
import type { NetworkConnectionStatus , import type { NetworkConnectionStatus, Logger } from "sync-client";
Logger} from "sync-client";
import { import {
SyncClient, SyncClient,
DEFAULT_SETTINGS, DEFAULT_SETTINGS,
@ -11,7 +10,7 @@ import {
type SyncSettings, type SyncSettings,
type StoredDatabase type StoredDatabase
} from "sync-client"; } from "sync-client";
import { parseArgs, type LineEndingMode } from "./args"; import { parseArgs } from "./args";
import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem"; import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem";
import { FileWatcher } from "./file-watcher"; import { FileWatcher } from "./file-watcher";
import { formatLogLine } from "./logger-formatter"; import { formatLogLine } from "./logger-formatter";
@ -41,7 +40,6 @@ const LOG_LEVEL_ORDER = {
function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void {
return (logLine: LogLine): void => { return (logLine: LogLine): void => {
if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) { if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) {
console.log(formatLogLine(logLine)); console.log(formatLogLine(logLine));
} }
}; };
@ -50,17 +48,6 @@ function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void {
const HEALTH_CHECK_INTERVAL_MS = 30 * 1000; const HEALTH_CHECK_INTERVAL_MS = 30 * 1000;
const PROGRESS_LOG_INTERVAL_MS = 2000; const PROGRESS_LOG_INTERVAL_MS = 2000;
function resolveLineEndings(mode: LineEndingMode): string {
switch (mode) {
case "lf":
return "\n";
case "crlf":
return "\r\n";
case "auto":
return process.platform === "win32" ? "\r\n" : "\n";
}
}
async function main(): Promise<void> { async function main(): Promise<void> {
const args = parseArgs(process.argv); const args = parseArgs(process.argv);
const absolutePath = path.resolve(args.localPath); const absolutePath = path.resolve(args.localPath);
@ -96,12 +83,6 @@ async function main(): Promise<void> {
emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`); emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`);
emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`); emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`);
emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`); emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`);
if (args.lineEndings !== "auto") {
emitBoot(
LogLevel.INFO,
`Line endings: ${args.lineEndings.toUpperCase()}`
);
}
} }
const dataDir = path.join(absolutePath, VAULTLINK_DIR); const dataDir = path.join(absolutePath, VAULTLINK_DIR);
@ -158,8 +139,7 @@ async function main(): Promise<void> {
JSON.stringify(persistedDatabase, null, 2) JSON.stringify(persistedDatabase, null, 2)
); );
} }
}, }
nativeLineEndings: resolveLineEndings(args.lineEndings)
}); });
if (args.health !== undefined) { if (args.health !== undefined) {
@ -279,7 +259,6 @@ async function main(): Promise<void> {
} }
main().catch((error: unknown) => { main().catch((error: unknown) => {
console.error( console.error(
`Unexpected error: ${error instanceof Error ? error.message : String(error)}` `Unexpected error: ${error instanceof Error ? error.message : String(error)}`
); );

View file

@ -5,7 +5,7 @@ import type {
TAbstractFile, TAbstractFile,
WorkspaceLeaf WorkspaceLeaf
} from "obsidian"; } from "obsidian";
import { Notice, Platform, Plugin, TFile } from "obsidian"; import { Notice, Plugin, TFile } from "obsidian";
import "../manifest.json"; import "../manifest.json";
import { HistoryView } from "./views/history/history-view"; import { HistoryView } from "./views/history/history-view";
import { StatusBar } from "./views/status-bar/status-bar"; import { StatusBar } from "./views/status-bar/status-bar";
@ -132,7 +132,6 @@ export default class VaultLinkPlugin extends Plugin {
load: this.loadData.bind(this), load: this.loadData.bind(this),
save: this.saveData.bind(this) save: this.saveData.bind(this)
}, },
nativeLineEndings: Platform.isWin ? "\r\n" : "\n",
...(IS_DEBUG_BUILD ...(IS_DEBUG_BUILD
? { ? {
fetch: debugging.slowFetchFactory(1), fetch: debugging.slowFetchFactory(1),

View file

@ -1,7 +1,7 @@
# Sync client (API v5) # Sync client (API v5)
The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket
messages wake it; complete disk scans and the durable server event log are the `vaultChanged` hints wake it; complete disk scans and the durable server event log are the
sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a
positive interval to repair missed notifications. Unset or `0` disables polling. positive interval to repair missed notifications. Unset or `0` disables polling.
Create, delete and move notifications persist logical identity changes before Create, delete and move notifications persist logical identity changes before
@ -13,6 +13,8 @@ as content updates. This distinguishes a deliberate delete/recreate from an edit
Use `waitUntilFinished()` to await the active attempt; errors leave Use `waitUntilFinished()` to await the active attempt; errors leave
pending requests saved and are reported to the caller. Transient failures retry; pending requests saved and are reported to the caller. Transient failures retry;
authentication and protocol errors wait for an explicit wake or settings change. authentication and protocol errors wait for an explicit wake or settings change.
Pausing or destroying the client aborts its HTTP session, including body reads;
resuming starts a fresh session. Connection checks can run while sync is disabled.
## State and policy ## State and policy

View file

@ -2,5 +2,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
export const DIFF_CACHE_SIZE_MB = 2; export const DIFF_CACHE_SIZE_MB = 2;
export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_LOG_MESSAGE_COUNT = 100000;
export const MAX_HISTORY_ENTRY_COUNT = 5000; export const MAX_HISTORY_ENTRY_COUNT = 5000;
export const SUPPORTED_API_VERSION = 4; export const SUPPORTED_API_VERSION = 5;
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;

View file

@ -57,3 +57,33 @@ it("owns settings arrays instead of sharing them with callers and listeners", as
await settings.setSettings({ maxFileSizeMB: 12 }); await settings.setSettings({ maxFileSizeMB: 12 });
assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]); assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]);
}); });
it("file eligibility follows successfully saved settings", async () => {
let fail = false;
const settings = new Settings(
new Logger(),
{
ignorePatterns: ["private/**"],
maxFileSizeMB: 1
},
async () => {
if (fail) throw new Error("save failed");
}
);
assert(settings.isIgnored("private/note.md"));
assert(settings.isIgnored(".vault-link-sync/state.json"));
assert(!settings.isIgnored("public/note.md"));
assert(!settings.isOversized(1024 * 1024));
assert(settings.isOversized(1024 * 1024 + 1));
fail = true;
await assert.rejects(
settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 }),
/save failed/
);
assert(settings.isIgnored("private/note.md"));
assert(settings.isOversized(1024 * 1024 + 1));
fail = false;
await settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 });
assert(!settings.isIgnored("private/note.md"));
assert(!settings.isOversized(1024 * 1024 + 1));
});

View file

@ -1,4 +1,6 @@
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
import { globsToRegexes } from "../utils/globs-to-regexes";
import { isInternalPath } from "../utils/portable-path";
import { Lock } from "../utils/data-structures/locks"; import { Lock } from "../utils/data-structures/locks";
import { EventListeners } from "../utils/data-structures/event-listeners"; import { EventListeners } from "../utils/data-structures/event-listeners";
@ -43,6 +45,7 @@ export class Settings {
>(); >();
private settings: SyncSettings; private settings: SyncSettings;
private ignorePatterns: RegExp[];
private readonly lock: Lock = new Lock(); private readonly lock: Lock = new Lock();
public constructor( public constructor(
@ -55,11 +58,27 @@ export class Settings {
...structuredClone(initialState ?? {}) ...structuredClone(initialState ?? {})
}; };
this.ignorePatterns = globsToRegexes(
this.settings.ignorePatterns,
logger
);
this.logger.debug( this.logger.debug(
`Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}` `Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}`
); );
} }
public isIgnored(path: string): boolean {
return (
isInternalPath(path) ||
this.ignorePatterns.some((pattern) => pattern.test(path))
);
}
public isOversized(size: number): boolean {
return size > this.settings.maxFileSizeMB * 1024 * 1024;
}
public getSettings(): SyncSettings { public getSettings(): SyncSettings {
return { return {
...this.settings, ...this.settings,
@ -91,6 +110,11 @@ export class Settings {
const next = { ...this.settings, ...update }; const next = { ...this.settings, ...update };
await this.saveData(next); await this.saveData(next);
this.settings = next; this.settings = next;
if (update.ignorePatterns !== undefined)
this.ignorePatterns = globsToRegexes(
next.ignorePatterns,
this.logger
);
return [this.getSettings(), oldSettings] as const; return [this.getSettings(), oldSettings] as const;
}); });

View file

@ -1,177 +0,0 @@
import type { Mock } from "node:test";
import { describe, it, mock, beforeEach, afterEach } from "node:test";
import assert from "node:assert";
import { FetchController } from "./fetch-controller";
import { Logger } from "../tracing/logger";
import { SyncResetError } from "../errors/errors";
import { sleep } from "../utils/sleep";
describe("FetchController", () => {
const createMockFetch = (
shouldSleep: boolean
): Mock<() => Promise<Response>> =>
mock.fn(async () => {
if (shouldSleep) {
await sleep(30);
}
return Promise.resolve(new Response("OK", { status: 200 }));
});
beforeEach(() => {
mock.timers.enable({ apis: ["setTimeout"] });
});
afterEach(() => {
mock.timers.reset();
});
it("should allow fetch when canFetch is true", async () => {
const logger = new Logger();
const controller = new FetchController(true, logger);
const mockFetch = createMockFetch(false);
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
const response = await controlledFetch("http://example.com");
assert.strictEqual(await response.text(), "OK");
assert.strictEqual(mockFetch.mock.calls.length, 1);
});
it("should block fetch until canFetch becomes true", async () => {
const logger = new Logger();
const controller = new FetchController(false, logger);
const mockFetch = createMockFetch(true);
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
const fetchPromise = controlledFetch("http://example.com");
assert.strictEqual(mockFetch.mock.calls.length, 0);
controller.canFetch = true;
await Promise.resolve();
mock.timers.tick(30);
const response = await fetchPromise;
assert.strictEqual(await response.text(), "OK");
assert.strictEqual(mockFetch.mock.calls.length, 1);
});
it("should reject during reset", async () => {
const logger = new Logger();
const controller = new FetchController(true, logger);
const mockFetch = createMockFetch(true);
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
const firstRequest = controlledFetch("http://example.com");
assert.strictEqual(mockFetch.mock.calls.length, 1);
controller.startReset();
const secondRequest = controlledFetch("http://example.com");
await assert.rejects(
firstRequest,
(error: unknown) => error instanceof SyncResetError
);
await assert.rejects(
secondRequest,
(error: unknown) => error instanceof SyncResetError
);
assert.strictEqual(mockFetch.mock.calls.length, 1);
});
it("should allow fetch after reset finishes", async () => {
const logger = new Logger();
const controller = new FetchController(true, logger);
const mockFetch = createMockFetch(false);
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
controller.startReset();
controller.finishReset();
const response = await controlledFetch("http://example.com");
assert.strictEqual(await response.text(), "OK");
});
it("should defer canFetch changes during reset", async () => {
const logger = new Logger();
const controller = new FetchController(false, logger);
const mockFetch = createMockFetch(true);
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
controller.startReset();
controller.canFetch = true;
await assert.rejects(
async () => controlledFetch("http://example.com"),
(error: unknown) => error instanceof SyncResetError
);
controller.finishReset();
const fetchPromise = controlledFetch("http://example.com");
mock.timers.tick(30);
const response = await fetchPromise;
assert.strictEqual(await response.text(), "OK");
});
it("should handle different input types", async () => {
const logger = new Logger();
const controller = new FetchController(true, logger);
const mockFetch = createMockFetch(false);
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
await controlledFetch("http://example.com");
await controlledFetch(new URL("http://example.com"));
await controlledFetch(
new Request("http://example.com", { method: "POST" })
);
assert.strictEqual(mockFetch.mock.calls.length, 3);
});
it("should handle fetch errors", async () => {
const logger = new Logger();
const controller = new FetchController(true, logger);
const mockFetch = mock.fn(async () => {
throw new Error("Network error");
});
const controlledFetch = controller.getControlledFetchImplementation(
logger,
mockFetch
);
await assert.rejects(
async () => controlledFetch("http://example.com"),
(error: unknown) =>
error instanceof Error && error.message === "Network error"
);
});
it("should not create unhandled rejection on reset with no waiting fetches", async () => {
const logger = new Logger();
const controller = new FetchController(true, logger);
controller.startReset();
mock.timers.tick(10);
controller.finishReset();
});
});

View file

@ -1,149 +0,0 @@
import type { Logger } from "../tracing/logger";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "../errors/errors";
/**
* Offers a resettable fetch implementation that waits until syncing is enabled
* and aborts outstanding requests when a reset is started.
*/
export class FetchController {
private static readonly UNTIL_RESOLUTION = Symbol();
private isResetting = false;
// Promise resolves on the next state change: sync enabled/disabled or reset started/ended
private until: Promise<symbol>;
private resolveUntil: (result: symbol) => unknown;
private rejectUntil: (reason: unknown) => unknown;
public constructor(
private _canFetch: boolean,
private readonly logger: Logger
) {
[this.until, this.resolveUntil, this.rejectUntil] =
createPromise<symbol>();
}
/**
* Whether the fetch implementation can immediately send requests once outside of a reset.
*/
public get canFetch(): boolean {
return this._canFetch;
}
/**
* Allow or disallow fetching. The changes only take effect if not resetting.
* When called during a reset, its effect is deferred until the reset is finished.
*
* @param canFetch Whether fetching is enabled
*/
public set canFetch(canFetch: boolean) {
this._canFetch = canFetch;
if (!this.isResetting) {
const previousResolve = this.resolveUntil;
[this.until, this.resolveUntil, this.rejectUntil] =
createPromise<symbol>();
previousResolve(FetchController.UNTIL_RESOLUTION);
}
}
private static getUrlFromInput(input: RequestInfo | URL): string {
if (input instanceof URL) {
return input.href;
}
if (typeof input === "string") {
return input;
}
return input.url;
}
/**
* Starts a reset, causing all ongoing and future fetches to be rejected
* with a SyncResetError until finishReset is called.
*/
public startReset(): void {
this.isResetting = true;
this.rejectUntil(new SyncResetError());
// Catch unhandled rejection if no fetches are waiting
this.until.catch(() => {
// Intentionally ignore - this rejection is handled by waiting fetches
});
}
/**
* Finishes a reset, allowing fetches to proceed or wait again depending on
* the current sync settings.
*/
public finishReset(): void {
if (!this.isResetting) {
return;
}
this.isResetting = false;
[this.until, this.resolveUntil, this.rejectUntil] = createPromise();
}
/**
*
* |------------------|---------------|-----------------------------------------------------|
* | | Sync enabled | Sync disabled |
* |------------------|-------------- |-----------------------------------------------------|
* | During reset | Rejects with SyncResetError without sending request |
* |------------------|-------------- |-----------------------------------------------------|
* | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch |
* |------------------|---------------|-----------------------------------------------------|
*
* @param logger for errors
* @param fetch to wrap
* @returns a wrapped fetch implementation affected by the FetchController state
*/
public getControlledFetchImplementation(
logger: Logger,
fetch: typeof globalThis.fetch = globalThis.fetch
): typeof globalThis.fetch {
return async (
input: RequestInfo | URL,
init?: RequestInit
): Promise<Response> => {
while (!this.canFetch || this.isResetting) {
await this.until;
}
try {
// https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21
const _input =
typeof Request !== "undefined" && input instanceof Request
? input.clone()
: input;
const fetchPromise = fetch(_input, init);
// We only want to catch rejections from `this.until`
let result: symbol | Response | undefined = undefined;
do {
result = await Promise.race([this.until, fetchPromise]);
} while (result === FetchController.UNTIL_RESOLUTION);
const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
if (!fetchResult.ok) {
this.logger.warn(
`Fetch for ${FetchController.getUrlFromInput(
input
)}, got status ${fetchResult.status}`
);
}
return fetchResult;
} catch (error) {
logger.warn(
`Fetch for ${FetchController.getUrlFromInput(
input
)}, got error: ${error}`
);
throw error;
}
};
}
}

View file

@ -0,0 +1,146 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { Settings } from "../persistence/settings";
import { Logger } from "../tracing/logger";
import { SyncResetError } from "../errors/errors";
import { SyncService } from "./sync-service";
const settings = () =>
new Settings(
new Logger(),
{
remoteUri: "http://test",
vaultName: "test",
requestTimeoutMs: 1000
},
async () => {}
);
test("paused sync sends no requests, while connection checks still work", async () => {
let requests = 0;
const service = new SyncService("device", settings(), async () => {
requests++;
return Response.json({ headEventId: 0, events: [] });
});
await assert.rejects(service.events(0), SyncResetError);
assert.equal(requests, 0);
await service.ping();
assert.equal(requests, 1);
service.resume();
assert.equal((await service.events(0)).headEventId, 0);
assert.equal(requests, 2);
});
test("pause aborts requests and fences late responses from an injected fetch", async () => {
const response = Promise.withResolvers<Response>();
const signals: AbortSignal[] = [];
const saved: (string | undefined)[] = [];
const service = new SyncService(
"device",
settings(),
async (_, init) => {
signals.push(init!.signal!);
return signals.length === 1
? response.promise
: Response.json({ headEventId: 0, events: [] });
},
{
get: () => undefined,
save: async (value) => {
saved.push(value);
}
}
);
service.resume();
const first = service.events(0);
service.pause();
await assert.rejects(first, SyncResetError);
assert.equal(signals[0].aborted, true);
service.resume();
await service.events(0);
assert.equal(signals[1].aborted, false);
response.resolve(
Response.json(
{},
{ headers: { "X-Vault-Link-History": "99:abandoned" } }
)
);
await new Promise((resolve) => setImmediate(resolve));
assert.deepEqual(saved, []);
});
for (const binary of [false, true]) {
test(`pause interrupts a stalled ${binary ? "binary" : "JSON"} response body`, async () => {
const headersSaved = Promise.withResolvers<void>();
let body!: ReadableStreamDefaultController<Uint8Array>;
const service = new SyncService(
"device",
settings(),
async () =>
new Response(
new ReadableStream({
start(controller) {
body = controller;
}
}),
{ headers: { "X-Vault-Link-History": "1:head" } }
),
{
get: () => undefined,
save: async () => {
headersSaved.resolve();
}
}
);
service.resume();
const request = binary
? service.getDocumentVersionContent({
documentId: "a",
vaultUpdateId: 1
})
: service.events(0);
await headersSaved.promise;
await new Promise((resolve) => setImmediate(resolve));
service.pause();
await assert.rejects(request, SyncResetError);
body.enqueue(new TextEncoder().encode("{}"));
body.close();
});
}
test("pause waits for a checkpoint save already in progress", async () => {
const saving = Promise.withResolvers<void>();
const release = Promise.withResolvers<void>();
let settled = false;
let checkpoint: string | undefined;
const service = new SyncService(
"device",
settings(),
async () =>
Response.json(
{},
{
headers: { "X-Vault-Link-History": "1:head" }
}
),
{
get: () => checkpoint,
save: async (value) => {
saving.resolve();
await release.promise;
checkpoint = value;
}
}
);
service.resume();
const request = service.events(0).finally(() => {
settled = true;
});
await saving.promise;
service.pause();
await new Promise((resolve) => setImmediate(resolve));
assert.equal(settled, false);
release.resolve();
await assert.rejects(request, SyncResetError);
assert.equal(checkpoint, "1:head");
});

View file

@ -1,11 +1,7 @@
import type { Settings } from "../persistence/settings"; import type { Settings } from "../persistence/settings";
import type { FetchController } from "./fetch-controller";
import type { Logger } from "../tracing/logger";
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
import type { DocumentVersion } from "./types/DocumentVersion";
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
import type { EventBatch } from "./types/EventBatch"; import type { EventBatch } from "./types/EventBatch";
import type { FileManifest } from "./types/FileManifest";
import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse"; import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse";
import type { PutFileContent } from "./types/PutFileContent"; import type { PutFileContent } from "./types/PutFileContent";
import type { PushFileManifest } from "./types/PushFileManifest"; import type { PushFileManifest } from "./types/PushFileManifest";
@ -14,29 +10,47 @@ import type { PingResponse } from "./types/PingResponse";
import { import {
AuthenticationError, AuthenticationError,
PermanentSyncError, PermanentSyncError,
ServerHistoryChangedError ServerHistoryChangedError,
SyncResetError
} from "../errors/errors"; } from "../errors/errors";
// Injected fetch implementations may ignore cancellation. Stop waiting anyway,
// and never let their late responses reach checkpoint or engine persistence.
async function abortable<T>(
signal: AbortSignal,
operation: () => Promise<T>
): Promise<T> {
signal.throwIfAborted();
let abort = (): void => undefined;
try {
return await Promise.race([
new Promise<never>((_, reject) => {
abort = (): void => {
// eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- Only reset errors and timeout errors reach this helper.
reject(signal.reason);
};
signal.addEventListener("abort", abort, { once: true });
}),
operation()
]);
} finally {
signal.removeEventListener("abort", abort);
}
}
export class SyncService { export class SyncService {
private readonly client: typeof fetch; private session = new AbortController();
private readonly rawFetch: typeof fetch;
public constructor( public constructor(
private readonly deviceId: string, private readonly deviceId: string,
fetchController: FetchController,
private readonly settings: Settings, private readonly settings: Settings,
logger: Logger, private readonly fetchImplementation: typeof fetch = globalThis.fetch,
fetchImplementation: typeof fetch = globalThis.fetch,
private readonly history?: { private readonly history?: {
get: () => string | undefined; get: () => string | undefined;
save: (checkpoint: string | undefined) => Promise<void>; save: (checkpoint: string | undefined) => Promise<void>;
} }
) { ) {
this.rawFetch = async (...args) => fetchImplementation(...args); this.pause();
this.client = fetchController.getControlledFetchImplementation(
logger,
this.rawFetch
);
} }
public get verifiesHistory(): boolean { public get verifiesHistory(): boolean {
@ -46,21 +60,80 @@ export class SyncService {
return this.history?.get() !== undefined; return this.history?.get() !== undefined;
} }
public resume(): void {
this.session = new AbortController();
}
public pause(): void {
this.session.abort(new SyncResetError());
}
public async resetHistory(): Promise<void> { public async resetHistory(): Promise<void> {
await this.history?.save(undefined); await this.history?.save(undefined);
} }
public async ping(): Promise<PingResponse> {
return this.request("/ping", undefined, true);
}
public async vaultSnapshot(): Promise<VaultSnapshot> {
return this.request("/vault-snapshot");
}
public async events(after: number): Promise<EventBatch> {
return this.request(`/events-since?after=${after}`);
}
public async pushFileManifest(
request: PushFileManifest
): Promise<FileManifestUpdateResponse> {
return this.request("/file-manifest", request);
}
public async putFileContent(
id: string,
request: PutFileContent
): Promise<DocumentUpdateResponse> {
return this.request(`/documents/${id}`, request);
}
public async metadata(id: string): Promise<DocumentVersionWithoutContent> {
return this.request(`/documents/${id}/metadata`);
}
public async getDocumentVersionContent({
documentId,
vaultUpdateId
}: {
documentId: string;
vaultUpdateId: number;
}): Promise<Uint8Array> {
return this.request(
`/documents/${documentId}/versions/${vaultUpdateId}/content`,
undefined,
false,
async (response) => new Uint8Array(await response.arrayBuffer())
);
}
private historyHeaders(): Record<string, string> { private historyHeaders(): Record<string, string> {
const checkpoint = this.history?.get(); const checkpoint = this.history?.get();
return checkpoint ? { "X-Vault-Link-History": checkpoint } : {}; return checkpoint !== undefined && checkpoint !== ""
? { "X-Vault-Link-History": checkpoint }
: {};
} }
private async recordHistory(response: Response): Promise<void> { private async recordHistory(response: Response): Promise<void> {
const checkpoint = response.headers.get("x-vault-link-history"); const checkpoint = response.headers.get("x-vault-link-history");
if (checkpoint && this.history) { if (
checkpoint !== null &&
checkpoint !== "" &&
this.history !== undefined
) {
const previous = this.history.get(); const previous = this.history.get();
if ( if (
!previous || previous === undefined ||
previous === "" ||
Number(checkpoint.split(":")[0]) >= Number(checkpoint.split(":")[0]) >=
Number(previous.split(":")[0]) Number(previous.split(":")[0])
) )
@ -75,7 +148,7 @@ export class SyncService {
); );
} }
public getUrl(path: string): string { private getUrl(path: string): string {
const { remoteUri, vaultName } = this.settings.getSettings(); const { remoteUri, vaultName } = this.settings.getSettings();
return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`; return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`;
} }
@ -83,35 +156,42 @@ export class SyncService {
private async request<T>( private async request<T>(
path: string, path: string,
body?: unknown, body?: unknown,
raw = false raw = false,
decode: (response: Response) => Promise<T> = async (response) =>
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Endpoint methods supply the generated protocol response type.
response.json() as Promise<T>
): Promise<T> { ): Promise<T> {
const response = await (raw ? this.rawFetch : this.client)( const timeout = AbortSignal.timeout(
this.getUrl(path), this.settings.getSettings().requestTimeoutMs
{ );
const signal = raw
? timeout
: AbortSignal.any([timeout, this.session.signal]);
const response = await abortable(signal, async () =>
this.fetchImplementation(this.getUrl(path), {
method: body === undefined ? "GET" : "PUT", method: body === undefined ? "GET" : "PUT",
body: body === undefined ? undefined : JSON.stringify(body), body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout( signal,
this.settings.getSettings().requestTimeoutMs
),
headers: { headers: {
...(raw ? {} : this.historyHeaders()), ...(raw ? {} : this.historyHeaders()),
Authorization: `Bearer ${this.settings.getSettings().token}`, Authorization: `Bearer ${this.settings.getSettings().token}`,
"Device-Id": this.deviceId, "Device-Id": this.deviceId,
"Content-Type": "application/json" "Content-Type": "application/json"
} }
} })
); );
signal.throwIfAborted();
this.checkHistory(response); this.checkHistory(response);
if (!response.ok) { if (!response.ok) {
this.throwForErrorResponse( this.throwForErrorResponse(
response, response,
`HTTP ${response.status}: ${await response.text()}` `HTTP ${response.status}: ${await abortable(signal, async () => response.text())}`
); );
} }
// Finish metadata saves even if the transport is paused meanwhile.
if (!raw) await this.recordHistory(response); if (!raw) await this.recordHistory(response);
return (await response.json()) as T; return abortable(signal, async () => decode(response));
} }
private throwForErrorResponse(response: Response, message: string): never { private throwForErrorResponse(response: Response, message: string): never {
@ -128,75 +208,4 @@ export class SyncService {
} }
throw new Error(message); throw new Error(message);
} }
public async ping(): Promise<PingResponse> {
return this.request("/ping", undefined, true);
}
public async vaultSnapshot(): Promise<VaultSnapshot> {
return this.request("/vault-snapshot");
}
public async events(after: number): Promise<EventBatch> {
return this.request(`/events-since?after=${after}`);
}
public async fileManifest(): Promise<FileManifest> {
return this.request("/file-manifest");
}
public async pushFileManifest(
request: PushFileManifest
): Promise<FileManifestUpdateResponse> {
return this.request("/file-manifest", request);
}
public async putFileContent(
id: string,
request: PutFileContent
): Promise<DocumentUpdateResponse> {
return this.request(`/documents/${id}`, request);
}
public async get(id: string): Promise<DocumentVersion> {
return this.request(`/documents/${id}`);
}
public async metadata(id: string): Promise<DocumentVersionWithoutContent> {
return this.request(`/documents/${id}/metadata`);
}
public async getDocumentVersionContent({
documentId,
vaultUpdateId
}: {
documentId: string;
vaultUpdateId: number;
}): Promise<Uint8Array> {
const response = await this.client(
this.getUrl(
`/documents/${documentId}/versions/${vaultUpdateId}/content`
),
{
signal: AbortSignal.timeout(
this.settings.getSettings().requestTimeoutMs
),
headers: {
...this.historyHeaders(),
Authorization: `Bearer ${this.settings.getSettings().token}`
}
}
);
this.checkHistory(response);
if (!response.ok) {
this.throwForErrorResponse(
response,
`Cannot fetch version ${vaultUpdateId}: HTTP ${response.status}`
);
}
await this.recordHistory(response);
return new Uint8Array(await response.arrayBuffer());
}
} }

View file

@ -1,3 +1,3 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type WebSocketHandshake = { token: string, deviceId: string, lastSeenVaultUpdateId: number | null, }; export type WebSocketHandshake = { token: string, deviceId: string, };

View file

@ -1,5 +1,4 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { CursorPositionFromServer } from "./CursorPositionFromServer"; import type { CursorPositionFromServer } from "./CursorPositionFromServer";
import type { EventBatch } from "./EventBatch";
export type WebSocketServerMessage = { "type": "vaultEvents" } & EventBatch | { "type": "cursorPositions" } & CursorPositionFromServer; export type WebSocketServerMessage = { "type": "vaultChanged" } | { "type": "cursorPositions" } & CursorPositionFromServer;

View file

@ -136,19 +136,13 @@ describe("WebSocketManager", () => {
.webSocket; .webSocket;
mockWs.simulateMessage({ mockWs.simulateMessage({
type: "vaultEvents", type: "vaultChanged"
headEventId: 0,
events: []
}); });
mockWs.simulateMessage({ mockWs.simulateMessage({
type: "vaultEvents", type: "vaultChanged"
headEventId: 0,
events: []
}); });
mockWs.simulateMessage({ mockWs.simulateMessage({
type: "vaultEvents", type: "vaultChanged"
headEventId: 0,
events: []
}); });
await new Promise((resolve) => setTimeout(resolve, 100)); await new Promise((resolve) => setTimeout(resolve, 100));
@ -208,8 +202,7 @@ describe("WebSocketManager", () => {
manager.sendHandshakeMessage({ manager.sendHandshakeMessage({
type: "handshake", type: "handshake",
token: "test", token: "test",
deviceId: "test", deviceId: "test"
lastSeenVaultUpdateId: null
}); });
}); });
@ -317,9 +310,7 @@ describe("WebSocketManager", () => {
const mockWs = (manager as unknown as { webSocket: MockWebSocket }) const mockWs = (manager as unknown as { webSocket: MockWebSocket })
.webSocket; .webSocket;
mockWs.simulateMessage({ mockWs.simulateMessage({
type: "vaultEvents", type: "vaultChanged"
headEventId: 0,
events: []
}); });
await new Promise((resolve) => setTimeout(resolve, 10)); await new Promise((resolve) => setTimeout(resolve, 10));
@ -465,9 +456,7 @@ for (const closeBehavior of ["throws", "stays open"]) {
assert.equal(manager.isWebSocketConnected, false); assert.equal(manager.isWebSocketConnected, false);
assert.equal(connected, false); assert.equal(connected, false);
socket.simulateMessage({ socket.simulateMessage({
type: "vaultEvents", type: "vaultChanged"
headEventId: 0,
events: []
}); });
await manager.waitUntilFinished(); await manager.waitUntilFinished();
assert.equal(delivered, 0); assert.equal(delivered, 0);
@ -499,8 +488,7 @@ it("a handshake send failure retires the connection without escaping the open ca
manager.sendHandshakeMessage({ manager.sendHandshakeMessage({
type: "handshake", type: "handshake",
token: "test", token: "test",
deviceId: "test", deviceId: "test"
lastSeenVaultUpdateId: 0
}); });
}); });
const statuses: boolean[] = []; const statuses: boolean[] = [];

View file

@ -5,7 +5,6 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage";
import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient";
import type { ClientCursors } from "./types/ClientCursors"; import type { ClientCursors } from "./types/ClientCursors";
import { createPromise } from "../utils/create-promise"; import { createPromise } from "../utils/create-promise";
import type { EventBatch } from "./types/EventBatch";
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts";
import { removeFromArray } from "../utils/remove-from-array"; import { removeFromArray } from "../utils/remove-from-array";
import { EventListeners } from "../utils/data-structures/event-listeners"; import { EventListeners } from "../utils/data-structures/event-listeners";
@ -17,7 +16,7 @@ export class WebSocketManager {
>(); >();
public readonly onRemoteVaultUpdateReceived = new EventListeners< public readonly onRemoteVaultUpdateReceived = new EventListeners<
(update: EventBatch) => Promise<void> () => Promise<void>
>(); >();
public readonly onRemoteCursorsUpdateReceived = new EventListeners< public readonly onRemoteCursorsUpdateReceived = new EventListeners<
@ -303,7 +302,7 @@ export class WebSocketManager {
event.data event.data
) as WebSocketServerMessage; ) as WebSocketServerMessage;
if (["vaultEvents", "cursorPositions"].includes(message.type)) if (["vaultChanged", "cursorPositions"].includes(message.type))
this.expectResponse(socket); this.expectResponse(socket);
// Track the message handling promise // Track the message handling promise
const messageHandlingPromise = this.handleWebSocketMessage( const messageHandlingPromise = this.handleWebSocketMessage(
@ -340,8 +339,8 @@ export class WebSocketManager {
private async handleWebSocketMessage( private async handleWebSocketMessage(
message: WebSocketServerMessage message: WebSocketServerMessage
): Promise<void> { ): Promise<void> {
if (message.type === "vaultEvents") { if (message.type === "vaultChanged") {
await this.onRemoteVaultUpdateReceived.triggerAsync(message); await this.onRemoteVaultUpdateReceived.triggerAsync();
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
} else if (message.type === "cursorPositions") { } else if (message.type === "cursorPositions") {

View file

@ -11,7 +11,6 @@ import type { FileSystemOperations } from "./file-operations/filesystem-operatio
import { FileOperations } from "./file-operations/file-operations"; import { FileOperations } from "./file-operations/file-operations";
import { Logger, LogLevel } from "./tracing/logger"; import { Logger, LogLevel } from "./tracing/logger";
import { SyncHistory } from "./tracing/sync-history"; import { SyncHistory } from "./tracing/sync-history";
import { FetchController } from "./services/fetch-controller";
import { SyncService } from "./services/sync-service"; import { SyncService } from "./services/sync-service";
import { ServerConfig } from "./services/server-config"; import { ServerConfig } from "./services/server-config";
import { WebSocketManager } from "./services/websocket-manager"; import { WebSocketManager } from "./services/websocket-manager";
@ -23,7 +22,6 @@ import { DocumentSyncStatus } from "./types/document-sync-status";
import type { CursorSpan } from "./services/types/CursorSpan"; import type { CursorSpan } from "./services/types/CursorSpan";
import { isInternalPath } from "./utils/portable-path"; import { isInternalPath } from "./utils/portable-path";
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
import { globsToRegexes } from "./utils/globs-to-regexes";
import { Lock } from "./utils/data-structures/locks"; import { Lock } from "./utils/data-structures/locks";
import { import {
unappliedChanges, unappliedChanges,
@ -61,7 +59,7 @@ export class SyncClient {
private readonly database: Database, private readonly database: Database,
private readonly syncer: Syncer, private readonly syncer: Syncer,
private readonly webSocketManager: WebSocketManager, private readonly webSocketManager: WebSocketManager,
private readonly fetchController: FetchController, private readonly service: SyncService,
private readonly serverConfig: ServerConfig, private readonly serverConfig: ServerConfig,
private readonly cursorTracker: CursorTracker, private readonly cursorTracker: CursorTracker,
private readonly fileChangeNotifier: FileChangeNotifier, private readonly fileChangeNotifier: FileChangeNotifier,
@ -95,8 +93,6 @@ export class SyncClient {
persistence: PersistenceProvider<StoredClient>; persistence: PersistenceProvider<StoredClient>;
fetch?: typeof globalThis.fetch; fetch?: typeof globalThis.fetch;
webSocket?: typeof globalThis.WebSocket; webSocket?: typeof globalThis.WebSocket;
/** V4 preserves bytes and line endings as stored, on every platform. */
nativeLineEndings?: string;
}): Promise<SyncClient> { }): Promise<SyncClient> {
const logger = new Logger(); const logger = new Logger();
let stored = (await persistence.load()) ?? {}; let stored = (await persistence.load()) ?? {};
@ -152,18 +148,8 @@ export class SyncClient {
"Offline notifications belong to another vault; use a separate state store" "Offline notifications belong to another vault; use a separate state store"
); );
} }
const fetchController = new FetchController(
settings.getSettings().isSyncEnabled,
logger
);
const deviceId = createClientId(); const deviceId = createClientId();
const service = new SyncService( const service = new SyncService(deviceId, settings, fetch, {
deviceId,
fetchController,
settings,
logger,
fetch,
{
get: () => get: () =>
stored.historyCheckpoint?.vaultKey === stored.historyCheckpoint?.vaultKey ===
vaultKey(settings.getSettings()) vaultKey(settings.getSettings())
@ -176,8 +162,7 @@ export class SyncClient {
checkpoint checkpoint
} }
}) })
} });
);
const serverConfig = new ServerConfig(service); const serverConfig = new ServerConfig(service);
const notifier = new FileChangeNotifier(); const notifier = new FileChangeNotifier();
let syncer: Syncer; let syncer: Syncer;
@ -192,15 +177,8 @@ export class SyncClient {
entries: () => localChanges, entries: () => localChanges,
flush: async () => syncer?.flushLocalChanges() flush: async () => syncer?.flushLocalChanges()
}, },
(path, size) => { (path, size) =>
const current = settings.getSettings(); settings.isIgnored(path) || settings.isOversized(size)
return (
size > current.maxFileSizeMB * 1024 * 1024 ||
globsToRegexes(current.ignorePatterns, logger).some(
(pattern) => pattern.test(path)
)
);
}
); );
const websocket = new WebSocketManager( const websocket = new WebSocketManager(
deviceId, deviceId,
@ -240,7 +218,7 @@ export class SyncClient {
database, database,
syncer, syncer,
websocket, websocket,
fetchController, service,
serverConfig, serverConfig,
cursors, cursors,
notifier, notifier,
@ -308,15 +286,14 @@ export class SyncClient {
if (this.destroyed) return; if (this.destroyed) return;
if (!this.settings.getSettings().isSyncEnabled) return; if (!this.settings.getSettings().isSyncEnabled) return;
await this.database.bindVault(vaultKey(this.settings.getSettings())); await this.database.bindVault(vaultKey(this.settings.getSettings()));
this.fetchController.finishReset(); this.service.resume();
this.fetchController.canFetch = true;
this.syncer.start(); this.syncer.start();
await this.syncer.waitUntilFinished(); await this.syncer.waitUntilFinished();
} }
private async pause(): Promise<void> { private async pause(): Promise<void> {
const stopped = this.syncer.stop(); const stopped = this.syncer.stop();
this.fetchController.startReset(); this.service.pause();
await this.webSocketManager.stop(); await this.webSocketManager.stop();
await stopped; await stopped;
} }
@ -470,7 +447,7 @@ export class SyncClient {
this.destroyed = true; this.destroyed = true;
// Interrupt a start/reset awaiting network progress before waiting for // Interrupt a start/reset awaiting network progress before waiting for
// its lifecycle operation to finish. // its lifecycle operation to finish.
this.fetchController.startReset(); this.service.pause();
const stopped = this.syncer.stop(); const stopped = this.syncer.stop();
this.destroying = this.lifecycle.withLock(async () => { this.destroying = this.lifecycle.withLock(async () => {
try { try {

View file

@ -4,16 +4,15 @@ import { afterEach, beforeEach, describe, it, mock } from "node:test";
import type { Database } from "../persistence/database"; import type { Database } from "../persistence/database";
import { import {
DEFAULT_SETTINGS, DEFAULT_SETTINGS,
type Settings, Settings,
type SyncSettings type SyncSettings
} from "../persistence/settings"; } from "../persistence/settings";
import type { FileOperations } from "../file-operations/file-operations"; import type { FileOperations } from "../file-operations/file-operations";
import type { SyncService } from "../services/sync-service"; import type { SyncService } from "../services/sync-service";
import type { ServerConfig } from "../services/server-config"; import type { ServerConfig } from "../services/server-config";
import type { WebSocketManager } from "../services/websocket-manager"; import type { WebSocketManager } from "../services/websocket-manager";
import type { Logger } from "../tracing/logger"; import { Logger } from "../tracing/logger";
import type { SyncHistory } from "../tracing/sync-history"; import type { SyncHistory } from "../tracing/sync-history";
import type { EventBatch } from "../services/types/EventBatch";
import { EventListeners } from "../utils/data-structures/event-listeners"; import { EventListeners } from "../utils/data-structures/event-listeners";
import { Syncer } from "./syncer"; import { Syncer } from "./syncer";
import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache";
@ -24,12 +23,11 @@ function createSyncer(overrides: Partial<SyncSettings> = {}): {
syncer: Syncer; syncer: Syncer;
runCount: () => number; runCount: () => number;
} { } {
const settings = { const settings = new Settings(
getSettings: () => ({ new Logger(),
...DEFAULT_SETTINGS, overrides,
...overrides async () => undefined
}) );
} as unknown as Settings;
const database = { const database = {
state: {} state: {}
} as unknown as Database; } as unknown as Database;
@ -162,7 +160,7 @@ describe("Syncer per-file state and event delivery", () => {
(connected: boolean) => unknown (connected: boolean) => unknown
>(), >(),
onRemoteVaultUpdateReceived: new EventListeners< onRemoteVaultUpdateReceived: new EventListeners<
(batch: EventBatch) => Promise<void> () => Promise<void>
>() >()
} as unknown as WebSocketManager; } as unknown as WebSocketManager;
const service = { const service = {
@ -174,9 +172,11 @@ describe("Syncer per-file state and event delivery", () => {
"device", "device",
{} as Logger, {} as Logger,
database, database,
{ new Settings(
getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) new Logger(),
} as Settings, { syncIntervalMs: 0 },
async () => undefined
),
service, service,
websocket, websocket,
{} as FileOperations, {} as FileOperations,
@ -238,9 +238,11 @@ describe("Syncer diff uploads", () => {
"device", "device",
{} as Logger, {} as Logger,
{ state: {} } as Database, { state: {} } as Database,
{ new Settings(
getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) new Logger(),
} as Settings, { syncIntervalMs: 0 },
async () => undefined
),
{ {
getDocumentVersionContent: async () => parent getDocumentVersionContent: async () => parent
} as unknown as SyncService, } as unknown as SyncService,

View file

@ -26,7 +26,6 @@ import type { VaultSnapshot } from "../services/types/VaultSnapshot";
import type { PushContent } from "../services/types/PushContent"; import type { PushContent } from "../services/types/PushContent";
import type { Logger } from "../tracing/logger"; import type { Logger } from "../tracing/logger";
import { EventListeners } from "../utils/data-structures/event-listeners"; import { EventListeners } from "../utils/data-structures/event-listeners";
import { globsToRegexes } from "../utils/globs-to-regexes";
import { mergeContent, toStoredSnapshot } from "./content"; import { mergeContent, toStoredSnapshot } from "./content";
import { scanLocalFiles, type LocalChange } from "./scan"; import { scanLocalFiles, type LocalChange } from "./scan";
import { import {
@ -91,8 +90,7 @@ export class Syncer {
websocket.sendHandshakeMessage({ websocket.sendHandshakeMessage({
type: "handshake", type: "handshake",
token: settings.getSettings().token, token: settings.getSettings().token,
deviceId, deviceId
lastSeenVaultUpdateId: database.state.lastSeenUpdateId
}); });
} }
// Failed upgrades and lost transports still need HTTP catchup, // Failed upgrades and lost transports still need HTTP catchup,
@ -128,7 +126,7 @@ export class Syncer {
!this.hasScanned || !this.hasScanned ||
this.pendingPaths.has(path) || this.pendingPaths.has(path) ||
this.unsyncablePaths.has(path) || this.unsyncablePaths.has(path) ||
this.ignored(path) this.settings.isIgnored(path)
) { ) {
return false; return false;
} }
@ -414,7 +412,10 @@ export class Syncer {
const info = await this.files.fs.stat(path); const info = await this.files.fs.stat(path);
if (info?.kind !== "file") continue; if (info?.kind !== "file") continue;
const doc = previous.documents[id]; const doc = previous.documents[id];
if (this.ignored(path) || this.oversized(info.size)) { if (
this.settings.isIgnored(path) ||
this.settings.isOversized(info.size)
) {
next.local[id] = path; next.local[id] = path;
next.documents[id] = { next.documents[id] = {
materialized: true, materialized: true,
@ -457,20 +458,6 @@ export class Syncer {
}; };
} }
private ignored(path: string): boolean {
return (
isInternalPath(path) ||
globsToRegexes(
this.settings.getSettings().ignorePatterns,
this.logger
).some((pattern) => pattern.test(path))
);
}
private oversized(size: number): boolean {
return size > this.settings.getSettings().maxFileSizeMB * 1024 * 1024;
}
private async cacheContent( private async cacheContent(
updateId: number, updateId: number,
path: string | undefined, path: string | undefined,
@ -550,8 +537,8 @@ export class Syncer {
guard: this.localGuard(), guard: this.localGuard(),
changes: this.changes, changes: this.changes,
files: this.files, files: this.files,
ignored: (path) => this.ignored(path), ignored: (path) => this.settings.isIgnored(path),
oversized: (size) => this.oversized(size), oversized: (size) => this.settings.isOversized(size),
commit: async (next) => this.database.commit(next) commit: async (next) => this.database.commit(next)
}, },
initial initial
@ -735,8 +722,8 @@ export class Syncer {
}); });
if ( if (
next.excluded?.[head.documentId] !== undefined || next.excluded?.[head.documentId] !== undefined ||
this.ignored(path) || this.settings.isIgnored(path) ||
this.oversized(head.contentSize) this.settings.isOversized(head.contentSize)
) { ) {
return undefined; return undefined;
} }
@ -748,7 +735,7 @@ export class Syncer {
: undefined; : undefined;
if ( if (
local !== undefined && local !== undefined &&
this.oversized(base64ToBytes(local.contentBase64).length) this.settings.isOversized(base64ToBytes(local.contentBase64).length)
) { ) {
return undefined; return undefined;
} }
@ -869,9 +856,9 @@ export class Syncer {
for (const [id, oldPath] of Object.entries(before)) { for (const [id, oldPath] of Object.entries(before)) {
const path = next.local[id]; const path = next.local[id];
if ( if (
this.ignored(oldPath) || this.settings.isIgnored(oldPath) ||
this.unsyncablePaths.has(oldPath) || this.unsyncablePaths.has(oldPath) ||
(path !== undefined && this.ignored(path)) (path !== undefined && this.settings.isIgnored(path))
) { ) {
next.excluded ??= {}; next.excluded ??= {};
if (next.excluded[id] === undefined) if (next.excluded[id] === undefined)
@ -882,7 +869,7 @@ export class Syncer {
for (const [id, path] of Object.entries(next.excluded ?? {})) { for (const [id, path] of Object.entries(next.excluded ?? {})) {
if ( if (
before[id] === undefined && before[id] === undefined &&
(path === null || !this.ignored(path)) (path === null || !this.settings.isIgnored(path))
) )
delete next.excluded![id]; delete next.excluded![id];
} }
@ -906,7 +893,7 @@ export class Syncer {
next.documents[id].materialized = false; next.documents[id].materialized = false;
if ( if (
next.excluded?.[id] === undefined && next.excluded?.[id] === undefined &&
!this.ignored(path) && !this.settings.isIgnored(path) &&
remote.entries[id] !== undefined && remote.entries[id] !== undefined &&
(!this.database.state.local[id] || (!this.database.state.local[id] ||
!next.documents[id].base || !next.documents[id].base ||
@ -971,11 +958,11 @@ export class Syncer {
} }
for (const [id, path] of Object.entries(this.database.state.local)) { for (const [id, path] of Object.entries(this.database.state.local)) {
if ( if (
this.ignored(path) || this.settings.isIgnored(path) ||
this.database.state.excluded?.[id] !== undefined || this.database.state.excluded?.[id] !== undefined ||
!this.database.state.documents[id]?.materialized || !this.database.state.documents[id]?.materialized ||
this.unsyncablePaths.has(path) || this.unsyncablePaths.has(path) ||
this.oversized( this.settings.isOversized(
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime. // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime.
this.database.state.remoteHeads[id]?.contentSize ?? 0 this.database.state.remoteHeads[id]?.contentSize ?? 0
) )
@ -985,7 +972,9 @@ export class Syncer {
const snapshot = await this.files.snapshot(path); const snapshot = await this.files.snapshot(path);
if ( if (
!snapshot || !snapshot ||
this.oversized(base64ToBytes(snapshot.contentBase64).length) this.settings.isOversized(
base64ToBytes(snapshot.contentBase64).length
)
) { ) {
continue; continue;
} }
@ -1088,7 +1077,7 @@ export class Syncer {
doc?.rejected && doc?.rejected &&
doc.observedHash === doc.rejected.hash && doc.observedHash === doc.rejected.hash &&
doc.base?.hash !== doc.rejected.hash && doc.base?.hash !== doc.rejected.hash &&
!this.ignored(path) && !this.settings.isIgnored(path) &&
state.excluded?.[id] === undefined state.excluded?.[id] === undefined
) )
return doc.rejected.message; return doc.rejected.message;

Some files were not shown because too many files have changed in this diff Show more