simplify 2
This commit is contained in:
parent
2fcdbfd27d
commit
82929d7479
30 changed files with 503 additions and 1126 deletions
|
|
@ -76,8 +76,9 @@ old base are preserved as separate local documents, with conflict names if neede
|
||||||
Interrupted recovery saves are replayable. Backups must be restored with the server
|
Interrupted recovery saves are replayable. Backups must be restored with the server
|
||||||
stopped.
|
stopped.
|
||||||
|
|
||||||
WebSocket messages wake HTTP replay; missed notifications are recovered by replay
|
WebSocket `vaultChanged` hints wake HTTP replay. The server periodically compares
|
||||||
and transport liveness checks. Authentication and permanent errors remain visible;
|
its history checkpoint to detect missed broadcasts; transport liveness checks
|
||||||
|
recover disconnected clients. Authentication and permanent errors remain visible;
|
||||||
transient errors retry according to client settings. Observer callback failures
|
transient errors retry according to client settings. Observer callback failures
|
||||||
are isolated from the background sync loop.
|
are isolated from the background sync loop.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,6 @@ vaultlink \
|
||||||
| `--ignore-pattern <pattern>` | - | Glob pattern to ignore (repeatable) |
|
| `--ignore-pattern <pattern>` | - | Glob pattern to ignore (repeatable) |
|
||||||
| `--websocket-retry-interval-ms <ms>` | `3500` | WebSocket reconnection interval |
|
| `--websocket-retry-interval-ms <ms>` | `3500` | WebSocket reconnection interval |
|
||||||
| `--log-level <level>` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
|
| `--log-level <level>` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR |
|
||||||
| `--line-endings <mode>` | `auto` | Line ending style: auto, lf, crlf |
|
|
||||||
| `-q, --quiet` | - | Suppress startup banner for non-interactive use |
|
| `-q, --quiet` | - | Suppress startup banner for non-interactive use |
|
||||||
| `-h, --help` | - | Show help |
|
| `-h, --help` | - | Show help |
|
||||||
| `-V, --version` | - | Show version |
|
| `-V, --version` | - | Show version |
|
||||||
|
|
@ -96,12 +95,7 @@ vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \
|
||||||
--log-level DEBUG --quiet
|
--log-level DEBUG --quiet
|
||||||
```
|
```
|
||||||
|
|
||||||
Force LF line endings (useful for cross-platform vaults):
|
File bytes, including line endings, are preserved on every platform.
|
||||||
|
|
||||||
```bash
|
|
||||||
vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \
|
|
||||||
--line-endings lf
|
|
||||||
```
|
|
||||||
|
|
||||||
## Docker Deployment
|
## Docker Deployment
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,9 +2,6 @@ import { Command, Option } from "commander";
|
||||||
import packageJson from "../package.json";
|
import packageJson from "../package.json";
|
||||||
import { LogLevel } from "sync-client";
|
import { LogLevel } from "sync-client";
|
||||||
|
|
||||||
export const LINE_ENDING_MODES = ["auto", "lf", "crlf"] as const;
|
|
||||||
export type LineEndingMode = (typeof LINE_ENDING_MODES)[number];
|
|
||||||
|
|
||||||
interface CliArgs {
|
interface CliArgs {
|
||||||
remoteUri: string;
|
remoteUri: string;
|
||||||
token: string;
|
token: string;
|
||||||
|
|
@ -17,7 +14,6 @@ interface CliArgs {
|
||||||
health?: string;
|
health?: string;
|
||||||
enableTelemetry?: boolean;
|
enableTelemetry?: boolean;
|
||||||
quiet: boolean;
|
quiet: boolean;
|
||||||
lineEndings: LineEndingMode;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"];
|
const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"];
|
||||||
|
|
@ -131,15 +127,6 @@ export function parseArgs(argv: string[]): CliArgs {
|
||||||
"[OPTIONAL] Suppress startup banner for non-interactive use"
|
"[OPTIONAL] Suppress startup banner for non-interactive use"
|
||||||
).env("VAULTLINK_QUIET")
|
).env("VAULTLINK_QUIET")
|
||||||
)
|
)
|
||||||
.addOption(
|
|
||||||
new Option(
|
|
||||||
"--line-endings <mode>",
|
|
||||||
"[OPTIONAL] Line ending style: auto (platform default), lf, crlf"
|
|
||||||
)
|
|
||||||
.default("auto")
|
|
||||||
.choices([...LINE_ENDING_MODES])
|
|
||||||
.env("VAULTLINK_LINE_ENDINGS")
|
|
||||||
)
|
|
||||||
.addHelpText(
|
.addHelpText(
|
||||||
"after",
|
"after",
|
||||||
`
|
`
|
||||||
|
|
@ -173,7 +160,6 @@ Environment variables:
|
||||||
const health = opts.health as string | undefined;
|
const health = opts.health as string | undefined;
|
||||||
const enableTelemetry = opts.enableTelemetry as boolean | undefined;
|
const enableTelemetry = opts.enableTelemetry as boolean | undefined;
|
||||||
const quiet = (opts.quiet as boolean | undefined) ?? false;
|
const quiet = (opts.quiet as boolean | undefined) ?? false;
|
||||||
const lineEndingsStr = (opts.lineEndings as string | undefined) ?? "auto";
|
|
||||||
/* eslint-enable @typescript-eslint/no-unsafe-type-assertion */
|
/* eslint-enable @typescript-eslint/no-unsafe-type-assertion */
|
||||||
|
|
||||||
const requiredLocalPath = requireOption(localPath, "localPath");
|
const requiredLocalPath = requireOption(localPath, "localPath");
|
||||||
|
|
@ -203,15 +189,6 @@ Environment variables:
|
||||||
}
|
}
|
||||||
const logLevel = logLevelUpper;
|
const logLevel = logLevelUpper;
|
||||||
|
|
||||||
const isLineEndingMode = (value: string): value is LineEndingMode =>
|
|
||||||
(LINE_ENDING_MODES as readonly string[]).includes(value);
|
|
||||||
if (!isLineEndingMode(lineEndingsStr)) {
|
|
||||||
throw new Error(
|
|
||||||
`Invalid line endings mode '${lineEndingsStr}'. Valid values are: ${LINE_ENDING_MODES.join(", ")}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const lineEndings = lineEndingsStr;
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
localPath: requiredLocalPath,
|
localPath: requiredLocalPath,
|
||||||
remoteUri: requiredRemoteUri,
|
remoteUri: requiredRemoteUri,
|
||||||
|
|
@ -223,7 +200,6 @@ Environment variables:
|
||||||
logLevel,
|
logLevel,
|
||||||
health,
|
health,
|
||||||
enableTelemetry,
|
enableTelemetry,
|
||||||
quiet,
|
quiet
|
||||||
lineEndings
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,7 @@
|
||||||
import * as path from "path";
|
import * as path from "path";
|
||||||
import * as fs from "fs/promises";
|
import * as fs from "fs/promises";
|
||||||
import * as fsSync from "fs";
|
import * as fsSync from "fs";
|
||||||
import type { NetworkConnectionStatus ,
|
import type { NetworkConnectionStatus, Logger } from "sync-client";
|
||||||
Logger} from "sync-client";
|
|
||||||
import {
|
import {
|
||||||
SyncClient,
|
SyncClient,
|
||||||
DEFAULT_SETTINGS,
|
DEFAULT_SETTINGS,
|
||||||
|
|
@ -11,7 +10,7 @@ import {
|
||||||
type SyncSettings,
|
type SyncSettings,
|
||||||
type StoredDatabase
|
type StoredDatabase
|
||||||
} from "sync-client";
|
} from "sync-client";
|
||||||
import { parseArgs, type LineEndingMode } from "./args";
|
import { parseArgs } from "./args";
|
||||||
import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem";
|
import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem";
|
||||||
import { FileWatcher } from "./file-watcher";
|
import { FileWatcher } from "./file-watcher";
|
||||||
import { formatLogLine } from "./logger-formatter";
|
import { formatLogLine } from "./logger-formatter";
|
||||||
|
|
@ -41,7 +40,6 @@ const LOG_LEVEL_ORDER = {
|
||||||
function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void {
|
function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void {
|
||||||
return (logLine: LogLine): void => {
|
return (logLine: LogLine): void => {
|
||||||
if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) {
|
if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) {
|
||||||
|
|
||||||
console.log(formatLogLine(logLine));
|
console.log(formatLogLine(logLine));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
@ -50,17 +48,6 @@ function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void {
|
||||||
const HEALTH_CHECK_INTERVAL_MS = 30 * 1000;
|
const HEALTH_CHECK_INTERVAL_MS = 30 * 1000;
|
||||||
const PROGRESS_LOG_INTERVAL_MS = 2000;
|
const PROGRESS_LOG_INTERVAL_MS = 2000;
|
||||||
|
|
||||||
function resolveLineEndings(mode: LineEndingMode): string {
|
|
||||||
switch (mode) {
|
|
||||||
case "lf":
|
|
||||||
return "\n";
|
|
||||||
case "crlf":
|
|
||||||
return "\r\n";
|
|
||||||
case "auto":
|
|
||||||
return process.platform === "win32" ? "\r\n" : "\n";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
async function main(): Promise<void> {
|
||||||
const args = parseArgs(process.argv);
|
const args = parseArgs(process.argv);
|
||||||
const absolutePath = path.resolve(args.localPath);
|
const absolutePath = path.resolve(args.localPath);
|
||||||
|
|
@ -96,12 +83,6 @@ async function main(): Promise<void> {
|
||||||
emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`);
|
emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`);
|
||||||
emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`);
|
emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`);
|
||||||
emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`);
|
emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`);
|
||||||
if (args.lineEndings !== "auto") {
|
|
||||||
emitBoot(
|
|
||||||
LogLevel.INFO,
|
|
||||||
`Line endings: ${args.lineEndings.toUpperCase()}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const dataDir = path.join(absolutePath, VAULTLINK_DIR);
|
const dataDir = path.join(absolutePath, VAULTLINK_DIR);
|
||||||
|
|
@ -158,8 +139,7 @@ async function main(): Promise<void> {
|
||||||
JSON.stringify(persistedDatabase, null, 2)
|
JSON.stringify(persistedDatabase, null, 2)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
nativeLineEndings: resolveLineEndings(args.lineEndings)
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (args.health !== undefined) {
|
if (args.health !== undefined) {
|
||||||
|
|
@ -279,7 +259,6 @@ async function main(): Promise<void> {
|
||||||
}
|
}
|
||||||
|
|
||||||
main().catch((error: unknown) => {
|
main().catch((error: unknown) => {
|
||||||
|
|
||||||
console.error(
|
console.error(
|
||||||
`Unexpected error: ${error instanceof Error ? error.message : String(error)}`
|
`Unexpected error: ${error instanceof Error ? error.message : String(error)}`
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import type {
|
||||||
TAbstractFile,
|
TAbstractFile,
|
||||||
WorkspaceLeaf
|
WorkspaceLeaf
|
||||||
} from "obsidian";
|
} from "obsidian";
|
||||||
import { Notice, Platform, Plugin, TFile } from "obsidian";
|
import { Notice, Plugin, TFile } from "obsidian";
|
||||||
import "../manifest.json";
|
import "../manifest.json";
|
||||||
import { HistoryView } from "./views/history/history-view";
|
import { HistoryView } from "./views/history/history-view";
|
||||||
import { StatusBar } from "./views/status-bar/status-bar";
|
import { StatusBar } from "./views/status-bar/status-bar";
|
||||||
|
|
@ -132,7 +132,6 @@ export default class VaultLinkPlugin extends Plugin {
|
||||||
load: this.loadData.bind(this),
|
load: this.loadData.bind(this),
|
||||||
save: this.saveData.bind(this)
|
save: this.saveData.bind(this)
|
||||||
},
|
},
|
||||||
nativeLineEndings: Platform.isWin ? "\r\n" : "\n",
|
|
||||||
...(IS_DEBUG_BUILD
|
...(IS_DEBUG_BUILD
|
||||||
? {
|
? {
|
||||||
fetch: debugging.slowFetchFactory(1),
|
fetch: debugging.slowFetchFactory(1),
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
# Sync client (API v5)
|
# Sync client (API v5)
|
||||||
|
|
||||||
The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket
|
The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket
|
||||||
messages wake it; complete disk scans and the durable server event log are the
|
`vaultChanged` hints wake it; complete disk scans and the durable server event log are the
|
||||||
sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a
|
sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a
|
||||||
positive interval to repair missed notifications. Unset or `0` disables polling.
|
positive interval to repair missed notifications. Unset or `0` disables polling.
|
||||||
Create, delete and move notifications persist logical identity changes before
|
Create, delete and move notifications persist logical identity changes before
|
||||||
|
|
@ -13,6 +13,8 @@ as content updates. This distinguishes a deliberate delete/recreate from an edit
|
||||||
Use `waitUntilFinished()` to await the active attempt; errors leave
|
Use `waitUntilFinished()` to await the active attempt; errors leave
|
||||||
pending requests saved and are reported to the caller. Transient failures retry;
|
pending requests saved and are reported to the caller. Transient failures retry;
|
||||||
authentication and protocol errors wait for an explicit wake or settings change.
|
authentication and protocol errors wait for an explicit wake or settings change.
|
||||||
|
Pausing or destroying the client aborts its HTTP session, including body reads;
|
||||||
|
resuming starts a fresh session. Connection checks can run while sync is disabled.
|
||||||
|
|
||||||
## State and policy
|
## State and policy
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,5 +2,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
|
||||||
export const DIFF_CACHE_SIZE_MB = 2;
|
export const DIFF_CACHE_SIZE_MB = 2;
|
||||||
export const MAX_LOG_MESSAGE_COUNT = 100000;
|
export const MAX_LOG_MESSAGE_COUNT = 100000;
|
||||||
export const MAX_HISTORY_ENTRY_COUNT = 5000;
|
export const MAX_HISTORY_ENTRY_COUNT = 5000;
|
||||||
export const SUPPORTED_API_VERSION = 4;
|
export const SUPPORTED_API_VERSION = 5;
|
||||||
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
|
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
|
||||||
|
|
|
||||||
|
|
@ -57,3 +57,33 @@ it("owns settings arrays instead of sharing them with callers and listeners", as
|
||||||
await settings.setSettings({ maxFileSizeMB: 12 });
|
await settings.setSettings({ maxFileSizeMB: 12 });
|
||||||
assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]);
|
assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("file eligibility follows successfully saved settings", async () => {
|
||||||
|
let fail = false;
|
||||||
|
const settings = new Settings(
|
||||||
|
new Logger(),
|
||||||
|
{
|
||||||
|
ignorePatterns: ["private/**"],
|
||||||
|
maxFileSizeMB: 1
|
||||||
|
},
|
||||||
|
async () => {
|
||||||
|
if (fail) throw new Error("save failed");
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert(settings.isIgnored("private/note.md"));
|
||||||
|
assert(settings.isIgnored(".vault-link-sync/state.json"));
|
||||||
|
assert(!settings.isIgnored("public/note.md"));
|
||||||
|
assert(!settings.isOversized(1024 * 1024));
|
||||||
|
assert(settings.isOversized(1024 * 1024 + 1));
|
||||||
|
fail = true;
|
||||||
|
await assert.rejects(
|
||||||
|
settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 }),
|
||||||
|
/save failed/
|
||||||
|
);
|
||||||
|
assert(settings.isIgnored("private/note.md"));
|
||||||
|
assert(settings.isOversized(1024 * 1024 + 1));
|
||||||
|
fail = false;
|
||||||
|
await settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 });
|
||||||
|
assert(!settings.isIgnored("private/note.md"));
|
||||||
|
assert(!settings.isOversized(1024 * 1024 + 1));
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,6 @@
|
||||||
import type { Logger } from "../tracing/logger";
|
import type { Logger } from "../tracing/logger";
|
||||||
|
import { globsToRegexes } from "../utils/globs-to-regexes";
|
||||||
|
import { isInternalPath } from "../utils/portable-path";
|
||||||
import { Lock } from "../utils/data-structures/locks";
|
import { Lock } from "../utils/data-structures/locks";
|
||||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
|
|
||||||
|
|
@ -43,6 +45,7 @@ export class Settings {
|
||||||
>();
|
>();
|
||||||
|
|
||||||
private settings: SyncSettings;
|
private settings: SyncSettings;
|
||||||
|
private ignorePatterns: RegExp[];
|
||||||
private readonly lock: Lock = new Lock();
|
private readonly lock: Lock = new Lock();
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
|
|
@ -55,11 +58,27 @@ export class Settings {
|
||||||
...structuredClone(initialState ?? {})
|
...structuredClone(initialState ?? {})
|
||||||
};
|
};
|
||||||
|
|
||||||
|
this.ignorePatterns = globsToRegexes(
|
||||||
|
this.settings.ignorePatterns,
|
||||||
|
logger
|
||||||
|
);
|
||||||
|
|
||||||
this.logger.debug(
|
this.logger.debug(
|
||||||
`Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}`
|
`Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public isIgnored(path: string): boolean {
|
||||||
|
return (
|
||||||
|
isInternalPath(path) ||
|
||||||
|
this.ignorePatterns.some((pattern) => pattern.test(path))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public isOversized(size: number): boolean {
|
||||||
|
return size > this.settings.maxFileSizeMB * 1024 * 1024;
|
||||||
|
}
|
||||||
|
|
||||||
public getSettings(): SyncSettings {
|
public getSettings(): SyncSettings {
|
||||||
return {
|
return {
|
||||||
...this.settings,
|
...this.settings,
|
||||||
|
|
@ -91,6 +110,11 @@ export class Settings {
|
||||||
const next = { ...this.settings, ...update };
|
const next = { ...this.settings, ...update };
|
||||||
await this.saveData(next);
|
await this.saveData(next);
|
||||||
this.settings = next;
|
this.settings = next;
|
||||||
|
if (update.ignorePatterns !== undefined)
|
||||||
|
this.ignorePatterns = globsToRegexes(
|
||||||
|
next.ignorePatterns,
|
||||||
|
this.logger
|
||||||
|
);
|
||||||
|
|
||||||
return [this.getSettings(), oldSettings] as const;
|
return [this.getSettings(), oldSettings] as const;
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,177 +0,0 @@
|
||||||
import type { Mock } from "node:test";
|
|
||||||
import { describe, it, mock, beforeEach, afterEach } from "node:test";
|
|
||||||
import assert from "node:assert";
|
|
||||||
import { FetchController } from "./fetch-controller";
|
|
||||||
import { Logger } from "../tracing/logger";
|
|
||||||
import { SyncResetError } from "../errors/errors";
|
|
||||||
import { sleep } from "../utils/sleep";
|
|
||||||
|
|
||||||
describe("FetchController", () => {
|
|
||||||
const createMockFetch = (
|
|
||||||
shouldSleep: boolean
|
|
||||||
): Mock<() => Promise<Response>> =>
|
|
||||||
mock.fn(async () => {
|
|
||||||
if (shouldSleep) {
|
|
||||||
await sleep(30);
|
|
||||||
}
|
|
||||||
return Promise.resolve(new Response("OK", { status: 200 }));
|
|
||||||
});
|
|
||||||
|
|
||||||
beforeEach(() => {
|
|
||||||
mock.timers.enable({ apis: ["setTimeout"] });
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(() => {
|
|
||||||
mock.timers.reset();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should allow fetch when canFetch is true", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(true, logger);
|
|
||||||
const mockFetch = createMockFetch(false);
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
const response = await controlledFetch("http://example.com");
|
|
||||||
|
|
||||||
assert.strictEqual(await response.text(), "OK");
|
|
||||||
assert.strictEqual(mockFetch.mock.calls.length, 1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should block fetch until canFetch becomes true", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(false, logger);
|
|
||||||
const mockFetch = createMockFetch(true);
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
const fetchPromise = controlledFetch("http://example.com");
|
|
||||||
assert.strictEqual(mockFetch.mock.calls.length, 0);
|
|
||||||
|
|
||||||
controller.canFetch = true;
|
|
||||||
await Promise.resolve();
|
|
||||||
mock.timers.tick(30);
|
|
||||||
|
|
||||||
const response = await fetchPromise;
|
|
||||||
assert.strictEqual(await response.text(), "OK");
|
|
||||||
assert.strictEqual(mockFetch.mock.calls.length, 1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should reject during reset", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(true, logger);
|
|
||||||
const mockFetch = createMockFetch(true);
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
const firstRequest = controlledFetch("http://example.com");
|
|
||||||
assert.strictEqual(mockFetch.mock.calls.length, 1);
|
|
||||||
|
|
||||||
controller.startReset();
|
|
||||||
|
|
||||||
const secondRequest = controlledFetch("http://example.com");
|
|
||||||
|
|
||||||
await assert.rejects(
|
|
||||||
firstRequest,
|
|
||||||
(error: unknown) => error instanceof SyncResetError
|
|
||||||
);
|
|
||||||
await assert.rejects(
|
|
||||||
secondRequest,
|
|
||||||
(error: unknown) => error instanceof SyncResetError
|
|
||||||
);
|
|
||||||
assert.strictEqual(mockFetch.mock.calls.length, 1);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should allow fetch after reset finishes", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(true, logger);
|
|
||||||
const mockFetch = createMockFetch(false);
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
controller.startReset();
|
|
||||||
controller.finishReset();
|
|
||||||
|
|
||||||
const response = await controlledFetch("http://example.com");
|
|
||||||
assert.strictEqual(await response.text(), "OK");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should defer canFetch changes during reset", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(false, logger);
|
|
||||||
const mockFetch = createMockFetch(true);
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
controller.startReset();
|
|
||||||
controller.canFetch = true;
|
|
||||||
|
|
||||||
await assert.rejects(
|
|
||||||
async () => controlledFetch("http://example.com"),
|
|
||||||
(error: unknown) => error instanceof SyncResetError
|
|
||||||
);
|
|
||||||
|
|
||||||
controller.finishReset();
|
|
||||||
|
|
||||||
const fetchPromise = controlledFetch("http://example.com");
|
|
||||||
mock.timers.tick(30);
|
|
||||||
|
|
||||||
const response = await fetchPromise;
|
|
||||||
assert.strictEqual(await response.text(), "OK");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should handle different input types", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(true, logger);
|
|
||||||
const mockFetch = createMockFetch(false);
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
await controlledFetch("http://example.com");
|
|
||||||
await controlledFetch(new URL("http://example.com"));
|
|
||||||
await controlledFetch(
|
|
||||||
new Request("http://example.com", { method: "POST" })
|
|
||||||
);
|
|
||||||
|
|
||||||
assert.strictEqual(mockFetch.mock.calls.length, 3);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should handle fetch errors", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(true, logger);
|
|
||||||
const mockFetch = mock.fn(async () => {
|
|
||||||
throw new Error("Network error");
|
|
||||||
});
|
|
||||||
const controlledFetch = controller.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
mockFetch
|
|
||||||
);
|
|
||||||
|
|
||||||
await assert.rejects(
|
|
||||||
async () => controlledFetch("http://example.com"),
|
|
||||||
(error: unknown) =>
|
|
||||||
error instanceof Error && error.message === "Network error"
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should not create unhandled rejection on reset with no waiting fetches", async () => {
|
|
||||||
const logger = new Logger();
|
|
||||||
const controller = new FetchController(true, logger);
|
|
||||||
|
|
||||||
controller.startReset();
|
|
||||||
mock.timers.tick(10);
|
|
||||||
controller.finishReset();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,149 +0,0 @@
|
||||||
import type { Logger } from "../tracing/logger";
|
|
||||||
import { createPromise } from "../utils/create-promise";
|
|
||||||
import { SyncResetError } from "../errors/errors";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Offers a resettable fetch implementation that waits until syncing is enabled
|
|
||||||
* and aborts outstanding requests when a reset is started.
|
|
||||||
*/
|
|
||||||
export class FetchController {
|
|
||||||
private static readonly UNTIL_RESOLUTION = Symbol();
|
|
||||||
|
|
||||||
private isResetting = false;
|
|
||||||
|
|
||||||
// Promise resolves on the next state change: sync enabled/disabled or reset started/ended
|
|
||||||
private until: Promise<symbol>;
|
|
||||||
private resolveUntil: (result: symbol) => unknown;
|
|
||||||
private rejectUntil: (reason: unknown) => unknown;
|
|
||||||
|
|
||||||
public constructor(
|
|
||||||
private _canFetch: boolean,
|
|
||||||
private readonly logger: Logger
|
|
||||||
) {
|
|
||||||
[this.until, this.resolveUntil, this.rejectUntil] =
|
|
||||||
createPromise<symbol>();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Whether the fetch implementation can immediately send requests once outside of a reset.
|
|
||||||
*/
|
|
||||||
public get canFetch(): boolean {
|
|
||||||
return this._canFetch;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Allow or disallow fetching. The changes only take effect if not resetting.
|
|
||||||
* When called during a reset, its effect is deferred until the reset is finished.
|
|
||||||
*
|
|
||||||
* @param canFetch Whether fetching is enabled
|
|
||||||
*/
|
|
||||||
public set canFetch(canFetch: boolean) {
|
|
||||||
this._canFetch = canFetch;
|
|
||||||
|
|
||||||
if (!this.isResetting) {
|
|
||||||
const previousResolve = this.resolveUntil;
|
|
||||||
[this.until, this.resolveUntil, this.rejectUntil] =
|
|
||||||
createPromise<symbol>();
|
|
||||||
previousResolve(FetchController.UNTIL_RESOLUTION);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static getUrlFromInput(input: RequestInfo | URL): string {
|
|
||||||
if (input instanceof URL) {
|
|
||||||
return input.href;
|
|
||||||
}
|
|
||||||
if (typeof input === "string") {
|
|
||||||
return input;
|
|
||||||
}
|
|
||||||
return input.url;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Starts a reset, causing all ongoing and future fetches to be rejected
|
|
||||||
* with a SyncResetError until finishReset is called.
|
|
||||||
*/
|
|
||||||
public startReset(): void {
|
|
||||||
this.isResetting = true;
|
|
||||||
this.rejectUntil(new SyncResetError());
|
|
||||||
// Catch unhandled rejection if no fetches are waiting
|
|
||||||
this.until.catch(() => {
|
|
||||||
// Intentionally ignore - this rejection is handled by waiting fetches
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Finishes a reset, allowing fetches to proceed or wait again depending on
|
|
||||||
* the current sync settings.
|
|
||||||
*/
|
|
||||||
public finishReset(): void {
|
|
||||||
if (!this.isResetting) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
this.isResetting = false;
|
|
||||||
[this.until, this.resolveUntil, this.rejectUntil] = createPromise();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
*
|
|
||||||
* |------------------|---------------|-----------------------------------------------------|
|
|
||||||
* | | Sync enabled | Sync disabled |
|
|
||||||
* |------------------|-------------- |-----------------------------------------------------|
|
|
||||||
* | During reset | Rejects with SyncResetError without sending request |
|
|
||||||
* |------------------|-------------- |-----------------------------------------------------|
|
|
||||||
* | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch |
|
|
||||||
* |------------------|---------------|-----------------------------------------------------|
|
|
||||||
*
|
|
||||||
* @param logger for errors
|
|
||||||
* @param fetch to wrap
|
|
||||||
* @returns a wrapped fetch implementation affected by the FetchController state
|
|
||||||
*/
|
|
||||||
public getControlledFetchImplementation(
|
|
||||||
logger: Logger,
|
|
||||||
fetch: typeof globalThis.fetch = globalThis.fetch
|
|
||||||
): typeof globalThis.fetch {
|
|
||||||
return async (
|
|
||||||
input: RequestInfo | URL,
|
|
||||||
init?: RequestInit
|
|
||||||
): Promise<Response> => {
|
|
||||||
while (!this.canFetch || this.isResetting) {
|
|
||||||
await this.until;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21
|
|
||||||
const _input =
|
|
||||||
typeof Request !== "undefined" && input instanceof Request
|
|
||||||
? input.clone()
|
|
||||||
: input;
|
|
||||||
|
|
||||||
const fetchPromise = fetch(_input, init);
|
|
||||||
|
|
||||||
// We only want to catch rejections from `this.until`
|
|
||||||
let result: symbol | Response | undefined = undefined;
|
|
||||||
do {
|
|
||||||
result = await Promise.race([this.until, fetchPromise]);
|
|
||||||
} while (result === FetchController.UNTIL_RESOLUTION);
|
|
||||||
|
|
||||||
const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
|
|
||||||
|
|
||||||
if (!fetchResult.ok) {
|
|
||||||
this.logger.warn(
|
|
||||||
`Fetch for ${FetchController.getUrlFromInput(
|
|
||||||
input
|
|
||||||
)}, got status ${fetchResult.status}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return fetchResult;
|
|
||||||
} catch (error) {
|
|
||||||
logger.warn(
|
|
||||||
`Fetch for ${FetchController.getUrlFromInput(
|
|
||||||
input
|
|
||||||
)}, got error: ${error}`
|
|
||||||
);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
146
frontend/sync-client/src/services/sync-service.test.ts
Normal file
146
frontend/sync-client/src/services/sync-service.test.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { test } from "node:test";
|
||||||
|
import { Settings } from "../persistence/settings";
|
||||||
|
import { Logger } from "../tracing/logger";
|
||||||
|
import { SyncResetError } from "../errors/errors";
|
||||||
|
import { SyncService } from "./sync-service";
|
||||||
|
|
||||||
|
const settings = () =>
|
||||||
|
new Settings(
|
||||||
|
new Logger(),
|
||||||
|
{
|
||||||
|
remoteUri: "http://test",
|
||||||
|
vaultName: "test",
|
||||||
|
requestTimeoutMs: 1000
|
||||||
|
},
|
||||||
|
async () => {}
|
||||||
|
);
|
||||||
|
|
||||||
|
test("paused sync sends no requests, while connection checks still work", async () => {
|
||||||
|
let requests = 0;
|
||||||
|
const service = new SyncService("device", settings(), async () => {
|
||||||
|
requests++;
|
||||||
|
return Response.json({ headEventId: 0, events: [] });
|
||||||
|
});
|
||||||
|
await assert.rejects(service.events(0), SyncResetError);
|
||||||
|
assert.equal(requests, 0);
|
||||||
|
await service.ping();
|
||||||
|
assert.equal(requests, 1);
|
||||||
|
service.resume();
|
||||||
|
assert.equal((await service.events(0)).headEventId, 0);
|
||||||
|
assert.equal(requests, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("pause aborts requests and fences late responses from an injected fetch", async () => {
|
||||||
|
const response = Promise.withResolvers<Response>();
|
||||||
|
const signals: AbortSignal[] = [];
|
||||||
|
const saved: (string | undefined)[] = [];
|
||||||
|
const service = new SyncService(
|
||||||
|
"device",
|
||||||
|
settings(),
|
||||||
|
async (_, init) => {
|
||||||
|
signals.push(init!.signal!);
|
||||||
|
return signals.length === 1
|
||||||
|
? response.promise
|
||||||
|
: Response.json({ headEventId: 0, events: [] });
|
||||||
|
},
|
||||||
|
{
|
||||||
|
get: () => undefined,
|
||||||
|
save: async (value) => {
|
||||||
|
saved.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
service.resume();
|
||||||
|
const first = service.events(0);
|
||||||
|
service.pause();
|
||||||
|
await assert.rejects(first, SyncResetError);
|
||||||
|
assert.equal(signals[0].aborted, true);
|
||||||
|
service.resume();
|
||||||
|
await service.events(0);
|
||||||
|
assert.equal(signals[1].aborted, false);
|
||||||
|
response.resolve(
|
||||||
|
Response.json(
|
||||||
|
{},
|
||||||
|
{ headers: { "X-Vault-Link-History": "99:abandoned" } }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
assert.deepEqual(saved, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const binary of [false, true]) {
|
||||||
|
test(`pause interrupts a stalled ${binary ? "binary" : "JSON"} response body`, async () => {
|
||||||
|
const headersSaved = Promise.withResolvers<void>();
|
||||||
|
let body!: ReadableStreamDefaultController<Uint8Array>;
|
||||||
|
const service = new SyncService(
|
||||||
|
"device",
|
||||||
|
settings(),
|
||||||
|
async () =>
|
||||||
|
new Response(
|
||||||
|
new ReadableStream({
|
||||||
|
start(controller) {
|
||||||
|
body = controller;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
{ headers: { "X-Vault-Link-History": "1:head" } }
|
||||||
|
),
|
||||||
|
{
|
||||||
|
get: () => undefined,
|
||||||
|
save: async () => {
|
||||||
|
headersSaved.resolve();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
service.resume();
|
||||||
|
const request = binary
|
||||||
|
? service.getDocumentVersionContent({
|
||||||
|
documentId: "a",
|
||||||
|
vaultUpdateId: 1
|
||||||
|
})
|
||||||
|
: service.events(0);
|
||||||
|
await headersSaved.promise;
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
service.pause();
|
||||||
|
await assert.rejects(request, SyncResetError);
|
||||||
|
body.enqueue(new TextEncoder().encode("{}"));
|
||||||
|
body.close();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test("pause waits for a checkpoint save already in progress", async () => {
|
||||||
|
const saving = Promise.withResolvers<void>();
|
||||||
|
const release = Promise.withResolvers<void>();
|
||||||
|
let settled = false;
|
||||||
|
let checkpoint: string | undefined;
|
||||||
|
const service = new SyncService(
|
||||||
|
"device",
|
||||||
|
settings(),
|
||||||
|
async () =>
|
||||||
|
Response.json(
|
||||||
|
{},
|
||||||
|
{
|
||||||
|
headers: { "X-Vault-Link-History": "1:head" }
|
||||||
|
}
|
||||||
|
),
|
||||||
|
{
|
||||||
|
get: () => checkpoint,
|
||||||
|
save: async (value) => {
|
||||||
|
saving.resolve();
|
||||||
|
await release.promise;
|
||||||
|
checkpoint = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
service.resume();
|
||||||
|
const request = service.events(0).finally(() => {
|
||||||
|
settled = true;
|
||||||
|
});
|
||||||
|
await saving.promise;
|
||||||
|
service.pause();
|
||||||
|
await new Promise((resolve) => setImmediate(resolve));
|
||||||
|
assert.equal(settled, false);
|
||||||
|
release.resolve();
|
||||||
|
await assert.rejects(request, SyncResetError);
|
||||||
|
assert.equal(checkpoint, "1:head");
|
||||||
|
});
|
||||||
|
|
@ -1,11 +1,7 @@
|
||||||
import type { Settings } from "../persistence/settings";
|
import type { Settings } from "../persistence/settings";
|
||||||
import type { FetchController } from "./fetch-controller";
|
|
||||||
import type { Logger } from "../tracing/logger";
|
|
||||||
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
|
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
|
||||||
import type { DocumentVersion } from "./types/DocumentVersion";
|
|
||||||
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
|
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
|
||||||
import type { EventBatch } from "./types/EventBatch";
|
import type { EventBatch } from "./types/EventBatch";
|
||||||
import type { FileManifest } from "./types/FileManifest";
|
|
||||||
import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse";
|
import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse";
|
||||||
import type { PutFileContent } from "./types/PutFileContent";
|
import type { PutFileContent } from "./types/PutFileContent";
|
||||||
import type { PushFileManifest } from "./types/PushFileManifest";
|
import type { PushFileManifest } from "./types/PushFileManifest";
|
||||||
|
|
@ -14,29 +10,47 @@ import type { PingResponse } from "./types/PingResponse";
|
||||||
import {
|
import {
|
||||||
AuthenticationError,
|
AuthenticationError,
|
||||||
PermanentSyncError,
|
PermanentSyncError,
|
||||||
ServerHistoryChangedError
|
ServerHistoryChangedError,
|
||||||
|
SyncResetError
|
||||||
} from "../errors/errors";
|
} from "../errors/errors";
|
||||||
|
|
||||||
|
// Injected fetch implementations may ignore cancellation. Stop waiting anyway,
|
||||||
|
// and never let their late responses reach checkpoint or engine persistence.
|
||||||
|
async function abortable<T>(
|
||||||
|
signal: AbortSignal,
|
||||||
|
operation: () => Promise<T>
|
||||||
|
): Promise<T> {
|
||||||
|
signal.throwIfAborted();
|
||||||
|
let abort = (): void => undefined;
|
||||||
|
try {
|
||||||
|
return await Promise.race([
|
||||||
|
new Promise<never>((_, reject) => {
|
||||||
|
abort = (): void => {
|
||||||
|
// eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- Only reset errors and timeout errors reach this helper.
|
||||||
|
reject(signal.reason);
|
||||||
|
};
|
||||||
|
signal.addEventListener("abort", abort, { once: true });
|
||||||
|
}),
|
||||||
|
operation()
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
signal.removeEventListener("abort", abort);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class SyncService {
|
export class SyncService {
|
||||||
private readonly client: typeof fetch;
|
private session = new AbortController();
|
||||||
private readonly rawFetch: typeof fetch;
|
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
private readonly deviceId: string,
|
private readonly deviceId: string,
|
||||||
fetchController: FetchController,
|
|
||||||
private readonly settings: Settings,
|
private readonly settings: Settings,
|
||||||
logger: Logger,
|
private readonly fetchImplementation: typeof fetch = globalThis.fetch,
|
||||||
fetchImplementation: typeof fetch = globalThis.fetch,
|
|
||||||
private readonly history?: {
|
private readonly history?: {
|
||||||
get: () => string | undefined;
|
get: () => string | undefined;
|
||||||
save: (checkpoint: string | undefined) => Promise<void>;
|
save: (checkpoint: string | undefined) => Promise<void>;
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
this.rawFetch = async (...args) => fetchImplementation(...args);
|
this.pause();
|
||||||
this.client = fetchController.getControlledFetchImplementation(
|
|
||||||
logger,
|
|
||||||
this.rawFetch
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public get verifiesHistory(): boolean {
|
public get verifiesHistory(): boolean {
|
||||||
|
|
@ -46,21 +60,80 @@ export class SyncService {
|
||||||
return this.history?.get() !== undefined;
|
return this.history?.get() !== undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public resume(): void {
|
||||||
|
this.session = new AbortController();
|
||||||
|
}
|
||||||
|
|
||||||
|
public pause(): void {
|
||||||
|
this.session.abort(new SyncResetError());
|
||||||
|
}
|
||||||
|
|
||||||
public async resetHistory(): Promise<void> {
|
public async resetHistory(): Promise<void> {
|
||||||
await this.history?.save(undefined);
|
await this.history?.save(undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async ping(): Promise<PingResponse> {
|
||||||
|
return this.request("/ping", undefined, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async vaultSnapshot(): Promise<VaultSnapshot> {
|
||||||
|
return this.request("/vault-snapshot");
|
||||||
|
}
|
||||||
|
|
||||||
|
public async events(after: number): Promise<EventBatch> {
|
||||||
|
return this.request(`/events-since?after=${after}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async pushFileManifest(
|
||||||
|
request: PushFileManifest
|
||||||
|
): Promise<FileManifestUpdateResponse> {
|
||||||
|
return this.request("/file-manifest", request);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async putFileContent(
|
||||||
|
id: string,
|
||||||
|
request: PutFileContent
|
||||||
|
): Promise<DocumentUpdateResponse> {
|
||||||
|
return this.request(`/documents/${id}`, request);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async metadata(id: string): Promise<DocumentVersionWithoutContent> {
|
||||||
|
return this.request(`/documents/${id}/metadata`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async getDocumentVersionContent({
|
||||||
|
documentId,
|
||||||
|
vaultUpdateId
|
||||||
|
}: {
|
||||||
|
documentId: string;
|
||||||
|
vaultUpdateId: number;
|
||||||
|
}): Promise<Uint8Array> {
|
||||||
|
return this.request(
|
||||||
|
`/documents/${documentId}/versions/${vaultUpdateId}/content`,
|
||||||
|
undefined,
|
||||||
|
false,
|
||||||
|
async (response) => new Uint8Array(await response.arrayBuffer())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
private historyHeaders(): Record<string, string> {
|
private historyHeaders(): Record<string, string> {
|
||||||
const checkpoint = this.history?.get();
|
const checkpoint = this.history?.get();
|
||||||
return checkpoint ? { "X-Vault-Link-History": checkpoint } : {};
|
return checkpoint !== undefined && checkpoint !== ""
|
||||||
|
? { "X-Vault-Link-History": checkpoint }
|
||||||
|
: {};
|
||||||
}
|
}
|
||||||
|
|
||||||
private async recordHistory(response: Response): Promise<void> {
|
private async recordHistory(response: Response): Promise<void> {
|
||||||
const checkpoint = response.headers.get("x-vault-link-history");
|
const checkpoint = response.headers.get("x-vault-link-history");
|
||||||
if (checkpoint && this.history) {
|
if (
|
||||||
|
checkpoint !== null &&
|
||||||
|
checkpoint !== "" &&
|
||||||
|
this.history !== undefined
|
||||||
|
) {
|
||||||
const previous = this.history.get();
|
const previous = this.history.get();
|
||||||
if (
|
if (
|
||||||
!previous ||
|
previous === undefined ||
|
||||||
|
previous === "" ||
|
||||||
Number(checkpoint.split(":")[0]) >=
|
Number(checkpoint.split(":")[0]) >=
|
||||||
Number(previous.split(":")[0])
|
Number(previous.split(":")[0])
|
||||||
)
|
)
|
||||||
|
|
@ -75,7 +148,7 @@ export class SyncService {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public getUrl(path: string): string {
|
private getUrl(path: string): string {
|
||||||
const { remoteUri, vaultName } = this.settings.getSettings();
|
const { remoteUri, vaultName } = this.settings.getSettings();
|
||||||
return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`;
|
return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`;
|
||||||
}
|
}
|
||||||
|
|
@ -83,35 +156,42 @@ export class SyncService {
|
||||||
private async request<T>(
|
private async request<T>(
|
||||||
path: string,
|
path: string,
|
||||||
body?: unknown,
|
body?: unknown,
|
||||||
raw = false
|
raw = false,
|
||||||
|
decode: (response: Response) => Promise<T> = async (response) =>
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Endpoint methods supply the generated protocol response type.
|
||||||
|
response.json() as Promise<T>
|
||||||
): Promise<T> {
|
): Promise<T> {
|
||||||
const response = await (raw ? this.rawFetch : this.client)(
|
const timeout = AbortSignal.timeout(
|
||||||
this.getUrl(path),
|
this.settings.getSettings().requestTimeoutMs
|
||||||
{
|
);
|
||||||
|
const signal = raw
|
||||||
|
? timeout
|
||||||
|
: AbortSignal.any([timeout, this.session.signal]);
|
||||||
|
const response = await abortable(signal, async () =>
|
||||||
|
this.fetchImplementation(this.getUrl(path), {
|
||||||
method: body === undefined ? "GET" : "PUT",
|
method: body === undefined ? "GET" : "PUT",
|
||||||
body: body === undefined ? undefined : JSON.stringify(body),
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
signal: AbortSignal.timeout(
|
signal,
|
||||||
this.settings.getSettings().requestTimeoutMs
|
|
||||||
),
|
|
||||||
headers: {
|
headers: {
|
||||||
...(raw ? {} : this.historyHeaders()),
|
...(raw ? {} : this.historyHeaders()),
|
||||||
Authorization: `Bearer ${this.settings.getSettings().token}`,
|
Authorization: `Bearer ${this.settings.getSettings().token}`,
|
||||||
"Device-Id": this.deviceId,
|
"Device-Id": this.deviceId,
|
||||||
"Content-Type": "application/json"
|
"Content-Type": "application/json"
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
);
|
);
|
||||||
|
signal.throwIfAborted();
|
||||||
this.checkHistory(response);
|
this.checkHistory(response);
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
this.throwForErrorResponse(
|
this.throwForErrorResponse(
|
||||||
response,
|
response,
|
||||||
`HTTP ${response.status}: ${await response.text()}`
|
`HTTP ${response.status}: ${await abortable(signal, async () => response.text())}`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Finish metadata saves even if the transport is paused meanwhile.
|
||||||
if (!raw) await this.recordHistory(response);
|
if (!raw) await this.recordHistory(response);
|
||||||
return (await response.json()) as T;
|
return abortable(signal, async () => decode(response));
|
||||||
}
|
}
|
||||||
|
|
||||||
private throwForErrorResponse(response: Response, message: string): never {
|
private throwForErrorResponse(response: Response, message: string): never {
|
||||||
|
|
@ -128,75 +208,4 @@ export class SyncService {
|
||||||
}
|
}
|
||||||
throw new Error(message);
|
throw new Error(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async ping(): Promise<PingResponse> {
|
|
||||||
return this.request("/ping", undefined, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async vaultSnapshot(): Promise<VaultSnapshot> {
|
|
||||||
return this.request("/vault-snapshot");
|
|
||||||
}
|
|
||||||
|
|
||||||
public async events(after: number): Promise<EventBatch> {
|
|
||||||
return this.request(`/events-since?after=${after}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async fileManifest(): Promise<FileManifest> {
|
|
||||||
return this.request("/file-manifest");
|
|
||||||
}
|
|
||||||
|
|
||||||
public async pushFileManifest(
|
|
||||||
request: PushFileManifest
|
|
||||||
): Promise<FileManifestUpdateResponse> {
|
|
||||||
return this.request("/file-manifest", request);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async putFileContent(
|
|
||||||
id: string,
|
|
||||||
request: PutFileContent
|
|
||||||
): Promise<DocumentUpdateResponse> {
|
|
||||||
return this.request(`/documents/${id}`, request);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async get(id: string): Promise<DocumentVersion> {
|
|
||||||
return this.request(`/documents/${id}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async metadata(id: string): Promise<DocumentVersionWithoutContent> {
|
|
||||||
return this.request(`/documents/${id}/metadata`);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async getDocumentVersionContent({
|
|
||||||
documentId,
|
|
||||||
vaultUpdateId
|
|
||||||
}: {
|
|
||||||
documentId: string;
|
|
||||||
vaultUpdateId: number;
|
|
||||||
}): Promise<Uint8Array> {
|
|
||||||
const response = await this.client(
|
|
||||||
this.getUrl(
|
|
||||||
`/documents/${documentId}/versions/${vaultUpdateId}/content`
|
|
||||||
),
|
|
||||||
{
|
|
||||||
signal: AbortSignal.timeout(
|
|
||||||
this.settings.getSettings().requestTimeoutMs
|
|
||||||
),
|
|
||||||
headers: {
|
|
||||||
...this.historyHeaders(),
|
|
||||||
Authorization: `Bearer ${this.settings.getSettings().token}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
);
|
|
||||||
|
|
||||||
this.checkHistory(response);
|
|
||||||
if (!response.ok) {
|
|
||||||
this.throwForErrorResponse(
|
|
||||||
response,
|
|
||||||
`Cannot fetch version ${vaultUpdateId}: HTTP ${response.status}`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await this.recordHistory(response);
|
|
||||||
return new Uint8Array(await response.arrayBuffer());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,3 @@
|
||||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||||
|
|
||||||
export type WebSocketHandshake = { token: string, deviceId: string, lastSeenVaultUpdateId: number | null, };
|
export type WebSocketHandshake = { token: string, deviceId: string, };
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,4 @@
|
||||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||||
import type { CursorPositionFromServer } from "./CursorPositionFromServer";
|
import type { CursorPositionFromServer } from "./CursorPositionFromServer";
|
||||||
import type { EventBatch } from "./EventBatch";
|
|
||||||
|
|
||||||
export type WebSocketServerMessage = { "type": "vaultEvents" } & EventBatch | { "type": "cursorPositions" } & CursorPositionFromServer;
|
export type WebSocketServerMessage = { "type": "vaultChanged" } | { "type": "cursorPositions" } & CursorPositionFromServer;
|
||||||
|
|
|
||||||
|
|
@ -136,19 +136,13 @@ describe("WebSocketManager", () => {
|
||||||
.webSocket;
|
.webSocket;
|
||||||
|
|
||||||
mockWs.simulateMessage({
|
mockWs.simulateMessage({
|
||||||
type: "vaultEvents",
|
type: "vaultChanged"
|
||||||
headEventId: 0,
|
|
||||||
events: []
|
|
||||||
});
|
});
|
||||||
mockWs.simulateMessage({
|
mockWs.simulateMessage({
|
||||||
type: "vaultEvents",
|
type: "vaultChanged"
|
||||||
headEventId: 0,
|
|
||||||
events: []
|
|
||||||
});
|
});
|
||||||
mockWs.simulateMessage({
|
mockWs.simulateMessage({
|
||||||
type: "vaultEvents",
|
type: "vaultChanged"
|
||||||
headEventId: 0,
|
|
||||||
events: []
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
|
|
@ -208,8 +202,7 @@ describe("WebSocketManager", () => {
|
||||||
manager.sendHandshakeMessage({
|
manager.sendHandshakeMessage({
|
||||||
type: "handshake",
|
type: "handshake",
|
||||||
token: "test",
|
token: "test",
|
||||||
deviceId: "test",
|
deviceId: "test"
|
||||||
lastSeenVaultUpdateId: null
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -317,9 +310,7 @@ describe("WebSocketManager", () => {
|
||||||
const mockWs = (manager as unknown as { webSocket: MockWebSocket })
|
const mockWs = (manager as unknown as { webSocket: MockWebSocket })
|
||||||
.webSocket;
|
.webSocket;
|
||||||
mockWs.simulateMessage({
|
mockWs.simulateMessage({
|
||||||
type: "vaultEvents",
|
type: "vaultChanged"
|
||||||
headEventId: 0,
|
|
||||||
events: []
|
|
||||||
});
|
});
|
||||||
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||||
|
|
@ -465,9 +456,7 @@ for (const closeBehavior of ["throws", "stays open"]) {
|
||||||
assert.equal(manager.isWebSocketConnected, false);
|
assert.equal(manager.isWebSocketConnected, false);
|
||||||
assert.equal(connected, false);
|
assert.equal(connected, false);
|
||||||
socket.simulateMessage({
|
socket.simulateMessage({
|
||||||
type: "vaultEvents",
|
type: "vaultChanged"
|
||||||
headEventId: 0,
|
|
||||||
events: []
|
|
||||||
});
|
});
|
||||||
await manager.waitUntilFinished();
|
await manager.waitUntilFinished();
|
||||||
assert.equal(delivered, 0);
|
assert.equal(delivered, 0);
|
||||||
|
|
@ -499,8 +488,7 @@ it("a handshake send failure retires the connection without escaping the open ca
|
||||||
manager.sendHandshakeMessage({
|
manager.sendHandshakeMessage({
|
||||||
type: "handshake",
|
type: "handshake",
|
||||||
token: "test",
|
token: "test",
|
||||||
deviceId: "test",
|
deviceId: "test"
|
||||||
lastSeenVaultUpdateId: 0
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
const statuses: boolean[] = [];
|
const statuses: boolean[] = [];
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage";
|
||||||
import type { CursorPositionFromClient } from "./types/CursorPositionFromClient";
|
import type { CursorPositionFromClient } from "./types/CursorPositionFromClient";
|
||||||
import type { ClientCursors } from "./types/ClientCursors";
|
import type { ClientCursors } from "./types/ClientCursors";
|
||||||
import { createPromise } from "../utils/create-promise";
|
import { createPromise } from "../utils/create-promise";
|
||||||
import type { EventBatch } from "./types/EventBatch";
|
|
||||||
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts";
|
import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts";
|
||||||
import { removeFromArray } from "../utils/remove-from-array";
|
import { removeFromArray } from "../utils/remove-from-array";
|
||||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
|
|
@ -17,7 +16,7 @@ export class WebSocketManager {
|
||||||
>();
|
>();
|
||||||
|
|
||||||
public readonly onRemoteVaultUpdateReceived = new EventListeners<
|
public readonly onRemoteVaultUpdateReceived = new EventListeners<
|
||||||
(update: EventBatch) => Promise<void>
|
() => Promise<void>
|
||||||
>();
|
>();
|
||||||
|
|
||||||
public readonly onRemoteCursorsUpdateReceived = new EventListeners<
|
public readonly onRemoteCursorsUpdateReceived = new EventListeners<
|
||||||
|
|
@ -303,7 +302,7 @@ export class WebSocketManager {
|
||||||
event.data
|
event.data
|
||||||
) as WebSocketServerMessage;
|
) as WebSocketServerMessage;
|
||||||
|
|
||||||
if (["vaultEvents", "cursorPositions"].includes(message.type))
|
if (["vaultChanged", "cursorPositions"].includes(message.type))
|
||||||
this.expectResponse(socket);
|
this.expectResponse(socket);
|
||||||
// Track the message handling promise
|
// Track the message handling promise
|
||||||
const messageHandlingPromise = this.handleWebSocketMessage(
|
const messageHandlingPromise = this.handleWebSocketMessage(
|
||||||
|
|
@ -340,8 +339,8 @@ export class WebSocketManager {
|
||||||
private async handleWebSocketMessage(
|
private async handleWebSocketMessage(
|
||||||
message: WebSocketServerMessage
|
message: WebSocketServerMessage
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
if (message.type === "vaultEvents") {
|
if (message.type === "vaultChanged") {
|
||||||
await this.onRemoteVaultUpdateReceived.triggerAsync(message);
|
await this.onRemoteVaultUpdateReceived.triggerAsync();
|
||||||
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
||||||
} else if (message.type === "cursorPositions") {
|
} else if (message.type === "cursorPositions") {
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ import type { FileSystemOperations } from "./file-operations/filesystem-operatio
|
||||||
import { FileOperations } from "./file-operations/file-operations";
|
import { FileOperations } from "./file-operations/file-operations";
|
||||||
import { Logger, LogLevel } from "./tracing/logger";
|
import { Logger, LogLevel } from "./tracing/logger";
|
||||||
import { SyncHistory } from "./tracing/sync-history";
|
import { SyncHistory } from "./tracing/sync-history";
|
||||||
import { FetchController } from "./services/fetch-controller";
|
|
||||||
import { SyncService } from "./services/sync-service";
|
import { SyncService } from "./services/sync-service";
|
||||||
import { ServerConfig } from "./services/server-config";
|
import { ServerConfig } from "./services/server-config";
|
||||||
import { WebSocketManager } from "./services/websocket-manager";
|
import { WebSocketManager } from "./services/websocket-manager";
|
||||||
|
|
@ -23,7 +22,6 @@ import { DocumentSyncStatus } from "./types/document-sync-status";
|
||||||
import type { CursorSpan } from "./services/types/CursorSpan";
|
import type { CursorSpan } from "./services/types/CursorSpan";
|
||||||
import { isInternalPath } from "./utils/portable-path";
|
import { isInternalPath } from "./utils/portable-path";
|
||||||
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache";
|
||||||
import { globsToRegexes } from "./utils/globs-to-regexes";
|
|
||||||
import { Lock } from "./utils/data-structures/locks";
|
import { Lock } from "./utils/data-structures/locks";
|
||||||
import {
|
import {
|
||||||
unappliedChanges,
|
unappliedChanges,
|
||||||
|
|
@ -61,7 +59,7 @@ export class SyncClient {
|
||||||
private readonly database: Database,
|
private readonly database: Database,
|
||||||
private readonly syncer: Syncer,
|
private readonly syncer: Syncer,
|
||||||
private readonly webSocketManager: WebSocketManager,
|
private readonly webSocketManager: WebSocketManager,
|
||||||
private readonly fetchController: FetchController,
|
private readonly service: SyncService,
|
||||||
private readonly serverConfig: ServerConfig,
|
private readonly serverConfig: ServerConfig,
|
||||||
private readonly cursorTracker: CursorTracker,
|
private readonly cursorTracker: CursorTracker,
|
||||||
private readonly fileChangeNotifier: FileChangeNotifier,
|
private readonly fileChangeNotifier: FileChangeNotifier,
|
||||||
|
|
@ -95,8 +93,6 @@ export class SyncClient {
|
||||||
persistence: PersistenceProvider<StoredClient>;
|
persistence: PersistenceProvider<StoredClient>;
|
||||||
fetch?: typeof globalThis.fetch;
|
fetch?: typeof globalThis.fetch;
|
||||||
webSocket?: typeof globalThis.WebSocket;
|
webSocket?: typeof globalThis.WebSocket;
|
||||||
/** V4 preserves bytes and line endings as stored, on every platform. */
|
|
||||||
nativeLineEndings?: string;
|
|
||||||
}): Promise<SyncClient> {
|
}): Promise<SyncClient> {
|
||||||
const logger = new Logger();
|
const logger = new Logger();
|
||||||
let stored = (await persistence.load()) ?? {};
|
let stored = (await persistence.load()) ?? {};
|
||||||
|
|
@ -152,32 +148,21 @@ export class SyncClient {
|
||||||
"Offline notifications belong to another vault; use a separate state store"
|
"Offline notifications belong to another vault; use a separate state store"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const fetchController = new FetchController(
|
|
||||||
settings.getSettings().isSyncEnabled,
|
|
||||||
logger
|
|
||||||
);
|
|
||||||
const deviceId = createClientId();
|
const deviceId = createClientId();
|
||||||
const service = new SyncService(
|
const service = new SyncService(deviceId, settings, fetch, {
|
||||||
deviceId,
|
get: () =>
|
||||||
fetchController,
|
stored.historyCheckpoint?.vaultKey ===
|
||||||
settings,
|
vaultKey(settings.getSettings())
|
||||||
logger,
|
? stored.historyCheckpoint.checkpoint
|
||||||
fetch,
|
: undefined,
|
||||||
{
|
save: async (checkpoint) =>
|
||||||
get: () =>
|
save({
|
||||||
stored.historyCheckpoint?.vaultKey ===
|
historyCheckpoint: {
|
||||||
vaultKey(settings.getSettings())
|
vaultKey: vaultKey(settings.getSettings()),
|
||||||
? stored.historyCheckpoint.checkpoint
|
checkpoint
|
||||||
: undefined,
|
}
|
||||||
save: async (checkpoint) =>
|
})
|
||||||
save({
|
});
|
||||||
historyCheckpoint: {
|
|
||||||
vaultKey: vaultKey(settings.getSettings()),
|
|
||||||
checkpoint
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
);
|
|
||||||
const serverConfig = new ServerConfig(service);
|
const serverConfig = new ServerConfig(service);
|
||||||
const notifier = new FileChangeNotifier();
|
const notifier = new FileChangeNotifier();
|
||||||
let syncer: Syncer;
|
let syncer: Syncer;
|
||||||
|
|
@ -192,15 +177,8 @@ export class SyncClient {
|
||||||
entries: () => localChanges,
|
entries: () => localChanges,
|
||||||
flush: async () => syncer?.flushLocalChanges()
|
flush: async () => syncer?.flushLocalChanges()
|
||||||
},
|
},
|
||||||
(path, size) => {
|
(path, size) =>
|
||||||
const current = settings.getSettings();
|
settings.isIgnored(path) || settings.isOversized(size)
|
||||||
return (
|
|
||||||
size > current.maxFileSizeMB * 1024 * 1024 ||
|
|
||||||
globsToRegexes(current.ignorePatterns, logger).some(
|
|
||||||
(pattern) => pattern.test(path)
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
);
|
);
|
||||||
const websocket = new WebSocketManager(
|
const websocket = new WebSocketManager(
|
||||||
deviceId,
|
deviceId,
|
||||||
|
|
@ -240,7 +218,7 @@ export class SyncClient {
|
||||||
database,
|
database,
|
||||||
syncer,
|
syncer,
|
||||||
websocket,
|
websocket,
|
||||||
fetchController,
|
service,
|
||||||
serverConfig,
|
serverConfig,
|
||||||
cursors,
|
cursors,
|
||||||
notifier,
|
notifier,
|
||||||
|
|
@ -308,15 +286,14 @@ export class SyncClient {
|
||||||
if (this.destroyed) return;
|
if (this.destroyed) return;
|
||||||
if (!this.settings.getSettings().isSyncEnabled) return;
|
if (!this.settings.getSettings().isSyncEnabled) return;
|
||||||
await this.database.bindVault(vaultKey(this.settings.getSettings()));
|
await this.database.bindVault(vaultKey(this.settings.getSettings()));
|
||||||
this.fetchController.finishReset();
|
this.service.resume();
|
||||||
this.fetchController.canFetch = true;
|
|
||||||
this.syncer.start();
|
this.syncer.start();
|
||||||
await this.syncer.waitUntilFinished();
|
await this.syncer.waitUntilFinished();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async pause(): Promise<void> {
|
private async pause(): Promise<void> {
|
||||||
const stopped = this.syncer.stop();
|
const stopped = this.syncer.stop();
|
||||||
this.fetchController.startReset();
|
this.service.pause();
|
||||||
await this.webSocketManager.stop();
|
await this.webSocketManager.stop();
|
||||||
await stopped;
|
await stopped;
|
||||||
}
|
}
|
||||||
|
|
@ -470,7 +447,7 @@ export class SyncClient {
|
||||||
this.destroyed = true;
|
this.destroyed = true;
|
||||||
// Interrupt a start/reset awaiting network progress before waiting for
|
// Interrupt a start/reset awaiting network progress before waiting for
|
||||||
// its lifecycle operation to finish.
|
// its lifecycle operation to finish.
|
||||||
this.fetchController.startReset();
|
this.service.pause();
|
||||||
const stopped = this.syncer.stop();
|
const stopped = this.syncer.stop();
|
||||||
this.destroying = this.lifecycle.withLock(async () => {
|
this.destroying = this.lifecycle.withLock(async () => {
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
|
|
@ -4,16 +4,15 @@ import { afterEach, beforeEach, describe, it, mock } from "node:test";
|
||||||
import type { Database } from "../persistence/database";
|
import type { Database } from "../persistence/database";
|
||||||
import {
|
import {
|
||||||
DEFAULT_SETTINGS,
|
DEFAULT_SETTINGS,
|
||||||
type Settings,
|
Settings,
|
||||||
type SyncSettings
|
type SyncSettings
|
||||||
} from "../persistence/settings";
|
} from "../persistence/settings";
|
||||||
import type { FileOperations } from "../file-operations/file-operations";
|
import type { FileOperations } from "../file-operations/file-operations";
|
||||||
import type { SyncService } from "../services/sync-service";
|
import type { SyncService } from "../services/sync-service";
|
||||||
import type { ServerConfig } from "../services/server-config";
|
import type { ServerConfig } from "../services/server-config";
|
||||||
import type { WebSocketManager } from "../services/websocket-manager";
|
import type { WebSocketManager } from "../services/websocket-manager";
|
||||||
import type { Logger } from "../tracing/logger";
|
import { Logger } from "../tracing/logger";
|
||||||
import type { SyncHistory } from "../tracing/sync-history";
|
import type { SyncHistory } from "../tracing/sync-history";
|
||||||
import type { EventBatch } from "../services/types/EventBatch";
|
|
||||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
import { Syncer } from "./syncer";
|
import { Syncer } from "./syncer";
|
||||||
import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache";
|
import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache";
|
||||||
|
|
@ -24,12 +23,11 @@ function createSyncer(overrides: Partial<SyncSettings> = {}): {
|
||||||
syncer: Syncer;
|
syncer: Syncer;
|
||||||
runCount: () => number;
|
runCount: () => number;
|
||||||
} {
|
} {
|
||||||
const settings = {
|
const settings = new Settings(
|
||||||
getSettings: () => ({
|
new Logger(),
|
||||||
...DEFAULT_SETTINGS,
|
overrides,
|
||||||
...overrides
|
async () => undefined
|
||||||
})
|
);
|
||||||
} as unknown as Settings;
|
|
||||||
const database = {
|
const database = {
|
||||||
state: {}
|
state: {}
|
||||||
} as unknown as Database;
|
} as unknown as Database;
|
||||||
|
|
@ -162,7 +160,7 @@ describe("Syncer per-file state and event delivery", () => {
|
||||||
(connected: boolean) => unknown
|
(connected: boolean) => unknown
|
||||||
>(),
|
>(),
|
||||||
onRemoteVaultUpdateReceived: new EventListeners<
|
onRemoteVaultUpdateReceived: new EventListeners<
|
||||||
(batch: EventBatch) => Promise<void>
|
() => Promise<void>
|
||||||
>()
|
>()
|
||||||
} as unknown as WebSocketManager;
|
} as unknown as WebSocketManager;
|
||||||
const service = {
|
const service = {
|
||||||
|
|
@ -174,9 +172,11 @@ describe("Syncer per-file state and event delivery", () => {
|
||||||
"device",
|
"device",
|
||||||
{} as Logger,
|
{} as Logger,
|
||||||
database,
|
database,
|
||||||
{
|
new Settings(
|
||||||
getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 })
|
new Logger(),
|
||||||
} as Settings,
|
{ syncIntervalMs: 0 },
|
||||||
|
async () => undefined
|
||||||
|
),
|
||||||
service,
|
service,
|
||||||
websocket,
|
websocket,
|
||||||
{} as FileOperations,
|
{} as FileOperations,
|
||||||
|
|
@ -238,9 +238,11 @@ describe("Syncer diff uploads", () => {
|
||||||
"device",
|
"device",
|
||||||
{} as Logger,
|
{} as Logger,
|
||||||
{ state: {} } as Database,
|
{ state: {} } as Database,
|
||||||
{
|
new Settings(
|
||||||
getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 })
|
new Logger(),
|
||||||
} as Settings,
|
{ syncIntervalMs: 0 },
|
||||||
|
async () => undefined
|
||||||
|
),
|
||||||
{
|
{
|
||||||
getDocumentVersionContent: async () => parent
|
getDocumentVersionContent: async () => parent
|
||||||
} as unknown as SyncService,
|
} as unknown as SyncService,
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,6 @@ import type { VaultSnapshot } from "../services/types/VaultSnapshot";
|
||||||
import type { PushContent } from "../services/types/PushContent";
|
import type { PushContent } from "../services/types/PushContent";
|
||||||
import type { Logger } from "../tracing/logger";
|
import type { Logger } from "../tracing/logger";
|
||||||
import { EventListeners } from "../utils/data-structures/event-listeners";
|
import { EventListeners } from "../utils/data-structures/event-listeners";
|
||||||
import { globsToRegexes } from "../utils/globs-to-regexes";
|
|
||||||
import { mergeContent, toStoredSnapshot } from "./content";
|
import { mergeContent, toStoredSnapshot } from "./content";
|
||||||
import { scanLocalFiles, type LocalChange } from "./scan";
|
import { scanLocalFiles, type LocalChange } from "./scan";
|
||||||
import {
|
import {
|
||||||
|
|
@ -91,8 +90,7 @@ export class Syncer {
|
||||||
websocket.sendHandshakeMessage({
|
websocket.sendHandshakeMessage({
|
||||||
type: "handshake",
|
type: "handshake",
|
||||||
token: settings.getSettings().token,
|
token: settings.getSettings().token,
|
||||||
deviceId,
|
deviceId
|
||||||
lastSeenVaultUpdateId: database.state.lastSeenUpdateId
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
// Failed upgrades and lost transports still need HTTP catchup,
|
// Failed upgrades and lost transports still need HTTP catchup,
|
||||||
|
|
@ -128,7 +126,7 @@ export class Syncer {
|
||||||
!this.hasScanned ||
|
!this.hasScanned ||
|
||||||
this.pendingPaths.has(path) ||
|
this.pendingPaths.has(path) ||
|
||||||
this.unsyncablePaths.has(path) ||
|
this.unsyncablePaths.has(path) ||
|
||||||
this.ignored(path)
|
this.settings.isIgnored(path)
|
||||||
) {
|
) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
@ -414,7 +412,10 @@ export class Syncer {
|
||||||
const info = await this.files.fs.stat(path);
|
const info = await this.files.fs.stat(path);
|
||||||
if (info?.kind !== "file") continue;
|
if (info?.kind !== "file") continue;
|
||||||
const doc = previous.documents[id];
|
const doc = previous.documents[id];
|
||||||
if (this.ignored(path) || this.oversized(info.size)) {
|
if (
|
||||||
|
this.settings.isIgnored(path) ||
|
||||||
|
this.settings.isOversized(info.size)
|
||||||
|
) {
|
||||||
next.local[id] = path;
|
next.local[id] = path;
|
||||||
next.documents[id] = {
|
next.documents[id] = {
|
||||||
materialized: true,
|
materialized: true,
|
||||||
|
|
@ -457,20 +458,6 @@ export class Syncer {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private ignored(path: string): boolean {
|
|
||||||
return (
|
|
||||||
isInternalPath(path) ||
|
|
||||||
globsToRegexes(
|
|
||||||
this.settings.getSettings().ignorePatterns,
|
|
||||||
this.logger
|
|
||||||
).some((pattern) => pattern.test(path))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private oversized(size: number): boolean {
|
|
||||||
return size > this.settings.getSettings().maxFileSizeMB * 1024 * 1024;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async cacheContent(
|
private async cacheContent(
|
||||||
updateId: number,
|
updateId: number,
|
||||||
path: string | undefined,
|
path: string | undefined,
|
||||||
|
|
@ -550,8 +537,8 @@ export class Syncer {
|
||||||
guard: this.localGuard(),
|
guard: this.localGuard(),
|
||||||
changes: this.changes,
|
changes: this.changes,
|
||||||
files: this.files,
|
files: this.files,
|
||||||
ignored: (path) => this.ignored(path),
|
ignored: (path) => this.settings.isIgnored(path),
|
||||||
oversized: (size) => this.oversized(size),
|
oversized: (size) => this.settings.isOversized(size),
|
||||||
commit: async (next) => this.database.commit(next)
|
commit: async (next) => this.database.commit(next)
|
||||||
},
|
},
|
||||||
initial
|
initial
|
||||||
|
|
@ -735,8 +722,8 @@ export class Syncer {
|
||||||
});
|
});
|
||||||
if (
|
if (
|
||||||
next.excluded?.[head.documentId] !== undefined ||
|
next.excluded?.[head.documentId] !== undefined ||
|
||||||
this.ignored(path) ||
|
this.settings.isIgnored(path) ||
|
||||||
this.oversized(head.contentSize)
|
this.settings.isOversized(head.contentSize)
|
||||||
) {
|
) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
@ -748,7 +735,7 @@ export class Syncer {
|
||||||
: undefined;
|
: undefined;
|
||||||
if (
|
if (
|
||||||
local !== undefined &&
|
local !== undefined &&
|
||||||
this.oversized(base64ToBytes(local.contentBase64).length)
|
this.settings.isOversized(base64ToBytes(local.contentBase64).length)
|
||||||
) {
|
) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
@ -869,9 +856,9 @@ export class Syncer {
|
||||||
for (const [id, oldPath] of Object.entries(before)) {
|
for (const [id, oldPath] of Object.entries(before)) {
|
||||||
const path = next.local[id];
|
const path = next.local[id];
|
||||||
if (
|
if (
|
||||||
this.ignored(oldPath) ||
|
this.settings.isIgnored(oldPath) ||
|
||||||
this.unsyncablePaths.has(oldPath) ||
|
this.unsyncablePaths.has(oldPath) ||
|
||||||
(path !== undefined && this.ignored(path))
|
(path !== undefined && this.settings.isIgnored(path))
|
||||||
) {
|
) {
|
||||||
next.excluded ??= {};
|
next.excluded ??= {};
|
||||||
if (next.excluded[id] === undefined)
|
if (next.excluded[id] === undefined)
|
||||||
|
|
@ -882,7 +869,7 @@ export class Syncer {
|
||||||
for (const [id, path] of Object.entries(next.excluded ?? {})) {
|
for (const [id, path] of Object.entries(next.excluded ?? {})) {
|
||||||
if (
|
if (
|
||||||
before[id] === undefined &&
|
before[id] === undefined &&
|
||||||
(path === null || !this.ignored(path))
|
(path === null || !this.settings.isIgnored(path))
|
||||||
)
|
)
|
||||||
delete next.excluded![id];
|
delete next.excluded![id];
|
||||||
}
|
}
|
||||||
|
|
@ -906,7 +893,7 @@ export class Syncer {
|
||||||
next.documents[id].materialized = false;
|
next.documents[id].materialized = false;
|
||||||
if (
|
if (
|
||||||
next.excluded?.[id] === undefined &&
|
next.excluded?.[id] === undefined &&
|
||||||
!this.ignored(path) &&
|
!this.settings.isIgnored(path) &&
|
||||||
remote.entries[id] !== undefined &&
|
remote.entries[id] !== undefined &&
|
||||||
(!this.database.state.local[id] ||
|
(!this.database.state.local[id] ||
|
||||||
!next.documents[id].base ||
|
!next.documents[id].base ||
|
||||||
|
|
@ -971,11 +958,11 @@ export class Syncer {
|
||||||
}
|
}
|
||||||
for (const [id, path] of Object.entries(this.database.state.local)) {
|
for (const [id, path] of Object.entries(this.database.state.local)) {
|
||||||
if (
|
if (
|
||||||
this.ignored(path) ||
|
this.settings.isIgnored(path) ||
|
||||||
this.database.state.excluded?.[id] !== undefined ||
|
this.database.state.excluded?.[id] !== undefined ||
|
||||||
!this.database.state.documents[id]?.materialized ||
|
!this.database.state.documents[id]?.materialized ||
|
||||||
this.unsyncablePaths.has(path) ||
|
this.unsyncablePaths.has(path) ||
|
||||||
this.oversized(
|
this.settings.isOversized(
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime.
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime.
|
||||||
this.database.state.remoteHeads[id]?.contentSize ?? 0
|
this.database.state.remoteHeads[id]?.contentSize ?? 0
|
||||||
)
|
)
|
||||||
|
|
@ -985,7 +972,9 @@ export class Syncer {
|
||||||
const snapshot = await this.files.snapshot(path);
|
const snapshot = await this.files.snapshot(path);
|
||||||
if (
|
if (
|
||||||
!snapshot ||
|
!snapshot ||
|
||||||
this.oversized(base64ToBytes(snapshot.contentBase64).length)
|
this.settings.isOversized(
|
||||||
|
base64ToBytes(snapshot.contentBase64).length
|
||||||
|
)
|
||||||
) {
|
) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -1088,7 +1077,7 @@ export class Syncer {
|
||||||
doc?.rejected &&
|
doc?.rejected &&
|
||||||
doc.observedHash === doc.rejected.hash &&
|
doc.observedHash === doc.rejected.hash &&
|
||||||
doc.base?.hash !== doc.rejected.hash &&
|
doc.base?.hash !== doc.rejected.hash &&
|
||||||
!this.ignored(path) &&
|
!this.settings.isIgnored(path) &&
|
||||||
state.excluded?.[id] === undefined
|
state.excluded?.[id] === undefined
|
||||||
)
|
)
|
||||||
return doc.rejected.message;
|
return doc.rejected.message;
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue