andras/vault-link
andras/vault-link
Watch
1
0
Fork
You've already forked vault-link
 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 35

Simplify syncing logic

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-28 11:55:37 +00:00
commit 4493365076
46 changed files with 1024 additions and 888 deletions
Download patch file Download diff file Expand all files Collapse all files

5
frontend/sync-client/src/consts.ts
View file

@ -2,5 +2,6 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
export const DIFF_CACHE_SIZE_MB = 2;
export const MAX_LOG_MESSAGE_COUNT = 100000;
export const MAX_HISTORY_ENTRY_COUNT = 5000;
export const SUPPORTED_API_VERSION = 2;
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
export const SUPPORTED_API_VERSION = 3;
export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10;
export const WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS = 10;

6
frontend/sync-client/src/errors/authentication-error.ts Normal file
View file

@ -0,0 +1,6 @@
export class AuthenticationError extends Error {
public constructor(message: string) {
super(message);
this.name = "AuthenticationError";
}
}

9
frontend/sync-client/src/errors/file-not-found-error.ts Normal file
View file

@ -0,0 +1,9 @@
export class FileNotFoundError extends Error {
public constructor(
message: string,
public readonly filePath: string
) {
super(message);
this.name = "FileNotFoundError";
}
}

6
frontend/sync-client/src/errors/server-version-mismatch-error.ts Normal file
View file

@ -0,0 +1,6 @@
export class ServerVersionMismatchError extends Error {
public constructor(message: string) {
super(message);
this.name = "ServerVersionMismatchError";
}
}

6
frontend/sync-client/src/errors/sync-reset-error.ts Normal file
View file

@ -0,0 +1,6 @@
export class SyncResetError extends Error {
public constructor() {
super("SyncClient has been reset, cleaning up");
this.name = "SyncResetError";
}
}

2
frontend/sync-client/src/file-operations/file-operations.test.ts
View file

@ -23,7 +23,7 @@ class MockServerConfig implements Pick<ServerConfig, "getConfig"> {
class MockDatabase implements Partial<Database> {
public getLatestDocumentByRelativePath(
_find: RelativePath
_target: RelativePath
): DocumentRecord | undefined {
// no-op
return undefined;

34
frontend/sync-client/src/file-operations/file-operations.ts
View file

@ -45,11 +45,11 @@ export class FileOperations {
}
/**
* Create a file at the specified path.
*
* If a file with the same name already exists, it is moved before creating the new one.
* Parent directories are created if necessary.
*/
* Create a file at the specified path.
*
* If a file with the same name already exists, it is moved before creating the new one.
* Parent directories are created if necessary.
*/
public async create(
path: RelativePath,
newContent: Uint8Array
@ -77,11 +77,11 @@ export class FileOperations {
}
/**
* Update the file at the given path.
*
* Performs a 3-way merge before writing if the file's content differs from `expectedContent`.
* Does not recreate the file if it no longer exists, returning an empty array instead.
*/
* Update the file at the given path.
*
* Performs a 3-way merge before writing if the file's content differs from `expectedContent`.
* Does not recreate the file if it no longer exists, returning an empty array instead.
*/
public async write(
path: RelativePath,
expectedContent: Uint8Array,
@ -169,9 +169,9 @@ export class FileOperations {
}
await this.ensureClearPath(newPath);
this.database.move(oldPath, newPath);
await this.fs.rename(oldPath, newPath);
await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
}
@ -239,12 +239,12 @@ export class FileOperations {
}
/**
* Deconflicts the given path by appending (1), (2), etc. before the file extension until a non-existent path is found.
* The returned path has a lock acquired on it; it must be released by the caller when no longer needed.
*
* @param path The starting path to deconflict
* @returns a non-existent path with a lock acquired on it
*/
* Deconflicts the given path by appending (1), (2), etc. before the file extension until a non-existent path is found.
* The returned path has a lock acquired on it; it must be released by the caller when no longer needed.
*
* @param path The starting path to deconflict
* @returns a non-existent path with a lock acquired on it
*/
private async deconflictPath(path: RelativePath): Promise<RelativePath> {
// eslint-disable-next-line prefer-const
let [directory, fileName] = FileOperations.getParentDirAndFile(path);

12
frontend/sync-client/src/file-operations/safe-filesystem-operations.ts
View file

@ -2,7 +2,7 @@ import type { RelativePath } from "../persistence/database";
import type { FileSystemOperations } from "./filesystem-operations";
import type { Logger } from "../tracing/logger";
import { Locks } from "../utils/data-structures/locks";
import { FileNotFoundError } from "./file-not-found-error";
import { FileNotFoundError } from "../errors/file-not-found-error";
import type { TextWithCursors } from "reconcile-text";
/**
@ -17,7 +17,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
private readonly fs: FileSystemOperations,
private readonly logger: Logger
) {
this.locks = new Locks(logger);
this.locks = new Locks(SafeFileSystemOperations.name, logger);
}
public async listFilesRecursively(
@ -135,10 +135,10 @@ export class SafeFileSystemOperations implements FileSystemOperations {
}
/**
* Decorate an operation to ensure that the file exists before running it.
* If the operation fails, it will check if the file still exists and throw
* a FileNotFoundError if it doesn't.
*/
* Decorate an operation to ensure that the file exists before running it.
* If the operation fails, it will check if the file still exists and throw
* a FileNotFoundError if it doesn't.
*/
private async safeOperation<T>(
path: RelativePath,
operation: () => Promise<T>,

8
frontend/sync-client/src/index.ts
View file

@ -2,6 +2,7 @@ import { awaitAll } from "./utils/await-all";
import { logToConsole } from "./utils/debugging/log-to-console";
import { slowFetchFactory } from "./utils/debugging/slow-fetch-factory";
import { slowWebSocketFactory } from "./utils/debugging/slow-web-socket-factory";
import { InMemoryFileSystem } from "./utils/debugging/in-memory-file-system";
import { getRandomColor } from "./utils/get-random-color";
import { lineAndColumnToPosition } from "./utils/line-and-column-to-position";
import { positionToLineAndColumn } from "./utils/position-to-line-and-column";
@ -27,8 +28,8 @@ export type { PersistenceProvider } from "./persistence/persistence";
export type { CursorSpan } from "./services/types/CursorSpan";
export type { ClientCursors } from "./services/types/ClientCursors";
export type { NetworkConnectionStatus } from "./types/network-connection-status";
export type { ServerVersionMismatchError } from "./services/server-version-mismatch-error";
export type { AuthenticationError } from "./services/authentication-error";
export type { ServerVersionMismatchError } from "./errors/server-version-mismatch-error";
export type { AuthenticationError } from "./errors/authentication-error";
export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
export { DocumentSyncStatus } from "./types/document-sync-status";
export { SyncClient } from "./sync-client";
@ -37,7 +38,8 @@ export type { TextWithCursors, CursorPosition } from "reconcile-text";
export const debugging = {
slowFetchFactory,
slowWebSocketFactory,
logToConsole
logToConsole,
InMemoryFileSystem
};
export const utils = {

167
frontend/sync-client/src/persistence/database.ts
View file

@ -9,6 +9,7 @@ export type DocumentId = string;
export type RelativePath = string;
export interface DocumentMetadata {
documentId: DocumentId;
parentVersionId: VaultUpdateId;
hash: string;
remoteRelativePath?: RelativePath;
@ -25,7 +26,6 @@ export interface StoredDocumentMetadata {
export interface StoredDatabase {
documents: StoredDocumentMetadata[];
lastSeenUpdateId: VaultUpdateId | undefined;
hasInitialSyncCompleted: boolean;
}
/**
@ -36,17 +36,14 @@ export interface StoredDatabase {
*/
export interface DocumentRecord {
relativePath: RelativePath;
documentId: DocumentId;
metadata: DocumentMetadata | undefined;
isDeleted: boolean;
updates: Promise<unknown>[];
parallelVersion: number;
}
export class Database {
private documents: DocumentRecord[];
private lastSeenUpdateIds: CoveredValues;
private hasInitialSyncCompleted: boolean;
public constructor(
private readonly logger: Logger,
@ -56,16 +53,12 @@ export class Database {
initialState ??= {};
this.documents =
initialState.documents?.map(
({ relativePath, documentId, ...metadata }) => ({
relativePath,
documentId,
metadata,
isDeleted: false,
updates: [],
parallelVersion: 0
})
) ?? [];
initialState.documents?.map(({ relativePath, ...metadata }) => ({
relativePath,
metadata,
isDeleted: false,
parallelVersion: 0
})) ?? [];
this.ensureConsistency();
this.logger.debug(`Loaded ${this.documents.length} documents`);
@ -79,12 +72,6 @@ export class Database {
this.documents.forEach((doc) => {
this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId);
});
this.hasInitialSyncCompleted =
initialState.hasInitialSyncCompleted ?? false;
this.logger.debug(
`Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
);
}
public get length(): number {
@ -127,91 +114,51 @@ export class Database {
public updateDocumentMetadata(
metadata: {
documentId: DocumentId;
parentVersionId: VaultUpdateId;
hash: string;
remoteRelativePath: RelativePath;
},
toUpdate: DocumentRecord
target: DocumentRecord
): void {
if (!this.documents.includes(toUpdate)) {
if (!this.documents.includes(target)) {
throw new Error("Document not found in database");
}
toUpdate.metadata = metadata;
this.saveInTheBackground();
}
public removeDocumentPromise(promise: Promise<unknown>): void {
const entry = this.documents.find(({ updates }) =>
updates.includes(promise)
this.logger.debug(
`Updating document metadata for ${target.relativePath} from ${JSON.stringify(
target.metadata,
null,
2
)} to ${JSON.stringify(metadata, null, 2)}`
);
if (entry === undefined) {
// This method should be idempotent and tolerant of
// stragglers calling it after the databse has been reset.
return;
}
target.metadata = metadata;
removeFromArray(entry.updates, promise);
// No need to save as Promises don't get serialized
}
public removeDocument(find: DocumentRecord): void {
removeFromArray(this.documents, find);
this.saveInTheBackground();
}
public getLatestDocumentByRelativePath(
find: RelativePath
target: RelativePath
): DocumentRecord | undefined {
const candidates = this.documents.filter(
({ relativePath }) => relativePath === find
({ relativePath }) => relativePath === target
);
candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
return candidates[0];
}
public async getResolvedDocumentByRelativePath(
relativePath: RelativePath,
promise: Promise<unknown>
): Promise<DocumentRecord> {
const entry = this.getLatestDocumentByRelativePath(relativePath);
if (entry === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}, ${JSON.stringify(
this.documents,
null,
2
)}`
);
}
const currentPromises = entry.updates;
entry.updates = [...currentPromises, promise];
await awaitAll(currentPromises);
return entry;
}
public createNewPendingDocument(
documentId: DocumentId,
relativePath: RelativePath,
promise: Promise<unknown>
relativePath: RelativePath
): DocumentRecord {
this.logger.debug(
`Creating new pending document: ${relativePath} (${documentId})`
);
this.logger.debug(`Creating new pending document: ${relativePath}`);
const previousEntry =
this.getLatestDocumentByRelativePath(relativePath);
const entry = {
relativePath,
documentId,
metadata: undefined,
isDeleted: false,
updates: [promise],
parallelVersion:
previousEntry?.parallelVersion === undefined
? 0
@ -219,39 +166,18 @@ export class Database {
};
this.documents.push(entry);
this.saveInTheBackground();
return entry;
}
public createNewEmptyDocument(
documentId: DocumentId,
parentVersionId: VaultUpdateId,
relativePath: RelativePath
): DocumentRecord {
const entry = {
relativePath,
documentId,
metadata: {
parentVersionId,
hash: EMPTY_HASH,
remoteRelativePath: relativePath
},
isDeleted: false,
updates: [],
parallelVersion: 0
};
this.documents.push(entry);
this.saveInTheBackground();
// no need to save as we only save documents which have metadata
return entry;
}
public getDocumentByDocumentId(
find: DocumentId
target: DocumentId
): DocumentRecord | undefined {
return this.documents.find(({ documentId }) => documentId === find);
return this.documents.find(
({ metadata }) => metadata?.documentId === target
);
}
public move(
@ -274,7 +200,7 @@ export class Database {
}
oldDocument.relativePath = newRelativePath;
// We're in a strange state where the target of the move has just got deleted,
// We might be in a strange state where the target of the move has just got deleted,
// however, its metadata might already have a bunch of updates queued up for
// the document at the new location. We need to keep these updates.
oldDocument.parallelVersion =
@ -286,19 +212,13 @@ export class Database {
public delete(relativePath: RelativePath): void {
const candidate = this.getLatestDocumentByRelativePath(relativePath);
if (candidate === undefined) {
throw new Error(
`Document not found by relative path: ${relativePath}`
);
return;
}
candidate.isDeleted = true;
}
public getHasInitialSyncCompleted(): boolean {
return this.hasInitialSyncCompleted;
}
public setHasInitialSyncCompleted(value: boolean): void {
this.hasInitialSyncCompleted = value;
public removeDocument(target: DocumentRecord): void {
removeFromArray(this.documents, target);
this.saveInTheBackground();
}
@ -324,43 +244,50 @@ export class Database {
this.lastSeenUpdateIds = new CoveredValues(
0 // the first updateId will be 1 which is the first integer after -1
);
this.hasInitialSyncCompleted = false;
this.saveInTheBackground();
}
public async save(): Promise<void> {
return this.saveData({
documents: this.resolvedDocuments.map(
({ relativePath, documentId, metadata }) => ({
documentId,
({ relativePath, metadata }) => ({
relativePath,
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
...metadata! // `resolvedDocuments` only returns docs with metadata set
})
),
lastSeenUpdateId: this.lastSeenUpdateIds.min,
hasInitialSyncCompleted: this.hasInitialSyncCompleted
lastSeenUpdateId: this.lastSeenUpdateIds.min
});
}
private ensureConsistency(): void {
const idToPath = new Map<string, string[]>();
this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
idToPath.set(documentId, [
...(idToPath.get(documentId) ?? []),
this.resolvedDocuments.forEach(({ relativePath, metadata }) => {
if (metadata === undefined) {
return;
}
idToPath.set(metadata.documentId, [
...(idToPath.get(metadata.documentId) ?? []),
relativePath
]);
});
const duplicates = Array.from(idToPath.entries())
.filter(([_, paths]) => paths.length > 1)
.map(([id, paths]) => `${id} (${paths.join(", ")})`);
.map(([id, paths]) => {
let details = "";
for (const path of paths) {
const doc = this.getLatestDocumentByRelativePath(path);
details += `\n- ${JSON.stringify(doc, null, 2)}`;
}
return `${id} (${paths.join(", ")}): ${details}`;
});
if (duplicates.length > 0) {
throw new Error(
"Document IDs are not unique, found duplicates: " +
duplicates.join("; ")
duplicates.join("; ")
);
}
}

4
frontend/sync-client/src/persistence/settings.ts
View file

@ -38,7 +38,7 @@ export class Settings {
>();
private settings: SyncSettings;
private readonly lock: Lock = new Lock();
private readonly lock: Lock;
public constructor(
private readonly logger: Logger,
@ -50,6 +50,8 @@ export class Settings {
...(initialState ?? {})
};
this.lock = new Lock(Settings.name, this.logger);
this.logger.debug(
`Loaded settings: ${JSON.stringify(this.settings, null, 2)}`
);

2
frontend/sync-client/src/services/fetch-controller.test.ts
View file

@ -3,7 +3,7 @@ import { describe, it, mock, beforeEach, afterEach } from "node:test";
import assert from "node:assert";
import { FetchController } from "./fetch-controller";
import { Logger } from "../tracing/logger";
import { SyncResetError } from "./sync-reset-error";
import { SyncResetError } from "../errors/sync-reset-error";
import { sleep } from "../utils/sleep";
describe("FetchController", () => {

54
frontend/sync-client/src/services/fetch-controller.ts
View file

@ -1,6 +1,6 @@
import type { Logger } from "../tracing/logger";
import { createPromise } from "../utils/create-promise";
import { SyncResetError } from "./sync-reset-error";
import { SyncResetError } from "../errors/sync-reset-error";
/**
* Offers a resettable fetch implementation that waits until syncing is enabled
@ -25,18 +25,18 @@ export class FetchController {
}
/**
* Whether the fetch implementation can immediately send requests once outside of a reset.
*/
* Whether the fetch implementation can immediately send requests once outside of a reset.
*/
public get canFetch(): boolean {
return this._canFetch;
}
/**
* Allow or disallow fetching. The changes only take effect if not resetting.
* When called during a reset, its effect is deferred until the reset is finished.
*
* @param canFetch Whether fetching is enabled
*/
* Allow or disallow fetching. The changes only take effect if not resetting.
* When called during a reset, its effect is deferred until the reset is finished.
*
* @param canFetch Whether fetching is enabled
*/
public set canFetch(canFetch: boolean) {
this._canFetch = canFetch;
@ -59,9 +59,9 @@ export class FetchController {
}
/**
* Starts a reset, causing all ongoing and future fetches to be rejected
* with a SyncResetError until finishReset is called.
*/
* Starts a reset, causing all ongoing and future fetches to be rejected
* with a SyncResetError until finishReset is called.
*/
public startReset(): void {
this.isResetting = true;
this.rejectUntil(new SyncResetError());
@ -72,9 +72,9 @@ export class FetchController {
}
/**
* Finishes a reset, allowing fetches to proceed or wait again depending on
* the current sync settings.
*/
* Finishes a reset, allowing fetches to proceed or wait again depending on
* the current sync settings.
*/
public finishReset(): void {
if (!this.isResetting) {
return;
@ -85,19 +85,19 @@ export class FetchController {
}
/**
*
* |------------------|---------------|-----------------------------------------------------|
* | | Sync enabled | Sync disabled |
* |------------------|-------------- |-----------------------------------------------------|
* | During reset | Rejects with SyncResetError without sending request |
* |------------------|-------------- |-----------------------------------------------------|
* | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch |
* |------------------|---------------|-----------------------------------------------------|
*
* @param logger for errors
* @param fetch to wrap
* @returns a wrapped fetch implementation affected by the FetchController state
*/
*
* |------------------|---------------|-----------------------------------------------------|
* | | Sync enabled | Sync disabled |
* |------------------|-------------- |-----------------------------------------------------|
* | During reset | Rejects with SyncResetError without sending request |
* |------------------|-------------- |-----------------------------------------------------|
* | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch |
* |------------------|---------------|-----------------------------------------------------|
*
* @param logger for errors
* @param fetch to wrap
* @returns a wrapped fetch implementation affected by the FetchController state
*/
public getControlledFetchImplementation(
logger: Logger,
fetch: typeof globalThis.fetch = globalThis.fetch

9
frontend/sync-client/src/services/server-config.ts
View file

@ -1,6 +1,6 @@
import { SUPPORTED_API_VERSION } from "../consts";
import { AuthenticationError } from "./authentication-error";
import { ServerVersionMismatchError } from "./server-version-mismatch-error";
import { AuthenticationError } from "../errors/authentication-error";
import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
import type { SyncService } from "./sync-service";
import type { PingResponse } from "./types/PingResponse";
@ -34,11 +34,6 @@ export class ServerConfig {
}
}
// warm the cache
public async initialize(): Promise<void> {
await this.getConfig();
}
public async checkConnection(forceUpdate = false): Promise<{
isSuccessful: boolean;
message: string;

16
frontend/sync-client/src/services/sync-service.ts
View file

@ -8,7 +8,7 @@ import type { Logger } from "../tracing/logger";
import type { Settings } from "../persistence/settings";
import type { FetchController } from "./fetch-controller";
import { sleep } from "../utils/sleep";
import { SyncResetError } from "./sync-reset-error";
import { SyncResetError } from "../errors/sync-reset-error";
import type { SerializedError } from "./types/SerializedError";
import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
@ -66,19 +66,15 @@ export class SyncService {
}
public async create({
documentId,
relativePath,
contentBytes
}: {
documentId?: DocumentId;
relativePath: RelativePath;
contentBytes: Uint8Array;
}): Promise<DocumentVersionWithoutContent> {
}): Promise<DocumentUpdateResponse> {
return this.retryForever(async () => {
const formData = new FormData();
if (documentId !== undefined) {
formData.append("document_id", documentId);
}
formData.append("relative_path", relativePath);
formData.append(
"content",
@ -86,7 +82,7 @@ export class SyncService {
);
this.logger.debug(
`Creating document with id ${documentId} and relative path ${relativePath}`
`Creating document with relative path ${relativePath}`
);
const response = await this.client(this.getUrl("/documents"), {
@ -103,8 +99,8 @@ export class SyncService {
);
}
const result: DocumentVersionWithoutContent =
(await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
const result: DocumentUpdateResponse =
(await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
this.logger.debug(`Created document ${JSON.stringify(result)}`);

6
frontend/sync-client/src/services/types/ClientCursors.ts
View file

@ -1,4 +1,8 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentWithCursors } from "./DocumentWithCursors";
export interface ClientCursors { userName: string, deviceId: string, documentsWithCursors: DocumentWithCursors[], }
export interface ClientCursors {
userName: string;
deviceId: string;
documentsWithCursors: DocumentWithCursors[];
}

5
frontend/sync-client/src/services/types/CreateDocumentVersion.ts
View file

@ -1,3 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CreateDocumentVersion { relative_path: string, content: number[], }
export interface CreateDocumentVersion {
relative_path: string;
content: number[];
}

4
frontend/sync-client/src/services/types/CursorPositionFromClient.ts
View file

@ -1,4 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { DocumentWithCursors } from "./DocumentWithCursors";
export interface CursorPositionFromClient { documentsWithCursors: DocumentWithCursors[], }
export interface CursorPositionFromClient {
documentsWithCursors: DocumentWithCursors[];
}

4
frontend/sync-client/src/services/types/CursorPositionFromServer.ts
View file

@ -1,4 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
import type { ClientCursors } from "./ClientCursors";
export interface CursorPositionFromServer { clients: ClientCursors[], }
export interface CursorPositionFromServer {
clients: ClientCursors[];
}

5
frontend/sync-client/src/services/types/CursorSpan.ts
View file

@ -1,3 +1,6 @@
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export interface CursorSpan { start: number, end: number, }
export interface CursorSpan {
start: number;
end: number;
}

Some files were not shown because too many files have changed in this diff Show more