Simplify syncing logic

frontend/sync-client/package.json

@@ -14,19 +14,17 @@
     },
     "devDependencies": {
         "byte-base64": "^1.1.0",
-        "minimatch": "^10.0.1",
+        "minimatch": "^10.1.1",
-        "p-queue": "^8.1.0",
+        "p-queue": "^9.0.1",
         "reconcile-text": "^0.8.0",
-        "uuid": "^13.0.0",
+        "@types/node": "^25.0.2",
-        "@types/node": "^24.8.1",
+        "ts-loader": "^9.5.4",
-        "ts-loader": "^9.5.2",
         "tslib": "2.8.1",
-        "tsx": "^4.20.6",
+        "tsx": "^4.21.0",
-        "typescript": "5.8.3",
+        "typescript": "5.9.3",
-        "webpack": "^5.99.9",
+        "webpack": "^5.103.0",
         "webpack-cli": "^6.0.1",
         "webpack-merge": "^6.0.1",
-        "@sentry/browser": "^10.8.0",
+        "@sentry/browser": "^10.30.0"
-        "ws": "^8.18.3"
     }
 }

frontend/sync-client/src/consts.ts

@@ -2,5 +2,6 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60;
 export const DIFF_CACHE_SIZE_MB = 2;
 export const MAX_LOG_MESSAGE_COUNT = 100000;
 export const MAX_HISTORY_ENTRY_COUNT = 5000;
-export const SUPPORTED_API_VERSION = 2;
+export const SUPPORTED_API_VERSION = 3;
-export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10;
+export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_SECONDS = 10;
+export const WEBSOCKET_CONNECTION_TIMEOUT_IN_SECONDS = 10;

frontend/sync-client/src/errors/authentication-error.ts

@@ -0,0 +1,6 @@
+export class AuthenticationError extends Error {
+    public constructor(message: string) {
+        super(message);
+        this.name = "AuthenticationError";
+    }
+}

frontend/sync-client/src/errors/file-not-found-error.ts

@@ -0,0 +1,9 @@
+export class FileNotFoundError extends Error {
+    public constructor(
+        message: string,
+        public readonly filePath: string
+    ) {
+        super(message);
+        this.name = "FileNotFoundError";
+    }
+}

frontend/sync-client/src/errors/server-version-mismatch-error.ts

@@ -0,0 +1,6 @@
+export class ServerVersionMismatchError extends Error {
+    public constructor(message: string) {
+        super(message);
+        this.name = "ServerVersionMismatchError";
+    }
+}

frontend/sync-client/src/errors/sync-reset-error.ts

@@ -0,0 +1,6 @@
+export class SyncResetError extends Error {
+    public constructor() {
+        super("SyncClient has been reset, cleaning up");
+        this.name = "SyncResetError";
+    }
+}

frontend/sync-client/src/file-operations/file-operations.test.ts

@@ -23,7 +23,7 @@ class MockServerConfig implements Pick<ServerConfig, "getConfig"> {
 
 class MockDatabase implements Partial<Database> {
     public getLatestDocumentByRelativePath(
-        _find: RelativePath
+        _target: RelativePath
     ): DocumentRecord | undefined {
         // no-op
         return undefined;

frontend/sync-client/src/file-operations/file-operations.ts

@@ -169,9 +169,9 @@ export class FileOperations {
         }
 
         await this.ensureClearPath(newPath);
-
         this.database.move(oldPath, newPath);
         await this.fs.rename(oldPath, newPath);
+
         await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath);
     }
 

frontend/sync-client/src/file-operations/safe-filesystem-operations.ts

@@ -2,7 +2,7 @@ import type { RelativePath } from "../persistence/database";
 import type { FileSystemOperations } from "./filesystem-operations";
 import type { Logger } from "../tracing/logger";
 import { Locks } from "../utils/data-structures/locks";
-import { FileNotFoundError } from "./file-not-found-error";
+import { FileNotFoundError } from "../errors/file-not-found-error";
 import type { TextWithCursors } from "reconcile-text";
 
 /**
@@ -17,7 +17,7 @@ export class SafeFileSystemOperations implements FileSystemOperations {
         private readonly fs: FileSystemOperations,
         private readonly logger: Logger
     ) {
-        this.locks = new Locks(logger);
+        this.locks = new Locks(SafeFileSystemOperations.name, logger);
     }
 
     public async listFilesRecursively(

frontend/sync-client/src/index.ts

@@ -2,6 +2,7 @@ import { awaitAll } from "./utils/await-all";
 import { logToConsole } from "./utils/debugging/log-to-console";
 import { slowFetchFactory } from "./utils/debugging/slow-fetch-factory";
 import { slowWebSocketFactory } from "./utils/debugging/slow-web-socket-factory";
+import { InMemoryFileSystem } from "./utils/debugging/in-memory-file-system";
 import { getRandomColor } from "./utils/get-random-color";
 import { lineAndColumnToPosition } from "./utils/line-and-column-to-position";
 import { positionToLineAndColumn } from "./utils/position-to-line-and-column";
@@ -27,8 +28,8 @@ export type { PersistenceProvider } from "./persistence/persistence";
 export type { CursorSpan } from "./services/types/CursorSpan";
 export type { ClientCursors } from "./services/types/ClientCursors";
 export type { NetworkConnectionStatus } from "./types/network-connection-status";
-export type { ServerVersionMismatchError } from "./services/server-version-mismatch-error";
+export type { ServerVersionMismatchError } from "./errors/server-version-mismatch-error";
-export type { AuthenticationError } from "./services/authentication-error";
+export type { AuthenticationError } from "./errors/authentication-error";
 export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors";
 export { DocumentSyncStatus } from "./types/document-sync-status";
 export { SyncClient } from "./sync-client";
@@ -37,7 +38,8 @@ export type { TextWithCursors, CursorPosition } from "reconcile-text";
 export const debugging = {
     slowFetchFactory,
     slowWebSocketFactory,
-    logToConsole
+    logToConsole,
+    InMemoryFileSystem
 };
 
 export const utils = {

frontend/sync-client/src/persistence/database.ts

@@ -9,6 +9,7 @@ export type DocumentId = string;
 export type RelativePath = string;
 
 export interface DocumentMetadata {
+    documentId: DocumentId;
     parentVersionId: VaultUpdateId;
     hash: string;
     remoteRelativePath?: RelativePath;
@@ -25,7 +26,6 @@ export interface StoredDocumentMetadata {
 export interface StoredDatabase {
     documents: StoredDocumentMetadata[];
     lastSeenUpdateId: VaultUpdateId | undefined;
-    hasInitialSyncCompleted: boolean;
 }
 
 /**
@@ -36,17 +36,14 @@ export interface StoredDatabase {
  */
 export interface DocumentRecord {
     relativePath: RelativePath;
-    documentId: DocumentId;
     metadata: DocumentMetadata | undefined;
     isDeleted: boolean;
-    updates: Promise<unknown>[];
     parallelVersion: number;
 }
 
 export class Database {
     private documents: DocumentRecord[];
     private lastSeenUpdateIds: CoveredValues;
-    private hasInitialSyncCompleted: boolean;
 
     public constructor(
         private readonly logger: Logger,
@@ -56,16 +53,12 @@ export class Database {
         initialState ??= {};
 
         this.documents =
-            initialState.documents?.map(
+            initialState.documents?.map(({ relativePath, ...metadata }) => ({
-                ({ relativePath, documentId, ...metadata }) => ({
                 relativePath,
-                    documentId,
                 metadata,
                 isDeleted: false,
-                    updates: [],
                 parallelVersion: 0
-                })
+            })) ?? [];
-            ) ?? [];
 
         this.ensureConsistency();
         this.logger.debug(`Loaded ${this.documents.length} documents`);
@@ -79,12 +72,6 @@ export class Database {
         this.documents.forEach((doc) => {
             this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId);
         });
-
-        this.hasInitialSyncCompleted =
-            initialState.hasInitialSyncCompleted ?? false;
-        this.logger.debug(
-            `Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}`
-        );
     }
 
     public get length(): number {
@@ -127,91 +114,51 @@ export class Database {
 
     public updateDocumentMetadata(
         metadata: {
+            documentId: DocumentId;
             parentVersionId: VaultUpdateId;
             hash: string;
             remoteRelativePath: RelativePath;
         },
-        toUpdate: DocumentRecord
+        target: DocumentRecord
     ): void {
-        if (!this.documents.includes(toUpdate)) {
+        if (!this.documents.includes(target)) {
             throw new Error("Document not found in database");
         }
 
-        toUpdate.metadata = metadata;
+        this.logger.debug(
-
+            `Updating document metadata for ${target.relativePath} from ${JSON.stringify(
-        this.saveInTheBackground();
+                target.metadata,
-    }
+                null,
-
+                2
-    public removeDocumentPromise(promise: Promise<unknown>): void {
+            )} to ${JSON.stringify(metadata, null, 2)}`
-        const entry = this.documents.find(({ updates }) =>
-            updates.includes(promise)
         );
 
-        if (entry === undefined) {
+        target.metadata = metadata;
-            // This method should be idempotent and tolerant of
-            // stragglers calling it after the databse has been reset.
-            return;
-        }
 
-        removeFromArray(entry.updates, promise);
-        // No need to save as Promises don't get serialized
-    }
-
-    public removeDocument(find: DocumentRecord): void {
-        removeFromArray(this.documents, find);
         this.saveInTheBackground();
     }
 
     public getLatestDocumentByRelativePath(
-        find: RelativePath
+        target: RelativePath
     ): DocumentRecord | undefined {
         const candidates = this.documents.filter(
-            ({ relativePath }) => relativePath === find
+            ({ relativePath }) => relativePath === target
         );
         candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending
         return candidates[0];
     }
 
-    public async getResolvedDocumentByRelativePath(
-        relativePath: RelativePath,
-        promise: Promise<unknown>
-    ): Promise<DocumentRecord> {
-        const entry = this.getLatestDocumentByRelativePath(relativePath);
-
-        if (entry === undefined) {
-            throw new Error(
-                `Document not found by relative path: ${relativePath}, ${JSON.stringify(
-                    this.documents,
-                    null,
-                    2
-                )}`
-            );
-        }
-
-        const currentPromises = entry.updates;
-        entry.updates = [...currentPromises, promise];
-        await awaitAll(currentPromises);
-
-        return entry;
-    }
-
     public createNewPendingDocument(
-        documentId: DocumentId,
+        relativePath: RelativePath
-        relativePath: RelativePath,
-        promise: Promise<unknown>
     ): DocumentRecord {
-        this.logger.debug(
+        this.logger.debug(`Creating new pending document: ${relativePath}`);
-            `Creating new pending document: ${relativePath} (${documentId})`
-        );
         const previousEntry =
             this.getLatestDocumentByRelativePath(relativePath);
 
         const entry = {
             relativePath,
-            documentId,
             metadata: undefined,
             isDeleted: false,
-            updates: [promise],
             parallelVersion:
                 previousEntry?.parallelVersion === undefined
                     ? 0
@@ -219,39 +166,18 @@ export class Database {
         };
 
         this.documents.push(entry);
-        this.saveInTheBackground();
 
-        return entry;
+        // no need to save as we only save documents which have metadata
-    }
-
-    public createNewEmptyDocument(
-        documentId: DocumentId,
-        parentVersionId: VaultUpdateId,
-        relativePath: RelativePath
-    ): DocumentRecord {
-        const entry = {
-            relativePath,
-            documentId,
-            metadata: {
-                parentVersionId,
-                hash: EMPTY_HASH,
-                remoteRelativePath: relativePath
-            },
-            isDeleted: false,
-            updates: [],
-            parallelVersion: 0
-        };
-
-        this.documents.push(entry);
-        this.saveInTheBackground();
 
         return entry;
     }
 
     public getDocumentByDocumentId(
-        find: DocumentId
+        target: DocumentId
     ): DocumentRecord | undefined {
-        return this.documents.find(({ documentId }) => documentId === find);
+        return this.documents.find(
+            ({ metadata }) => metadata?.documentId === target
+        );
     }
 
     public move(
@@ -274,7 +200,7 @@ export class Database {
         }
 
         oldDocument.relativePath = newRelativePath;
-        // We're in a strange state where the target of the move has just got deleted,
+        // We might be in a strange state where the target of the move has just got deleted,
         // however, its metadata might already have a bunch of updates queued up for
         // the document at the new location. We need to keep these updates.
         oldDocument.parallelVersion =
@@ -286,19 +212,13 @@ export class Database {
     public delete(relativePath: RelativePath): void {
         const candidate = this.getLatestDocumentByRelativePath(relativePath);
         if (candidate === undefined) {
-            throw new Error(
+            return;
-                `Document not found by relative path: ${relativePath}`
-            );
         }
         candidate.isDeleted = true;
     }
 
-    public getHasInitialSyncCompleted(): boolean {
+    public removeDocument(target: DocumentRecord): void {
-        return this.hasInitialSyncCompleted;
+        removeFromArray(this.documents, target);
-    }
-
-    public setHasInitialSyncCompleted(value: boolean): void {
-        this.hasInitialSyncCompleted = value;
         this.saveInTheBackground();
     }
 
@@ -324,38 +244,45 @@ export class Database {
         this.lastSeenUpdateIds = new CoveredValues(
             0 // the first updateId will be 1 which is the first integer after -1
         );
-        this.hasInitialSyncCompleted = false;
         this.saveInTheBackground();
     }
 
     public async save(): Promise<void> {
         return this.saveData({
             documents: this.resolvedDocuments.map(
-                ({ relativePath, documentId, metadata }) => ({
+                ({ relativePath, metadata }) => ({
-                    documentId,
                     relativePath,
                     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                     ...metadata! // `resolvedDocuments` only returns docs with metadata set
                 })
             ),
-            lastSeenUpdateId: this.lastSeenUpdateIds.min,
+            lastSeenUpdateId: this.lastSeenUpdateIds.min
-            hasInitialSyncCompleted: this.hasInitialSyncCompleted
         });
     }
 
     private ensureConsistency(): void {
         const idToPath = new Map<string, string[]>();
 
-        this.resolvedDocuments.forEach(({ relativePath, documentId }) => {
+        this.resolvedDocuments.forEach(({ relativePath, metadata }) => {
-            idToPath.set(documentId, [
+            if (metadata === undefined) {
-                ...(idToPath.get(documentId) ?? []),
+                return;
+            }
+            idToPath.set(metadata.documentId, [
+                ...(idToPath.get(metadata.documentId) ?? []),
                 relativePath
             ]);
         });
 
         const duplicates = Array.from(idToPath.entries())
             .filter(([_, paths]) => paths.length > 1)
-            .map(([id, paths]) => `${id} (${paths.join(", ")})`);
+            .map(([id, paths]) => {
+                let details = "";
+                for (const path of paths) {
+                    const doc = this.getLatestDocumentByRelativePath(path);
+                    details += `\n- ${JSON.stringify(doc, null, 2)}`;
+                }
+                return `${id} (${paths.join(", ")}): ${details}`;
+            });
 
         if (duplicates.length > 0) {
             throw new Error(

frontend/sync-client/src/persistence/settings.ts

@@ -38,7 +38,7 @@ export class Settings {
     >();
 
     private settings: SyncSettings;
-    private readonly lock: Lock = new Lock();
+    private readonly lock: Lock;
 
     public constructor(
         private readonly logger: Logger,
@@ -50,6 +50,8 @@ export class Settings {
             ...(initialState ?? {})
         };
 
+        this.lock = new Lock(Settings.name, this.logger);
+
         this.logger.debug(
             `Loaded settings: ${JSON.stringify(this.settings, null, 2)}`
         );

frontend/sync-client/src/services/fetch-controller.test.ts

@@ -3,7 +3,7 @@ import { describe, it, mock, beforeEach, afterEach } from "node:test";
 import assert from "node:assert";
 import { FetchController } from "./fetch-controller";
 import { Logger } from "../tracing/logger";
-import { SyncResetError } from "./sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 import { sleep } from "../utils/sleep";
 
 describe("FetchController", () => {

frontend/sync-client/src/services/fetch-controller.ts

@@ -1,6 +1,6 @@
 import type { Logger } from "../tracing/logger";
 import { createPromise } from "../utils/create-promise";
-import { SyncResetError } from "./sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 
 /**
  * Offers a resettable fetch implementation that waits until syncing is enabled

frontend/sync-client/src/services/server-config.ts

@@ -1,6 +1,6 @@
 import { SUPPORTED_API_VERSION } from "../consts";
-import { AuthenticationError } from "./authentication-error";
+import { AuthenticationError } from "../errors/authentication-error";
-import { ServerVersionMismatchError } from "./server-version-mismatch-error";
+import { ServerVersionMismatchError } from "../errors/server-version-mismatch-error";
 import type { SyncService } from "./sync-service";
 import type { PingResponse } from "./types/PingResponse";
 
@@ -34,11 +34,6 @@ export class ServerConfig {
         }
     }
 
-    // warm the cache
-    public async initialize(): Promise<void> {
-        await this.getConfig();
-    }
-
     public async checkConnection(forceUpdate = false): Promise<{
         isSuccessful: boolean;
         message: string;

frontend/sync-client/src/services/sync-service.ts

@@ -8,7 +8,7 @@ import type { Logger } from "../tracing/logger";
 import type { Settings } from "../persistence/settings";
 import type { FetchController } from "./fetch-controller";
 import { sleep } from "../utils/sleep";
-import { SyncResetError } from "./sync-reset-error";
+import { SyncResetError } from "../errors/sync-reset-error";
 import type { SerializedError } from "./types/SerializedError";
 import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent";
 import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse";
@@ -66,19 +66,15 @@ export class SyncService {
     }
 
     public async create({
-        documentId,
         relativePath,
         contentBytes
     }: {
-        documentId?: DocumentId;
         relativePath: RelativePath;
         contentBytes: Uint8Array;
-    }): Promise<DocumentVersionWithoutContent> {
+    }): Promise<DocumentUpdateResponse> {
         return this.retryForever(async () => {
             const formData = new FormData();
-            if (documentId !== undefined) {
+
-                formData.append("document_id", documentId);
-            }
             formData.append("relative_path", relativePath);
             formData.append(
                 "content",
@@ -86,7 +82,7 @@ export class SyncService {
             );
 
             this.logger.debug(
-                `Creating document with id ${documentId} and relative path ${relativePath}`
+                `Creating document with relative path ${relativePath}`
             );
 
             const response = await this.client(this.getUrl("/documents"), {
@@ -103,8 +99,8 @@ export class SyncService {
                 );
             }
 
-            const result: DocumentVersionWithoutContent =
+            const result: DocumentUpdateResponse =
-                (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
+                (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion
 
             this.logger.debug(`Created document ${JSON.stringify(result)}`);
 

frontend/sync-client/src/services/types/ClientCursors.ts

@@ -1,4 +1,8 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DocumentWithCursors } from "./DocumentWithCursors";
 
-export interface ClientCursors { userName: string, deviceId: string, documentsWithCursors: DocumentWithCursors[], }
+export interface ClientCursors {
+    userName: string;
+    deviceId: string;
+    documentsWithCursors: DocumentWithCursors[];
+}

frontend/sync-client/src/services/types/CreateDocumentVersion.ts

@@ -1,3 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface CreateDocumentVersion { relative_path: string, content: number[], }
+export interface CreateDocumentVersion {
+    relative_path: string;
+    content: number[];
+}

frontend/sync-client/src/services/types/CursorPositionFromClient.ts

@@ -1,4 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { DocumentWithCursors } from "./DocumentWithCursors";
 
-export interface CursorPositionFromClient { documentsWithCursors: DocumentWithCursors[], }
+export interface CursorPositionFromClient {
+    documentsWithCursors: DocumentWithCursors[];
+}

frontend/sync-client/src/services/types/CursorPositionFromServer.ts

@@ -1,4 +1,6 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { ClientCursors } from "./ClientCursors";
 
-export interface CursorPositionFromServer { clients: ClientCursors[], }
+export interface CursorPositionFromServer {
+    clients: ClientCursors[];
+}