diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index 4f3dc8bb..6272040e 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -54,7 +54,12 @@ updates only wake the loop to reread current bytes. Namespace changes detected before mutations invalidate the plan. Notifications during application are incorporated between files using their document identities; the next scan rereads content. Content is reread before replacement. Ignored and oversized occupants stay in place. Adapters -report atomic editor saves as updates and suppress engine-generated notifications. +use raw filesystem events only to wake a scan. These include engine writes and +atomic editor replacements, so they must not be reported as logical create/delete +notifications. Integrations with reliable user-action events can persist those +explicit notifications to preserve rename identities, as Obsidian does for +`Vault.rename`. Raw watchers infer a move +only when the missing and new files have a unique matching, nonempty content hash. Metadata persistence must replace a complete value atomically, leaving a readable old/new value after interruption. Metadata and user files need not describe the @@ -83,5 +88,10 @@ transient errors retry according to client settings. Observer callback failures are isolated from the background sync loop. API v5 clients and servers must be deployed together. V4 databases migrate without -removing history. The old Obsidian and CLI adapters still require the filesystem -and persistence contract required by the core engine. +removing history. CLI and desktop Obsidian share the Node filesystem adapter and +complete-record metadata persistence. Node file moves use exclusive creation +followed by unlink, so interruption may leave two ordinary files. Metadata alone +is flushed before an atomic rename; there is no user-file durability protocol. +Obsidian mobile uses the host storage API. Its exclusive-copy API requires a +scratch input file in the ignored internal directory; leftovers are never replayed +or used for recovery. Mobile metadata persistence uses Obsidian's data store. diff --git a/frontend/local-client-cli/README.md b/frontend/local-client-cli/README.md index f0cc6388..9c519a68 100644 --- a/frontend/local-client-cli/README.md +++ b/frontend/local-client-cli/README.md @@ -205,10 +205,21 @@ docker build -f local-client-cli/Dockerfile -t vault-link-cli:test . 1. Creates `.vaultlink` directory for sync metadata 2. Performs initial sync of local files to server -3. Watches filesystem for changes using Node's `fs.watch` +3. Uses filesystem events to wake a fresh scan, including atomic editor saves 4. Syncs changes bidirectionally in real-time 5. Handles graceful shutdown on SIGINT/SIGTERM +The complete client record (database, settings, queued logical notifications and +server history checkpoint) is saved atomically in `.vaultlink/sync-data.json`. +Existing database-only files migrate on the next save. Invalid or unreadable +metadata is reported instead of silently starting with empty state. + +File moves use an exclusive copy followed by deletion of the source. An +interruption may leave both files; the next scan reconciles the visible files. +Raw watcher events do not distinguish user edits from sync writes. A rename +preserves identity when the scan finds a unique matching nonempty content hash; +a simultaneous rename and edit may instead appear as deletion and creation. + ## License MIT diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index 0a40d9fb..5f2a8c9f 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -7,12 +7,12 @@ import { DEFAULT_SETTINGS, LogLevel, LogLine, - type SyncSettings, - type StoredDatabase + type SyncSettings } from "sync-client"; import { parseArgs } from "./args"; import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem"; import { FileWatcher } from "./file-watcher"; +import { ClientPersistence } from "./client-persistence"; import { formatLogLine } from "./logger-formatter"; import packageJson from "../package.json"; @@ -114,32 +114,7 @@ async function main(): Promise { const client = await SyncClient.create({ fs: fileSystem, - persistence: { - load: async () => { - let database: Partial | undefined = undefined; - try { - const content = await fs.readFile(dataFile, "utf-8"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - database = JSON.parse(content) as Partial; - } catch { - emitBoot( - LogLevel.WARNING, - `Cannot read data file at ${dataFile}` - ); - } - - return { - settings, - database - }; - }, - save: async ({ database: persistedDatabase }) => { - await fs.writeFile( - dataFile, - JSON.stringify(persistedDatabase, null, 2) - ); - } - } + persistence: new ClientPersistence(dataFile, settings) }); if (args.health !== undefined) { @@ -212,7 +187,6 @@ async function main(): Promise { client.logger.info(`${signal} received, shutting down gracefully`); fileWatcher.stop(); - await client.waitUntilFinished(); await client.destroy(); if (totalSyncOps > 0) { diff --git a/frontend/local-client-cli/src/client-persistence.test.ts b/frontend/local-client-cli/src/client-persistence.test.ts new file mode 100644 index 00000000..b400d50f --- /dev/null +++ b/frontend/local-client-cli/src/client-persistence.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import fs from "node:fs/promises"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { ClientPersistence } from "./client-persistence"; +import type { StoredClient } from "sync-client"; + +test("CLI persistence retains the whole record and migrates database-only saves", async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-metadata-") + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const file = path.join(directory, "state.json"); + const store = new ClientPersistence(file, { token: "current-token" }); + assert.deepEqual(await store.load(), { + settings: { token: "current-token" } + }); + const record: StoredClient = { + settings: { token: "old-token", vaultName: "vault" }, + database: { lastSeenUpdateId: 12 }, + localChanges: [ + { type: "delete", path: "removed.md", changeId: "queued" } + ], + localChangesVaultKey: "vault", + historyCheckpoint: { vaultKey: "vault", checkpoint: "12:incarnation" } + }; + await store.save(record); + assert.deepEqual(await store.load(), { + ...record, + settings: { ...record.settings, token: "current-token" } + }); + const legacy = { + fileManifest: { fileManifestId: 0, entries: {} }, + lastSeenUpdateId: 0 + }; + await fs.writeFile(file, JSON.stringify(legacy)); + assert.deepEqual((await store.load()).database, legacy); + await fs.writeFile(file, "{truncated"); + await assert.rejects(store.load(), SyntaxError); + await fs.rm(file); + await fs.mkdir(file); + await assert.rejects(store.load(), /EISDIR/); +}); + +for (const afterReplacement of [false, true]) { + test(`failed metadata save ${afterReplacement ? "after" : "before"} replacement leaves a complete record`, async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-save-") + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const store = new ClientPersistence( + path.join(directory, "state.json"), + {} + ); + const old: StoredClient = { + historyCheckpoint: { vaultKey: "v", checkpoint: "1:old" } + }; + const next: StoredClient = { + ...old, + localChanges: [{ type: "delete", path: "note.md", changeId: "new" }] + }; + await store.save(old); + const rename = fs.rename; + t.mock.method( + fs, + "rename", + async (...args: Parameters) => { + if (afterReplacement) await rename(...args); + throw new Error("interrupted replacement"); + } + ); + await assert.rejects(store.save(next), /interrupted replacement/); + assert.deepEqual(await store.load(), { + ...(afterReplacement ? next : old), + settings: {} + }); + assert.deepEqual(await fs.readdir(directory), ["state.json"]); + }); +} diff --git a/frontend/local-client-cli/src/client-persistence.ts b/frontend/local-client-cli/src/client-persistence.ts new file mode 100644 index 00000000..ce421654 --- /dev/null +++ b/frontend/local-client-cli/src/client-persistence.ts @@ -0,0 +1,67 @@ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { randomUUID } from "node:crypto"; +import type { + PersistenceProvider, + StoredClient, + StoredDatabase, + SyncSettings +} from "sync-client"; + +/** Only client metadata needs an atomic replacement; user files do not. */ +export class ClientPersistence implements PersistenceProvider { + public constructor( + private readonly file: string, + private readonly settings: Partial + ) {} + + public async load(): Promise { + const content = await fs + .readFile(this.file, "utf8") + .catch((error: unknown) => { + if ( + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) + return undefined; + throw error; + }); + if (content === undefined) return { settings: this.settings }; + const parsed: unknown = JSON.parse(content); + if ( + parsed === null || + typeof parsed !== "object" || + Array.isArray(parsed) + ) + throw new Error("Invalid client metadata"); + // The sync engine validates and migrates the persisted fields. + const data = parsed as StoredClient & Partial; + // Earlier CLI versions saved the database alone. + const stored: StoredClient = + "fileManifest" in data ? { database: data } : data; + return { + ...stored, + settings: { ...stored.settings, ...this.settings } + }; + } + + public async save(data: StoredClient): Promise { + await fs.mkdir(path.dirname(this.file), { recursive: true }); + const temporary = `${this.file}.${randomUUID()}.tmp`; + try { + const handle = await fs.open(temporary, "wx", 0o600); + try { + await handle.writeFile(JSON.stringify(data)); + // Flush the complete value before exposing its name. A crash can + // keep the previous value, but must not expose truncated JSON. + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, this.file); + } finally { + await fs.rm(temporary, { force: true }); + } + } +} diff --git a/frontend/local-client-cli/src/file-watcher.ts b/frontend/local-client-cli/src/file-watcher.ts index c273a412..541192f7 100644 --- a/frontend/local-client-cli/src/file-watcher.ts +++ b/frontend/local-client-cli/src/file-watcher.ts @@ -1,93 +1,54 @@ import Watcher from "watcher"; -import * as path from "path"; -import type { SyncClient, RelativePath } from "sync-client"; +import * as path from "node:path"; +import type { SyncClient } from "sync-client"; import { toUnixPath, matchesGlob } from "./path-utils"; +/** OS events include sync writes and editor temporary-file replacements. They + * wake reconciliation; only a fresh scan determines the current namespace. */ export class FileWatcher { private watcher: Watcher | undefined; - private isRunning = false; - private readonly ignorePatterns: string[]; public constructor( private readonly basePath: string, private readonly client: SyncClient, - ignorePatterns: string[] = [] - ) { - this.ignorePatterns = ignorePatterns; - } + private readonly ignorePatterns: string[] = [] + ) {} public start(): void { - if (this.isRunning) { - return; - } - - this.isRunning = true; - + if (this.watcher) return; this.watcher = new Watcher(this.basePath, { recursive: true, - renameDetection: true, - renameTimeout: 125, + renameDetection: false, ignoreInitial: true, - ignore: (filePath: string): boolean => this.shouldIgnore(filePath) + ignore: (filePath: string): boolean => + this.ignorePatterns.some((pattern) => + matchesGlob(this.toRelativePath(filePath), pattern) + ) }); - - this.watcher.on("add", (filePath: string) => { - this.handleCreate(this.toRelativePath(filePath)); + this.watcher.on("all", (_event: string, filePath: string) => { + void this.client + .syncLocallyUpdatedFile({ + relativePath: this.toRelativePath(filePath) + }) + .catch((error: unknown) => { + this.client.logger.error( + `File notification failed: ${error}` + ); + }); }); - - this.watcher.on("change", (filePath: string) => { - this.handleChange(this.toRelativePath(filePath)); + this.watcher.on("error", (error: Error) => { + this.client.logger.error(`File watcher failed: ${error.message}`); }); - - this.watcher.on("unlink", (filePath: string) => { - this.handleDelete(this.toRelativePath(filePath)); - }); - - this.watcher.on("rename", (oldPath: string, newPath: string) => { - this.handleRename( - this.toRelativePath(oldPath), - this.toRelativePath(newPath) - ); - }); - this.client.logger.info("File watcher started"); } public stop(): void { - if (this.watcher !== undefined) { - this.watcher.close(); - this.watcher = undefined; - } - this.isRunning = false; + this.watcher?.close(); + this.watcher = undefined; this.client.logger.info("File watcher stopped"); } - private shouldIgnore(filePath: string): boolean { - const rel = toUnixPath(path.relative(this.basePath, filePath)); - return this.ignorePatterns.some((pattern) => matchesGlob(rel, pattern)); - } - - private handleCreate(relativePath: RelativePath): void { - this.client.syncLocallyCreatedFile(relativePath); - } - - private handleChange(relativePath: RelativePath): void { - this.client.syncLocallyUpdatedFile({ relativePath }); - } - - private handleDelete(relativePath: RelativePath): void { - this.client.syncLocallyDeletedFile(relativePath); - } - - private handleRename(oldPath: RelativePath, newPath: RelativePath): void { - this.client.logger.info(`File renamed: ${oldPath} -> ${newPath}`); - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - } - - private toRelativePath(absolutePath: string): RelativePath { + private toRelativePath(absolutePath: string): string { return toUnixPath(path.relative(this.basePath, absolutePath)); } } diff --git a/frontend/local-client-cli/src/node-filesystem.test.ts b/frontend/local-client-cli/src/node-filesystem.test.ts index 55b286a4..3cab1147 100644 --- a/frontend/local-client-cli/src/node-filesystem.test.ts +++ b/frontend/local-client-cli/src/node-filesystem.test.ts @@ -11,10 +11,13 @@ test("NodeFileSystemOperations - read and write files", async () => { try { const content = new TextEncoder().encode("Hello, world!"); - await fsOps.write("test.txt", content); + await fsOps.write("test.txt", { content }); - const readContent = await fsOps.read("test.txt"); - assert.equal(new TextDecoder().decode(readContent), "Hello, world!"); + const readContent = await fsOps.readSnapshot("test.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "Hello, world!" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -27,10 +30,13 @@ test("NodeFileSystemOperations - create nested directories with forward slashes" try { const content = new TextEncoder().encode("Nested file"); // Always use forward slashes in API - await fsOps.write("dir1/dir2/test.txt", content); + await fsOps.write("dir1/dir2/test.txt", { content }); - const readContent = await fsOps.read("dir1/dir2/test.txt"); - assert.equal(new TextDecoder().decode(readContent), "Nested file"); + const readContent = await fsOps.readSnapshot("dir1/dir2/test.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "Nested file" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -43,7 +49,9 @@ test("NodeFileSystemOperations - exists with forward slashes", async () => { try { assert.equal(await fsOps.exists("test.txt"), false); - await fsOps.write("test.txt", new TextEncoder().encode("test")); + await fsOps.write("test.txt", { + content: new TextEncoder().encode("test") + }); assert.equal(await fsOps.exists("test.txt"), true); } finally { @@ -56,7 +64,9 @@ test("NodeFileSystemOperations - deleteFile with forward slashes", async () => { const fsOps = new NodeFileSystemOperations(tempDir); try { - await fsOps.write("test.txt", new TextEncoder().encode("test")); + await fsOps.write("test.txt", { + content: new TextEncoder().encode("test") + }); assert.equal(await fsOps.exists("test.txt"), true); await fsOps.deleteFile("test.txt"); @@ -74,15 +84,18 @@ test("NodeFileSystemOperations - rename with forward slashes", async () => { try { const content = new TextEncoder().encode("test content"); - await fsOps.write("old.txt", content); + await fsOps.write("old.txt", { content }); await fsOps.rename("old.txt", "new.txt"); assert.equal(await fsOps.exists("old.txt"), false); assert.equal(await fsOps.exists("new.txt"), true); - const readContent = await fsOps.read("new.txt"); - assert.equal(new TextDecoder().decode(readContent), "test content"); + const readContent = await fsOps.readSnapshot("new.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "test content" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -94,7 +107,7 @@ test("NodeFileSystemOperations - rename to nested path with forward slashes", as try { const content = new TextEncoder().encode("test content"); - await fsOps.write("old.txt", content); + await fsOps.write("old.txt", { content }); await fsOps.rename("old.txt", "dir1/dir2/new.txt"); @@ -105,42 +118,21 @@ test("NodeFileSystemOperations - rename to nested path with forward slashes", as } }); -test("NodeFileSystemOperations - getFileSize", async () => { +test("NodeFileSystemOperations - stat", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); try { const content = new TextEncoder().encode("Hello!"); - await fsOps.write("test.txt", content); + await fsOps.write("test.txt", { content }); - const size = await fsOps.getFileSize("test.txt"); + const size = (await fsOps.stat("test.txt"))!.size; assert.equal(size, content.length); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); -test("NodeFileSystemOperations - atomicUpdateText", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); - const fsOps = new NodeFileSystemOperations(tempDir); - - try { - await fsOps.write("test.txt", new TextEncoder().encode("Hello")); - - const result = await fsOps.atomicUpdateText("test.txt", (current) => ({ - text: current.text + " World", - cursors: [] - })); - - assert.equal(result, "Hello World"); - - const content = await fsOps.read("test.txt"); - assert.equal(new TextDecoder().decode(content), "Hello World"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } -}); - test("NodeFileSystemOperations - handles paths with forward slashes on all platforms", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); @@ -150,11 +142,11 @@ test("NodeFileSystemOperations - handles paths with forward slashes on all platf const testPath = "deep/nested/directory/file.txt"; const content = new TextEncoder().encode("test"); - await fsOps.write(testPath, content); + await fsOps.write(testPath, { content }); assert.equal(await fsOps.exists(testPath), true); - const readContent = await fsOps.read(testPath); - assert.equal(new TextDecoder().decode(readContent), "test"); + const readContent = await fsOps.readSnapshot(testPath); + assert.equal(new TextDecoder().decode(readContent!.content), "test"); await fsOps.deleteFile(testPath); assert.equal(await fsOps.exists(testPath), false); @@ -186,10 +178,9 @@ test("NodeFileSystemOperations - delete does not remove files", async () => { const fsOps = new NodeFileSystemOperations(tempDir); try { - await fsOps.write( - "parent/file.txt", - new TextEncoder().encode("preserve me") - ); + await fsOps.write("parent/file.txt", { + content: new TextEncoder().encode("preserve me") + }); await assert.rejects(async () => fsOps.delete("parent")); assert.equal(await fsOps.exists("parent/file.txt"), true); diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index 5902db8a..dadc8c5f 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -1,239 +1,188 @@ -import * as fs from "fs/promises"; -import type { Dirent } from "fs"; -import * as path from "path"; -import { randomUUID } from "crypto"; -import type { - FileSystemOperations, - RelativePath, - TextWithCursors -} from "sync-client"; -import { toUnixPath } from "./path-utils"; +import assert from "node:assert/strict"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { constants, type Stats } from "node:fs"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +function hasCode(error: unknown, ...codes: string[]): boolean { + return ( + error instanceof Error && + "code" in error && + typeof error.code === "string" && + codes.includes(error.code) + ); +} -// VaultLink's per-vault metadata directory. Holds the persisted sync database -// and the tmp files atomicWrite renames into place; the matching `${VAULTLINK_DIR}/**` -// ignore pattern keeps everything in here invisible to the file watcher. export const VAULTLINK_DIR = ".vaultlink"; +/** Preflights reject unsafe paths and links. This is not a sandbox against a + * hostile process concurrently replacing ancestor directories. */ export class NodeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly basePath: string) {} - - public async listFilesRecursively( - directory: RelativePath | undefined - ): Promise { - const files: RelativePath[] = []; - await this.walkDirectory(directory ?? "", files); - return files; + public constructor(private readonly root: string) {} + public async stat( + relative: string + ): ReturnType { + const info = await this.info(await this.resolve(relative)); + return info + ? { + kind: info.isFile() + ? ("file" as const) + : ("directory" as const), + size: info.size + } + : undefined; } - - public async read(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - return await fs.readFile(fullPath); - } catch (error) { - throw new Error( - `Failed to read file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } + public async exists(relative: string): Promise { + return (await this.stat(relative)) !== undefined; } - - public async write( - relativePath: RelativePath, - content: Uint8Array - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - const dir = path.dirname(fullPath); - - try { - await fs.mkdir(dir, { recursive: true }); - await this.atomicWrite(fullPath, content); - } catch (error) { - throw new Error( - `Failed to write file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async atomicUpdateText( - relativePath: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - - try { - const currentContent = await fs.readFile(fullPath, "utf-8"); - const result = updater({ text: currentContent, cursors: [] }); - await this.atomicWrite(fullPath, result.text, "utf-8"); - return result.text; - } catch (error) { - throw new Error( - `Failed to atomically update file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async getFileSize(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - const stats = await fs.stat(fullPath); - return stats.size; - } catch (error) { - throw new Error( - `Failed to get file size for ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async exists(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.access(fullPath); - return true; - } catch { - return false; - } - } - - public async createDirectory(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.mkdir(fullPath, { recursive: false }); - } catch (error) { - throw new Error( - `Failed to create directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async delete(relativePath: RelativePath): Promise { - if (!relativePath) { - throw new Error("Cannot delete the vault root"); - } - - const fullPath = path.join(this.basePath, relativePath); - try { - await this.deleteDirectoryTree(fullPath); - } catch (error) { - throw new Error( - `Failed to delete directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async deleteFile(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - const entry = await fs.lstat(fullPath).catch((error: unknown) => { - if (this.isMissing(error)) return undefined; + public async readSnapshot( + relative: string + ): Promise { + const absolute = await this.resolve(relative); + const handle = await fs + .open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW) + .catch((error: unknown) => { + if (hasCode(error, "ENOENT", "ENOTDIR")) return undefined; throw error; }); - if (!entry) return; - if (!entry.isFile() || entry.nlink !== 1) { - throw new Error("Cannot unlink a non-regular or linked file"); - } - - await fs.unlink(fullPath).catch((error: unknown) => { - if (!this.isMissing(error)) throw error; - }); - } catch (error) { - throw new Error( - `Failed to delete file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - const oldFullPath = path.join(this.basePath, oldPath); - const newFullPath = path.join(this.basePath, newPath); - const newDir = path.dirname(newFullPath); - + if (!handle) return undefined; try { - await fs.mkdir(newDir, { recursive: true }); - await fs.rename(oldFullPath, newFullPath); - } catch (error) { - throw new Error( - `Failed to rename file from ${oldFullPath} to ${newFullPath}: ${error instanceof Error ? error.message : String(error)}` + const before = await handle.stat({ bigint: true }); + assert( + before.isFile() && before.nlink === 1n, + "Not an independent regular file" ); + const content = new Uint8Array(await handle.readFile()); + const after = await handle.stat({ bigint: true }); + const atPath = await fs.lstat(absolute, { bigint: true }); + assert( + before.ino === after.ino && + after.ino === atPath.ino && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs, + "File changed during read" + ); + return { content }; + } finally { + await handle.close(); } } - - private async atomicWrite( - fullPath: string, - content: Uint8Array | string, - encoding?: BufferEncoding - ): Promise { - const tmpDir = path.join(this.basePath, VAULTLINK_DIR); - await fs.mkdir(tmpDir, { recursive: true }); - const tmpPath = path.join(tmpDir, `atomic-write-${randomUUID()}.tmp`); - try { - await fs.writeFile(tmpPath, content, encoding); - await fs.rename(tmpPath, fullPath); - } catch (error) { - await fs.unlink(tmpPath).catch(() => undefined); - throw error; + public async listFilesRecursively(root = ""): Promise { + const result: string[] = []; + for (const name of await fs.readdir(await this.resolve(root))) { + const relative = root ? `${root}/${name}` : name; + const entry = await this.stat(relative); + assert(entry, `File changed during scan: ${relative}`); + if (entry.kind === "directory") + result.push(...(await this.listFilesRecursively(relative))); + else result.push(relative); } + return result.sort(); } - - private async deleteDirectoryTree(fullPath: string): Promise { - const entry = await fs.lstat(fullPath).catch((error: unknown) => { - if (this.isMissing(error)) return undefined; - throw error; - }); - if (!entry) return; - if (!entry.isDirectory()) { - throw new Error("Cannot delete a regular file as a directory"); - } - - for (const child of await fs.readdir(fullPath, { - withFileTypes: true - })) { - if (!child.isDirectory()) { - throw new Error( - `Directory contains a non-directory entry: ${path.join(fullPath, child.name)}` - ); + public async createDirectory(relative: string): Promise { + await this.resolve(relative); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + const next = path.join(current, component); + try { + await fs.mkdir(next); + } catch (error) { + if (!hasCode(error, "EEXIST")) throw error; } - await this.deleteDirectoryTree(path.join(fullPath, child.name)); + assert((await fs.lstat(next)).isDirectory()); + current = next; } - - await fs.rmdir(fullPath); } - - private isMissing(error: unknown): boolean { - return ( - typeof error === "object" && - error !== null && - "code" in error && - error.code === "ENOENT" + public async write( + relative: string, + snapshot: FileSnapshot + ): Promise { + const destination = await this.resolve(relative); + await this.createDirectory( + path.posix.dirname(relative) === "." + ? "" + : path.posix.dirname(relative) ); + await fs.writeFile(destination, snapshot.content, { + flag: "wx", + mode: 0o600 + }); } - - private async walkDirectory( - relativePath: string, - files: RelativePath[] - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - let entries: Dirent[] = []; - - try { - entries = await fs.readdir(fullPath, { withFileTypes: true }); - } catch (error) { - throw new Error( - `Failed to read directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); + public async rename(from: string, to: string): Promise { + const snapshot = await this.readSnapshot(from); + assert(snapshot, `Missing source: ${from}`); + // Node's rename overwrites destinations on POSIX. Exclusive copy followed + // by unlink leaves ordinary discoverable files if interrupted. + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + assert( + current && + Buffer.from(current.content).equals( + Buffer.from(snapshot.content) + ), + `Source changed during move: ${from}` + ); + await this.deleteFile(from); + } + public async delete(relative: string): Promise { + assert(relative, "Cannot prune root"); + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isDirectory(), "Cannot delete regular file"); + for (const child of await fs.readdir(absolute)) + await this.delete(`${relative}/${child}`); + await fs.rmdir(absolute); + } + public async deleteFile(relative: string): Promise { + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); + await fs.unlink(absolute).catch((error: unknown) => { + if (!hasCode(error, "ENOENT")) throw error; + }); + } + private async resolve(relative: string): Promise { + assert( + relative === "" || + relative + .split("/") + .every( + (p) => + p !== "" && + p !== "." && + p !== ".." && + !p.includes("\\") && + !p.includes("\0") && + (process.platform !== "win32" || !p.includes(":")) + ), + "Unsafe path" + ); + assert( + (await fs.lstat(this.root)).isDirectory(), + "Vault root must be a directory, not a symlink" + ); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + current = path.join(current, component); + const entry = await this.info(current); + if (entry) + assert( + entry.isDirectory() || + (entry.isFile() && entry.nlink === 1), + "Symlink, hardlink or special file" + ); } - - for (const entry of entries) { - const entryName = entry.name; - const entryRelativePath = path.join(relativePath, entryName); - - if (entry.isDirectory()) { - await this.walkDirectory(entryRelativePath, files); - } else if (entry.isFile()) { - // Always return forward slashes - files.push(toUnixPath(entryRelativePath)); - } + return current; + } + private async info(absolute: string): Promise { + try { + return await fs.lstat(absolute); + } catch (error) { + if (hasCode(error, "ENOENT", "ENOTDIR")) return undefined; + throw error; } } } diff --git a/frontend/local-client-cli/webpack.config.js b/frontend/local-client-cli/webpack.config.js index 9226b9dc..d96048e0 100644 --- a/frontend/local-client-cli/webpack.config.js +++ b/frontend/local-client-cli/webpack.config.js @@ -7,6 +7,7 @@ module.exports = { healthcheck: "./src/healthcheck.ts" }, target: "node", + externals: { bufferutil: "bufferutil", "utf-8-validate": "utf-8-validate" }, mode: "production", optimization: { minimize: false diff --git a/frontend/obsidian-plugin/README.md b/frontend/obsidian-plugin/README.md index 68e10a83..de3f3027 100644 --- a/frontend/obsidian-plugin/README.md +++ b/frontend/obsidian-plugin/README.md @@ -1,63 +1,33 @@ -# Obsidian Sample Plugin +# VaultLink Obsidian plugin -This is a sample plugin for Obsidian (https://obsidian.md). +Syncs an Obsidian vault with the VaultLink server, including active editor text, +selections, remote cursors, history and connection status. Use an API v5 server. +The vault configuration directory, Git metadata and trash are ignored by default. -This project uses TypeScript to provide type checking and documentation. -The repo depends on the latest plugin API (obsidian.d.ts) in TypeScript Definition format, which contains TSDoc comments describing what it does. +From `frontend`: -**Note:** The Obsidian API is still in early alpha and is subject to change at any time! +```sh +npm run build -w sync-client -w vault-link-obsidian-plugin +npm run test -w vault-link-obsidian-plugin +``` -This sample plugin demonstrates some of the basic functionality the plugin API can do. +The production bundle is in `obsidian-plugin/dist`: `main.js`, `manifest.json` +and `styles.css`. Copy those files into the vault's plugin directory to install. -- Adds a ribbon icon, which shows a Notice when clicked. -- Adds a command "Open Sample Modal" which opens a Modal. -- Adds a plugin setting tab to the settings page. -- Registers a global click event and output 'click' to the console. -- Registers a global interval which logs 'setInterval' to the console. +Desktop Obsidian shares the CLI's Node filesystem and atomic metadata adapters. +The complete client record remains in the plugin's `data.json`; it includes +settings, sync state, queued logical notifications and the server history +checkpoint. Mobile uses Obsidian's storage and plugin data APIs. Because mobile +writes overwrite existing files, exclusive creation uses a disposable input for +`DataAdapter.copy` in `.vault-link-sync`. This is not a filesystem journal or a +recovery payload. -## First time developing plugins? +Raw create/modify/delete events wake a fresh scan, including events caused by +sync and editor replacements. Obsidian's logical rename event preserves file +and directory move identities. Editor snapshots include unsaved text and +selections; content application updates both disk and the active editor. -Quick starting guide for new plugin devs: - -- Check if [someone already developed a plugin for what you want](https://obsidian.md/plugins)! There might be an existing plugin similar enough that you can partner up with. -- Make a copy of this repo as a template with the "Use this template" button (login to GitHub if you don't see it). -- Clone your repo to a local development folder. For convenience, you can place this folder in your `.obsidian/plugins/your-plugin-name` folder. -- Install NodeJS, then run `npm i` in the command line under your repo folder. -- Run `npm run dev` to compile your plugin from `main.ts` to `main.js`. -- Make changes to `main.ts` (or create new `.ts` files). Those changes should be automatically compiled into `main.js`. -- Reload Obsidian to load the new version of your plugin. -- Enable plugin in settings window. -- For updates to the Obsidian API run `npm update` in the command line under your repo folder. - -## Releasing new releases - -- Update your `manifest.json` with your new version number, such as `1.0.1`, and the minimum Obsidian version required for your latest release. -- Update your `versions.json` file with `"new-plugin-version": "minimum-obsidian-version"` so older versions of Obsidian can download an older version of your plugin that's compatible. -- Create new GitHub release using your new version number as the "Tag version". Use the exact version number, don't include a prefix `v`. See here for an example: https://github.com/obsidianmd/obsidian-sample-plugin/releases -- Upload the files `manifest.json`, `main.js`, `styles.css` as binary attachments. Note: The manifest.json file must be in two places, first the root path of your repository and also in the release. -- Publish the release. - -> You can simplify the version bump process by running `npm version patch`, `npm version minor` or `npm version major` after updating `minAppVersion` manually in `manifest.json`. -> The command will bump version in `manifest.json` and `package.json`, and add the entry for the new version to `versions.json` - -## Adding your plugin to the community plugin list - -- Check the [plugin guidelines](https://docs.obsidian.md/Plugins/Releasing/Plugin+guidelines). -- Publish an initial version. -- Make sure you have a `README.md` file in the root of your repo. -- Make a pull request at https://github.com/obsidianmd/obsidian-releases to add your plugin. - -## How to use - -- Clone this repo. -- Make sure your NodeJS is at least v16 (`node --version`). -- `npm i` or `yarn` to install dependencies. -- `npm run dev` to start compilation in watch mode. - -## Manually installing the plugin - -- Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`. - -## API Documentation - -See https://github.com/obsidianmd/obsidian-api +Adapter tests cover unsaved text, cursor restoration, exclusive creation and +storage failures. The repository E2E runner builds the plugin and tests its +filesystem adapter against the server alongside real CLI processes. These tests +do not launch Obsidian itself. diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index d24e537b..f8d510eb 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "echo \"no tests defined\" && exit 0", + "test": "tsx --test 'src/**/*.test.ts'", "version": "node version-bump.mjs" }, "keywords": [], diff --git a/frontend/obsidian-plugin/src/mobile-file-system.ts b/frontend/obsidian-plugin/src/mobile-file-system.ts new file mode 100644 index 00000000..43869bcd --- /dev/null +++ b/frontend/obsidian-plugin/src/mobile-file-system.ts @@ -0,0 +1,130 @@ +import type { DataAdapter } from "obsidian"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +/** Mobile storage has no Node filesystem. DataAdapter.copy provides its + * documented exclusive-create primitive; writeBinary overwrites existing files. */ +export class MobileFileSystemOperations implements FileSystemOperations { + public constructor(private readonly adapter: DataAdapter) {} + + public async stat(path: string): ReturnType { + const entry = await this.adapter.stat(this.check(path)); + return entry + ? { + kind: + entry.type === "file" + ? ("file" as const) + : ("directory" as const), + size: entry.size + } + : undefined; + } + public async exists(path: string): Promise { + return (await this.stat(path)) !== undefined; + } + public async listFilesRecursively(root = ""): Promise { + const entries = await this.adapter.list(this.check(root)); + const files = [...entries.files]; + for (const folder of entries.folders) + files.push(...(await this.listFilesRecursively(folder))); + return files; + } + public async readSnapshot(path: string): Promise { + this.check(path); + const before = await this.adapter.stat(path); + if (!before) return undefined; + if (before.type !== "file") throw new Error(`Not a file: ${path}`); + const content = new Uint8Array(await this.adapter.readBinary(path)); + const after = await this.adapter.stat(path); + if ( + !after || + before.size !== after.size || + before.mtime !== after.mtime || + before.ctime !== after.ctime + ) + throw new Error(`File changed during read: ${path}`); + return { content }; + } + public async createDirectory(path: string): Promise { + this.check(path); + let current = ""; + for (const part of path.split("/").filter(Boolean)) { + current = current ? `${current}/${part}` : part; + if (!(await this.exists(current))) { + try { + await this.adapter.mkdir(current); + } catch (error) { + if ((await this.stat(current))?.kind !== "directory") + throw error; + } + } + if ((await this.stat(current))?.kind !== "directory") + throw new Error(`Not a directory: ${current}`); + } + } + public async write(path: string, snapshot: FileSnapshot): Promise { + this.check(path); + await this.createDirectory(path.split("/").slice(0, -1).join("/")); + await this.createDirectory(".vault-link-sync"); + const temporary = `.vault-link-sync/${crypto.randomUUID()}.tmp`; + try { + await this.adapter.writeBinary( + temporary, + new Uint8Array(snapshot.content).buffer + ); + await this.adapter.copy(temporary, path); + } finally { + if (await this.adapter.exists(temporary)) + await this.adapter.remove(temporary); + } + } + public async rename(from: string, to: string): Promise { + this.check(from); + this.check(to); + const snapshot = await this.readSnapshot(from); + if (!snapshot) throw new Error(`Missing source: ${from}`); + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + if ( + !current || + current.content.length !== snapshot.content.length || + current.content.some((byte, i) => byte !== snapshot.content[i]) + ) + throw new Error(`Source changed during move: ${from}`); + await this.deleteFile(from); + } + public async deleteFile(path: string): Promise { + const entry = await this.stat(path); + if (!entry) return; + if (entry.kind !== "file") throw new Error(`Not a file: ${path}`); + await this.adapter.remove(path); + } + public async delete(path: string): Promise { + if (!path) throw new Error("Cannot prune root"); + const entry = await this.stat(path); + if (!entry) return; + if (entry.kind !== "directory") + throw new Error(`Not a directory: ${path}`); + const children = await this.adapter.list(path); + if (children.files.length) + throw new Error(`Directory contains files: ${path}`); + for (const folder of children.folders) await this.delete(folder); + await this.adapter.rmdir(path, false); + } + private check(path: string): string { + if ( + path !== "" && + path + .split("/") + .some( + (part) => + !part || + part === "." || + part === ".." || + part.includes("\\") || + part.includes("\0") + ) + ) + throw new Error(`Unsafe path: ${path}`); + return path; + } +} diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.test.ts b/frontend/obsidian-plugin/src/obsidian-file-system.test.ts new file mode 100644 index 00000000..5faf5330 --- /dev/null +++ b/frontend/obsidian-plugin/src/obsidian-file-system.test.ts @@ -0,0 +1,167 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as fs from "node:fs/promises"; +import { constants } from "node:fs"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import type { DataAdapter, Editor, EditorSelection } from "obsidian"; +import { NodeFileSystemOperations } from "../../local-client-cli/src/node-filesystem"; +import { ObsidianFileSystemOperations } from "./obsidian-file-system"; +import { MobileFileSystemOperations } from "./mobile-file-system"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const text = (content: Uint8Array) => new TextDecoder().decode(content); + +test("Obsidian snapshots include unsaved editor text and restore merged selections", async (t) => { + const root = await fs.mkdtemp(path.join(tmpdir(), "vaultlink-editor-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const disk = new NodeFileSystemOperations(root); + await disk.write("note.md", { content: bytes("saved") }); + let value = "unsaved\ntext"; + let selections: EditorSelection[] = [ + { anchor: { line: 0, ch: 2 }, head: { line: 1, ch: 3 } } + ]; + const editor = { + getValue: () => value, + listSelections: () => selections, + setValue: (next: string) => { + value = next; + }, + setSelections: (next: EditorSelection[]) => { + selections = next; + } + } as Editor; + const adapter = new ObsidianFileSystemOperations(disk, () => ({ + path: "note.md", + editor + })); + const snapshot = await adapter.readSnapshot("note.md"); + assert.equal(text(snapshot!.content), value); + assert.deepEqual(snapshot!.cursors, [ + { id: 0, position: 2 }, + { id: 1, position: 11 } + ]); + assert.equal((await adapter.stat("note.md"))!.size, bytes(value).length); + await assert.rejects( + adapter.write("note.md", { content: bytes("collision") }) + ); + assert.equal(value, "unsaved\ntext"); + await adapter.deleteFile("note.md"); + await adapter.write("note.md", { + content: bytes("new\ntext"), + cursors: [ + { id: 1, position: 7 }, + { id: 0, position: 1 } + ] + }); + assert.equal(value, "new\ntext"); + assert.equal(text((await disk.readSnapshot("note.md"))!.content), value); + assert.deepEqual(selections, [ + { anchor: { line: 0, ch: 1 }, head: { line: 1, ch: 3 } } + ]); + value = "edited before rename"; + selections = [{ anchor: { line: 0, ch: 1 }, head: { line: 0, ch: 4 } }]; + await adapter.rename("note.md", "moved.md"); + assert.equal(text((await disk.readSnapshot("moved.md"))!.content), value); + assert.equal(await adapter.readSnapshot("note.md"), undefined); +}); + +// A storage host with Obsidian's documented semantics: writes replace, copies +// reject existing destinations. No runtime Obsidian module is needed in Node. +async function mobile(root: string) { + const disk = new NodeFileSystemOperations(root); + const host = { + stat: async (p: string) => { + const entry = await disk.stat(p); + return entry + ? { + type: entry.kind === "file" ? "file" : "folder", + size: entry.size, + mtime: 1, + ctime: 1 + } + : null; + }, + exists: (p: string) => disk.exists(p), + readBinary: async (p: string) => + new Uint8Array((await disk.readSnapshot(p))!.content).buffer, + writeBinary: (p: string, content: ArrayBuffer) => + fs.writeFile(path.join(root, p), new Uint8Array(content)), + copy: (from: string, to: string) => + fs.copyFile( + path.join(root, from), + path.join(root, to), + constants.COPYFILE_EXCL + ), + mkdir: (p: string) => disk.createDirectory(p), + remove: (p: string) => disk.deleteFile(p), + rmdir: (p: string) => fs.rmdir(path.join(root, p)), + list: async (p: string) => { + const entries = await fs.readdir(path.join(root, p), { + withFileTypes: true + }); + return { + files: entries + .filter((e) => e.isFile()) + .map((e) => (p ? `${p}/${e.name}` : e.name)), + folders: entries + .filter((e) => e.isDirectory()) + .map((e) => (p ? `${p}/${e.name}` : e.name)) + }; + } + } as unknown as DataAdapter; + return { adapter: new MobileFileSystemOperations(host), host }; +} + +test("mobile writes use exclusive copies and never overwrite a competing writer", async (t) => { + const root = await fs.mkdtemp(path.join(tmpdir(), "vaultlink-mobile-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { adapter } = await mobile(root); + await adapter.createDirectory("folder/nested"); + const writes = await Promise.allSettled([ + adapter.write("folder/note.md", { content: bytes("first") }), + adapter.write("folder/note.md", { content: bytes("second") }) + ]); + assert.equal(writes.filter((w) => w.status === "fulfilled").length, 1); + const original = (await adapter.readSnapshot("folder/note.md"))!; + await adapter.write("destination.md", { content: bytes("occupied") }); + await assert.rejects(adapter.rename("folder/note.md", "destination.md")); + assert.deepEqual(await adapter.readSnapshot("folder/note.md"), original); + assert.equal( + text((await adapter.readSnapshot("destination.md"))!.content), + "occupied" + ); + await assert.rejects(adapter.delete("folder")); + await adapter.rename("folder/note.md", "moved.md"); + assert.deepEqual(await adapter.readSnapshot("moved.md"), original); + await adapter.delete("folder"); + assert.equal(await adapter.exists("folder"), false); + assert.deepEqual(await fs.readdir(path.join(root, ".vault-link-sync")), []); + await assert.rejects( + adapter.write("../escape", { content: bytes("bad") }), + /Unsafe/ + ); +}); + +test("mobile scans propagate storage errors and snapshots reject read races", async (t) => { + const root = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-mobile-read-") + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { adapter, host } = await mobile(root); + await adapter.write("note.md", { content: bytes("before") }); + const read = host.readBinary.bind(host); + t.mock.method(host, "readBinary", async (p: string) => { + const content = await read(p); + await fs.writeFile(path.join(root, p), "changed length"); + return content; + }); + await assert.rejects( + adapter.readSnapshot("note.md"), + /changed during read/ + ); + t.mock.method(host, "list", async () => { + throw new Error("storage unavailable"); + }); + await assert.rejects(adapter.listFilesRecursively(), /storage unavailable/); +}); diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index fec25c59..2788ebff 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -1,204 +1,105 @@ -import type { Stat, Vault, Workspace } from "obsidian"; -import { MarkdownView, normalizePath } from "obsidian"; -import type { CursorPosition, TextWithCursors } from "sync-client"; +import type { Editor } from "obsidian"; import { utils, type FileSystemOperations, - type RelativePath + type FileSnapshot } from "sync-client"; import { getSelectionsFromEditor } from "./views/cursors/get-selections-from-editor"; +/** Editor contents and selections are one synchronous snapshot. Disk namespace + * operations remain the responsibility of the platform adapter. */ export class ObsidianFileSystemOperations implements FileSystemOperations { public constructor( - private readonly vault: Vault, - private readonly workspace: Workspace + private readonly disk: FileSystemOperations, + private readonly activeEditor: () => + | { path: string; editor: Editor } + | undefined ) {} - public async listFilesRecursively( - root: RelativePath | undefined - ): Promise { - // Let's implement this by hand because vault.adapter.listAllFiles doesn't always return all files. - const allFiles = []; - const remainingFolders = [root ?? this.vault.getRoot().path]; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - const folder = remainingFolders.pop(); - if (folder == undefined) { - break; - } - - // This would be a very bad idea to sync as it would mess with - // the integrity of the sync database. - if (folder.endsWith(".obsidian/plugins/vault-link/data.json")) { - continue; - } - - const files = await this.vault.adapter.list(normalizePath(folder)); - allFiles.push(...files.files); - remainingFolders.push(...files.folders); - } - - return allFiles; + public async listFilesRecursively(root?: string): Promise { + return this.disk.listFilesRecursively(root); + } + public async exists(path: string): Promise { + return this.disk.exists(path); + } + public async createDirectory(path: string): Promise { + return this.disk.createDirectory(path); + } + public async delete(path: string): Promise { + return this.disk.delete(path); + } + public async deleteFile(path: string): Promise { + return this.disk.deleteFile(path); + } + public async rename(from: string, to: string): Promise { + const snapshot = await this.readSnapshot(from); + if (!snapshot) throw new Error(`Missing source: ${from}`); + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + if ( + !current || + current.content.length !== snapshot.content.length || + current.content.some((byte, i) => byte !== snapshot.content[i]) + ) + throw new Error(`Source changed during move: ${from}`); + await this.deleteFile(from); } - public async read(path: RelativePath): Promise { - path = normalizePath(path); - const view = this.workspace.getActiveViewOfType(MarkdownView); - if (view?.file?.path === path) { - return new TextEncoder().encode(view.editor.getValue()); - } - - return new Uint8Array(await this.vault.adapter.readBinary(path)); + public async stat(path: string): ReturnType { + const entry = await this.disk.stat(path); + const view = this.activeEditor(); + if (entry?.kind === "file" && view?.path === path) + return { + ...entry, + size: new TextEncoder().encode(view.editor.getValue()) + .byteLength + }; + return entry; } - public async write(path: RelativePath, content: Uint8Array): Promise { - path = normalizePath(path); - - const view = this.workspace.getActiveViewOfType(MarkdownView); - if (view?.file?.path === path) { - const position = view.editor.getCursor(); - view.editor.setValue(new TextDecoder().decode(content)); - view.editor.setCursor(position); - return; + public async readSnapshot(path: string): Promise { + // Validate the disk path even when an editor is open. + const entry = await this.disk.stat(path); + if (!entry) return undefined; + const view = this.activeEditor(); + if (entry.kind === "file" && view?.path === path) { + return { + content: new TextEncoder().encode(view.editor.getValue()), + cursors: getSelectionsFromEditor(view.editor).flatMap( + ({ id, start, end }) => [ + { id: 2 * id, position: start }, + { id: 2 * id + 1, position: end } + ] + ) + }; } + return this.disk.readSnapshot(path); + } - return this.vault.adapter.writeBinary( - path, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - content.buffer as ArrayBuffer + public async write(path: string, snapshot: FileSnapshot): Promise { + await this.disk.write(path, snapshot); + const view = this.activeEditor(); + if (view?.path !== path) return; + const text = new TextDecoder().decode(snapshot.content); + view.editor.setValue(text); + const cursors = [...(snapshot.cursors ?? [])].sort( + (a, b) => a.id - b.id ); - } - - public async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - path = normalizePath(path); - - const view = this.workspace.getActiveViewOfType(MarkdownView); - - if (view?.file?.path === path) { - const text = view.editor.getValue(); - - const cursors: CursorPosition[] = getSelectionsFromEditor( - view.editor - ).flatMap(({ id, start: anchor, end: head }) => [ - { - id: 2 * id, - position: anchor - }, - { - id: 2 * id + 1, - position: head - } - ]); - - const result = updater({ + const selections = []; + for (let i = 0; i + 1 < cursors.length; i += 2) { + const anchor = utils.positionToLineAndColumn( text, - cursors + cursors[i].position + ); + const head = utils.positionToLineAndColumn( + text, + cursors[i + 1].position + ); + selections.push({ + anchor: { line: anchor.line, ch: anchor.column }, + head: { line: head.line, ch: head.column } }); - - if (result.text === text) { - return text; - } - - view.editor.setValue(result.text); - - const selections = []; - for (let i = 0; i < result.cursors.length / 2; i++) { - const from = result.cursors[2 * i]; - const to = result.cursors[2 * i + 1]; - const { line: fromLine, column: fromColumn } = - utils.positionToLineAndColumn(result.text, from.position); - - const { line: toLine, column: toColumn } = - utils.positionToLineAndColumn(result.text, to.position); - - selections.push({ - anchor: { line: fromLine, ch: fromColumn }, - head: { line: toLine, ch: toColumn } - }); - } - view.editor.setSelections(selections); - - return result.text; } - - return this.vault.adapter.process( - path, - (text) => - updater({ - text, - cursors: [] - }).text - ); - } - - public async getFileSize(path: RelativePath): Promise { - return (await this.statFile(path)).size; - } - - public async getModificationTime(path: RelativePath): Promise { - return new Date((await this.statFile(path)).mtime); - } - - public async exists(path: RelativePath): Promise { - return this.vault.adapter.exists(normalizePath(path)); - } - - public async createDirectory(path: RelativePath): Promise { - return this.vault.adapter.mkdir(normalizePath(path)); - } - - public async delete(path: RelativePath): Promise { - if (!path) { - throw new Error("Cannot delete the vault root"); - } - await this.deleteDirectoryTree(normalizePath(path)); - } - - public async deleteFile(path: RelativePath): Promise { - path = normalizePath(path); - const entry = await this.vault.adapter.stat(path); - if (!entry) return; - if (entry.type !== "file") { - throw new Error(`Cannot delete non-file: ${path}`); - } - await this.vault.adapter.remove(path); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - return this.vault.adapter.rename(oldPath, newPath); - } - - private async statFile(path: string): Promise { - const file = await this.vault.adapter.stat(normalizePath(path)); - - if (!file) { - throw new Error(`File not found: ${path}`); - } - - return file; - } - - private async deleteDirectoryTree(path: string): Promise { - const entry = await this.vault.adapter.stat(path); - if (!entry) return; - if (entry.type !== "folder") { - throw new Error(`Cannot delete non-directory: ${path}`); - } - - const children = await this.vault.adapter.list(path); - if (children.files.length > 0) { - throw new Error(`Cannot delete non-empty directory: ${path}`); - } - for (const folder of children.folders) { - await this.deleteDirectoryTree(folder); - } - await this.vault.adapter.rmdir(path, false); + if (selections.length) view.editor.setSelections(selections); } } diff --git a/frontend/obsidian-plugin/src/vault-link-plugin.ts b/frontend/obsidian-plugin/src/vault-link-plugin.ts index 2c456b02..de4326f2 100644 --- a/frontend/obsidian-plugin/src/vault-link-plugin.ts +++ b/frontend/obsidian-plugin/src/vault-link-plugin.ts @@ -1,11 +1,16 @@ import type { - MarkdownView, Editor, MarkdownFileInfo, TAbstractFile, WorkspaceLeaf } from "obsidian"; -import { Notice, Plugin, TFile } from "obsidian"; +import { + Notice, + Plugin, + TFile, + MarkdownView, + FileSystemAdapter +} from "obsidian"; import "../manifest.json"; import { HistoryView } from "./views/history/history-view"; import { StatusBar } from "./views/status-bar/status-bar"; @@ -14,11 +19,14 @@ import { StatusDescription } from "./views/status-description/status-description import { SyncClient, rateLimit, - DEFAULT_SETTINGS, Logger, - debugging + debugging, + type FileSystemOperations, + type PersistenceProvider, + type StoredClient } from "sync-client"; import { ObsidianFileSystemOperations } from "./obsidian-file-system"; +import { MobileFileSystemOperations } from "./mobile-file-system"; import { SyncSettingsTab } from "./views/settings/settings-tab"; import { EditorStatusDisplayManager } from "./views/editor-status-display-manager/editor-status-display-manager"; import { remoteCursorsTheme } from "./views/cursors/remote-cursor-theme"; @@ -38,7 +46,7 @@ export default class VaultLinkPlugin extends Plugin { () => Promise >(); - private readonly syncClient: SyncClient | undefined; + private syncClient: SyncClient | undefined; private settingsTab: SyncSettingsTab | undefined; public async onload(): Promise { @@ -53,14 +61,19 @@ export default class VaultLinkPlugin extends Plugin { // eslint-disable-next-line (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = this; - const client = await this.createSyncClient(); + this.register(() => { + // eslint-disable-next-line + (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; + }); + const client = (this.syncClient = await this.createSyncClient()); this.registerObsidianExtensions(client); this.registerEditorEvents(client); this.register(async () => { - await client.waitUntilFinished(); + this.syncClient = undefined; + this.rateLimitedUpdatesPerFile.clear(); await client.destroy(); }); @@ -116,21 +129,55 @@ export default class VaultLinkPlugin extends Plugin { } private async createSyncClient(): Promise { - DEFAULT_SETTINGS.ignorePatterns.push( - ".obsidian/**", - ".git/**", - ".trash/**", - "**/.DS_Store" - ); - - const client = await SyncClient.create({ - fs: new ObsidianFileSystemOperations( - this.app.vault, - this.app.workspace - ), - persistence: { + const { vault, workspace } = this.app; + let disk: FileSystemOperations; + let persistence: PersistenceProvider; + if (vault.adapter instanceof FileSystemAdapter) { + // Keep Node modules unevaluated on mobile. + const { NodeFileSystemOperations } = + require("../../local-client-cli/src/node-filesystem") as typeof import("../../local-client-cli/src/node-filesystem"); + const { ClientPersistence } = + require("../../local-client-cli/src/client-persistence") as typeof import("../../local-client-cli/src/client-persistence"); + const root = vault.adapter.getBasePath(); + disk = new NodeFileSystemOperations(root); + persistence = new ClientPersistence( + `${root}/${this.manifest.dir ?? `${vault.configDir}/plugins/${this.manifest.id}`}/data.json`, + {} + ); + } else { + disk = new MobileFileSystemOperations(vault.adapter); + persistence = { load: this.loadData.bind(this), save: this.saveData.bind(this) + }; + } + const client = await SyncClient.create({ + fs: new ObsidianFileSystemOperations(disk, () => { + const view = workspace.getActiveViewOfType(MarkdownView); + return view?.file + ? { path: view.file.path, editor: view.editor } + : undefined; + }), + persistence: { + load: async () => { + const stored = await persistence.load(); + return { + ...stored, + settings: { + ...stored?.settings, + ignorePatterns: [ + ...new Set([ + ...(stored?.settings?.ignorePatterns ?? []), + `${vault.configDir}/**`, + ".git/**", + ".trash/**", + "**/.DS_Store" + ]) + ] + } + }; + }, + save: (stored) => persistence.save(stored) }, ...(IS_DEBUG_BUILD ? { @@ -141,7 +188,7 @@ export default class VaultLinkPlugin extends Plugin { }); if (IS_DEBUG_BUILD) { - debugging.logToConsole(client.logger); + debugging.logToConsole(client); } return client; @@ -195,11 +242,6 @@ export default class VaultLinkPlugin extends Plugin { this.register(() => { editorStatusDisplayManager.dispose(); }); - - this.register(() => { - // eslint-disable-next-line - (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; - }); } private addRibbonIcons(): void { @@ -232,7 +274,7 @@ export default class VaultLinkPlugin extends Plugin { ), this.app.vault.on("create", (file: TAbstractFile) => { if (file instanceof TFile) { - client.syncLocallyCreatedFile(file.path); + void this.rateLimitedUpdate(file.path, client); } }), this.app.vault.on("modify", async (file: TAbstractFile) => { @@ -241,17 +283,17 @@ export default class VaultLinkPlugin extends Plugin { } }), this.app.vault.on("delete", (file: TAbstractFile) => { - client.syncLocallyDeletedFile(file.path); + void this.rateLimitedUpdate(file.path, client); }), + // Vault.rename is a logical move (including folders). Engine file + // moves use copy/unlink, so they cannot generate this event. this.app.vault.on( "rename", (file: TAbstractFile, oldPath: string) => { - if (file instanceof TFile) { - client.syncLocallyUpdatedFile({ - oldPath, - relativePath: file.path - }); - } + void client.syncLocallyUpdatedFile({ + oldPath, + relativePath: file.path + }); } ) ].forEach((event) => { @@ -267,7 +309,8 @@ export default class VaultLinkPlugin extends Plugin { this.rateLimitedUpdatesPerFile.set( path, rateLimit(async () => { - client.syncLocallyUpdatedFile({ + if (this.syncClient !== client) return; + await client.syncLocallyUpdatedFile({ relativePath: path }); }, MIN_WAIT_BETWEEN_UPDATES_IN_MS) diff --git a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts index 409402a4..bfebfbf4 100644 --- a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts +++ b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts @@ -36,7 +36,7 @@ export function renderCursorsInFileExplorer( (parent) => { cursors.forEach((cursor) => { cursor.documentsWithCursors.forEach((document) => { - if (document.relativePath.startsWith(key)) { + if (document.relative_path.startsWith(key)) { parent.appendChild( createSpan({ text: cursor.userName, diff --git a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts index 4200a72a..d6650dcb 100644 --- a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts +++ b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts @@ -61,7 +61,7 @@ export class RemoteCursorsPluginValue implements PluginValue { return clientCursors.flatMap((cursor) => cursor.cursors.map((span) => ({ name: client.userName, - path: cursor.relativePath, + path: cursor.relative_path, deviceId: client.deviceId, isOutdated: client.isOutdated, span: { ...span } diff --git a/frontend/obsidian-plugin/src/views/status-description/status-description.ts b/frontend/obsidian-plugin/src/views/status-description/status-description.ts index 6d8d74fe..53fea486 100644 --- a/frontend/obsidian-plugin/src/views/status-description/status-description.ts +++ b/frontend/obsidian-plugin/src/views/status-description/status-description.ts @@ -88,7 +88,7 @@ export class StatusDescription { text: ` and has indexed approximately ` }); container.createSpan({ - text: `${this.syncClient.syncedDocumentCount}`, + text: `${this.syncClient.documentCount}`, cls: "number" }); container.createSpan({ diff --git a/frontend/obsidian-plugin/tsconfig.json b/frontend/obsidian-plugin/tsconfig.json index 7ec2a9cd..dcad6394 100644 --- a/frontend/obsidian-plugin/tsconfig.json +++ b/frontend/obsidian-plugin/tsconfig.json @@ -4,6 +4,7 @@ "module": "ESNext", "target": "ES2023", "strict": true, + "skipLibCheck": true, "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, "lib": ["DOM", "ES2024"] diff --git a/frontend/obsidian-plugin/webpack.config.js b/frontend/obsidian-plugin/webpack.config.js index 12844fd7..0d287eb2 100644 --- a/frontend/obsidian-plugin/webpack.config.js +++ b/frontend/obsidian-plugin/webpack.config.js @@ -12,6 +12,11 @@ module.exports = (env, argv) => ({ ignored: "**/node_modules" }, externals: { + "node:assert/strict": "commonjs node:assert/strict", + "node:fs/promises": "commonjs node:fs/promises", + "node:fs": "commonjs node:fs", + "node:path": "commonjs node:path", + "node:crypto": "commonjs node:crypto", obsidian: "commonjs obsidian", electron: "commonjs electron", "@codemirror/autocomplete": "commonjs @codemirror/autocomplete", diff --git a/frontend/package.json b/frontend/package.json index 9e33d1b0..17a03fb3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,7 +20,7 @@ "test": "npm run test --workspaces", "check:harness": "tsc --noEmit -p test-support/tsconfig.json && tsc --noEmit -p deterministic-tests/tsconfig.json && tsc --noEmit -p test-client/tsconfig.json", "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/file-application.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", - "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/restart-reconciliation.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts'", + "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/restart-reconciliation.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts' 'test-support/production-adapters.test.ts'", "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md index c92671db..2373c040 100644 --- a/frontend/sync-client/README.md +++ b/frontend/sync-client/README.md @@ -9,7 +9,9 @@ resolving, including while offline. Shutdown waits for those saves. Content-upda notifications only wake reconciliation; scans read the latest bytes, including after restart. Notifications describe logical external changes: adapters suppress self-generated create/delete/move notifications, and report atomic editor saves -as content updates. This distinguishes a deliberate delete/recreate from an edit. +as content updates. This distinguishes a deliberate delete/recreate from an edit +when the host supplies reliable logical events. Raw OS watchers instead send +content-update hints for every event and let scans infer the current namespace. Use `waitUntilFinished()` to await the active attempt; errors leave pending requests saved and are reported to the caller. Transient failures retry; authentication and protocol errors wait for an explicit wake or settings change. @@ -82,7 +84,9 @@ Their failures are reported, and remaining observers continue receiving updates. ## Required adapter contract `FileSystemOperations` and `PersistenceProvider` define the adapter operations. -The Obsidian and CLI adapters still need to adopt the current core interface. +The CLI and desktop Obsidian share a Node filesystem adapter and atomic full-record +metadata persistence. Mobile Obsidian uses its host storage APIs. The E2E runner +builds and tests both integrations, including actual CLI processes. The sync client does not coordinate separate processes accessing the same vault. The filesystem adapter must provide complete scans, coherent snapshots, recursive @@ -92,8 +96,10 @@ file removal. Reject links, special files and paths escaping the vault. An unrea file or directory must fail the scan rather than look like a deletion. Optional editor cursor metadata is applied with file content. -User-file writes have no power-loss durability requirement. There is no filesystem -journal, hidden staging area, replay, backup archive, or flush protocol. Swaps and +User-file writes have no power-loss durability requirement. The engine has no +filesystem journal, recovery staging area, replay, backup archive, or flush protocol. +Mobile storage needs a disposable scratch file to use its exclusive-copy primitive; +that file is not used for recovery. Swaps and cycles temporarily move occupied files to visible conflict names; a fresh scan can discover them if application stops partway through. Partial writes and edits made before, during or after an interruption are treated as current disk content. diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index a649d067..9cdf64d2 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -1,5 +1,4 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; -export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; export const SUPPORTED_API_VERSION = 5; diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index e7476470..8d1db5be 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -36,7 +36,7 @@ export type { } from "./errors/errors"; export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; export { DocumentSyncStatus } from "./types/document-sync-status"; -export { SyncClient } from "./sync-client"; +export { SyncClient, type StoredClient } from "./sync-client"; export type { TextWithCursors, CursorPosition } from "reconcile-text"; export const debugging = { diff --git a/frontend/sync-client/src/services/server-config.test.ts b/frontend/sync-client/src/services/server-config.test.ts new file mode 100644 index 00000000..d82f8f06 --- /dev/null +++ b/frontend/sync-client/src/services/server-config.test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { ServerConfig } from "./server-config"; +import { SyncService } from "./sync-service"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { SUPPORTED_API_VERSION } from "../consts"; + +const config = { + supportedApiVersion: SUPPORTED_API_VERSION, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" +}; +const make = (fetch: typeof globalThis.fetch) => + new ServerConfig( + new SyncService( + "device", + new Settings( + new Logger(), + { remoteUri: "http://test" }, + async () => {} + ), + fetch + ) + ); + +test("configuration shares one validated request and retries failures", async () => { + let calls = 0; + const server = make(async () => { + if (++calls === 1) throw new Error("offline"); + return Response.json(config); + }); + await assert.rejects(server.getConfig(), /offline/); + const [a, b] = await Promise.all([server.getConfig(), server.getConfig()]); + assert.equal(a, b); + assert.equal(calls, 2); + await server.getConfig(); + assert.equal(calls, 2); +}); + +test("connection checks are independent and validate authentication and version", async () => { + let response = config; + const server = make(async () => Response.json(response)); + const cached = await server.getConfig(); + response = { ...config, isAuthenticated: false }; + assert.equal((await server.checkConnection()).isSuccessful, false); + response = { ...config, supportedApiVersion: SUPPORTED_API_VERSION + 1 }; + assert.equal((await server.checkConnection()).isSuccessful, false); + assert.equal(await server.getConfig(), cached); + server.reset(); + await assert.rejects(server.getConfig(), /Unsupported API/); + response = config; + assert.deepEqual(await server.getConfig(), config); +}); + +for (const fail of [false, true]) { + test(`reset fences a late ${fail ? "failed" : "successful"} configuration request`, async () => { + const first = Promise.withResolvers(); + let calls = 0; + const server = make(async () => + ++calls === 1 ? first.promise : Response.json(config) + ); + const old = server.getConfig(); + server.reset(); + const current = await server.getConfig(); + if (fail) { + first.reject(new Error("old request")); + await assert.rejects(old, /old request/); + } else { + first.resolve( + Response.json({ ...config, mergeableFileExtensions: ["txt"] }) + ); + await old; + } + assert.equal(await server.getConfig(), current); + assert.equal(calls, 2); + }); +} diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts index fcd8adb9..102cf425 100644 --- a/frontend/sync-client/src/services/server-config.ts +++ b/frontend/sync-client/src/services/server-config.ts @@ -4,7 +4,6 @@ import { ServerVersionMismatchError } from "../errors/errors"; import type { SyncService } from "./sync-service"; -import type { PingResponse } from "./types/PingResponse"; export interface ServerConfigData { mergeableFileExtensions: string[]; @@ -13,8 +12,7 @@ export interface ServerConfigData { } export class ServerConfig { - private response: Promise | undefined; - private config: ServerConfigData | undefined; + private config: Promise | undefined; public constructor(private readonly syncService: SyncService) {} @@ -41,29 +39,16 @@ export class ServerConfig { await this.getConfig(); } - public async checkConnection(forceUpdate = false): Promise<{ + public async checkConnection(): Promise<{ isSuccessful: boolean; message: string; }> { try { - let { response } = this; - if (!response || forceUpdate) { - response = this.response = this.syncService.ping(); - } - - const result: PingResponse = await response; // it must be defined, otherwise we would have thrown above - this.config = result; - - if (result.isAuthenticated) { - return { - isSuccessful: true, - message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` - }; - } - + const result = await this.syncService.ping(); + ServerConfig.validateConfig(result); return { - isSuccessful: false, - message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` + isSuccessful: true, + message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` }; } catch (e) { return { @@ -75,22 +60,22 @@ export class ServerConfig { public async getConfig(): Promise { if (!this.config) { - this.response ??= this.syncService.ping(); - try { - this.config = await this.response; - } catch (error) { - this.response = undefined; - throw error; - } + const pending = this.syncService + .ping() + .then((config) => { + ServerConfig.validateConfig(config); + return config; + }) + .catch((error: unknown) => { + if (this.config === pending) this.config = undefined; + throw error; + }); + this.config = pending; } - - ServerConfig.validateConfig(this.config); - return this.config; } public reset(): void { - this.response = undefined; this.config = undefined; } } diff --git a/frontend/sync-client/src/services/sync-service.test.ts b/frontend/sync-client/src/services/sync-service.test.ts index d4f72b47..bf56fec4 100644 --- a/frontend/sync-client/src/services/sync-service.test.ts +++ b/frontend/sync-client/src/services/sync-service.test.ts @@ -144,3 +144,38 @@ test("pause waits for a checkpoint save already in progress", async () => { await assert.rejects(request, SyncResetError); assert.equal(checkpoint, "1:head"); }); + +test("history saves only new checkpoints, including a changed token at the same event ID", async () => { + let checkpoint: string | undefined; + let responseCheckpoint = "1:first"; + const saved: (string | undefined)[] = []; + const service = new SyncService( + "device", + settings(), + async () => + Response.json( + {}, + { headers: { "X-Vault-Link-History": responseCheckpoint } } + ), + { + get: () => checkpoint, + save: async (value) => { + checkpoint = value; + saved.push(value); + } + } + ); + service.resume(); + for (const value of [ + "1:first", + "1:first", + "2:next", + "1:first", + "2:replacement", + "2:replacement" + ]) { + responseCheckpoint = value; + await service.events(0); + } + assert.deepEqual(saved, ["1:first", "2:next", "2:replacement"]); +}); diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index cf918151..24bac853 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -131,6 +131,7 @@ export class SyncService { this.history !== undefined ) { const previous = this.history.get(); + if (checkpoint === previous) return; if ( previous === undefined || previous === "" || diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 8660c0b1..7ea60cf1 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -28,7 +28,7 @@ import { type LocalChange } from "./sync-operations/local-changes"; -type StoredClient = Partial<{ +export type StoredClient = Partial<{ settings: Partial; database: Partial; localChanges: ( @@ -372,7 +372,7 @@ export class SyncClient { } public async checkConnection() { - const result = await this.serverConfig.checkConnection(true); + const result = await this.serverConfig.checkConnection(); return { isSuccessful: result.isSuccessful, serverMessage: result.message, diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 8a123c04..47ba91d0 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -340,18 +340,17 @@ export class Syncer { if (this.database.state.pending) { await this.finishPending(); } - await this.scan(); const batch = await this.service.events( this.database.state.lastSeenUpdateId ); await this.incorporateEventBatch(batch); - await this.scan(); // Previously excluded content becomes eligible after a setting // change. Its saved obligation survives advancing the cursor. if (Object.keys(this.database.state.excluded ?? {}).length) await this.incorporateFileManifest( this.database.state.fileManifest ); + else await this.scan(); for (const head of Object.values( this.database.state.remoteHeads )) { @@ -521,7 +520,6 @@ export class Syncer { bootstrap: vaultSnapshot }); } - await this.scan(vaultSnapshot); await this.incorporateFileManifest( vaultSnapshot.fileManifest, vaultSnapshot.headEventId, @@ -531,6 +529,7 @@ export class Syncer { private async scan(initial?: VaultSnapshot): Promise { await this.flushLocalChanges(); + const hadChanges = this.changes.length > 0; this.unsyncablePaths = await scanLocalFiles( { next: this.next(), @@ -544,7 +543,8 @@ export class Syncer { initial ); this.hasScanned = true; - await this.localChangeStorage.save(structuredClone(this.changes)); + if (hadChanges) + await this.localChangeStorage.save(structuredClone(this.changes)); } private async incorporateEventBatch(batch: EventBatch): Promise { diff --git a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts index 51ad41c1..7429cdf8 100644 --- a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts +++ b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts @@ -1,66 +1,32 @@ -// Implements an in-memory fixed-size cache for document contents, - import type { VaultUpdateId } from "../../persistence/database"; -// Doubly-linked list node for O(1) LRU operations -class LRUNode { - public constructor( - public key: VaultUpdateId, - public value: Uint8Array, - public prev: LRUNode | null = null, - public next: LRUNode | null = null - ) {} -} - -// evicting the least recently used documents when the size limit is exceeded. +/** Map insertion order tracks the least recently used document first. */ export class FixedSizeDocumentCache { - private currentSizeInBytes: number; - private readonly cache: Map; - private head: LRUNode | null; // Least recently used - private tail: LRUNode | null; // Most recently used + private currentSizeInBytes = 0; + private readonly cache = new Map(); - public constructor(private maxSizeInBytes: number) { - this.currentSizeInBytes = 0; - this.cache = new Map(); - this.head = null; - this.tail = null; - } + public constructor(private maxSizeInBytes: number) {} public get(updateId: VaultUpdateId): Uint8Array | undefined { - const node = this.cache.get(updateId); - if (node) { - this.moveToTail(node); - return node.value; + const content = this.cache.get(updateId); + if (content !== undefined) { + this.cache.delete(updateId); + this.cache.set(updateId, content); } - - return undefined; + return content; } public put(updateId: VaultUpdateId, content: Uint8Array): void { - if (content.byteLength > this.maxSizeInBytes) { - // Document is too large to fit in the cache - return; - } - - // If the document is already in the cache, update it - const existingNode = this.cache.get(updateId); - if (existingNode != null) { - this.currentSizeInBytes -= existingNode.value.byteLength; - this.removeNode(existingNode); - this.cache.delete(updateId); - } - - const newNode = new LRUNode(updateId, content); - this.cache.set(updateId, newNode); - this.addToTail(newNode); + if (content.byteLength > this.maxSizeInBytes) return; + this.currentSizeInBytes -= this.cache.get(updateId)?.byteLength ?? 0; + this.cache.delete(updateId); + this.cache.set(updateId, content); this.currentSizeInBytes += content.byteLength; this.fitBelowMaxSize(); } public reset(): void { this.cache.clear(); - this.head = null; - this.tail = null; this.currentSizeInBytes = 0; } @@ -70,50 +36,10 @@ export class FixedSizeDocumentCache { } private fitBelowMaxSize(): void { - // Evict least recently used documents if over size limit - while (this.currentSizeInBytes > this.maxSizeInBytes && this.head) { - const lruNode = this.head; - this.removeNode(lruNode); - this.cache.delete(lruNode.key); - this.currentSizeInBytes -= lruNode.value.byteLength; + for (const [id, content] of this.cache) { + if (this.currentSizeInBytes <= this.maxSizeInBytes) break; + this.cache.delete(id); + this.currentSizeInBytes -= content.byteLength; } } - - private removeNode(node: LRUNode): void { - if (node.prev) { - node.prev.next = node.next; - } else { - this.head = node.next; - } - - if (node.next) { - node.next.prev = node.prev; - } else { - this.tail = node.prev; - } - - node.prev = null; - node.next = null; - } - - private addToTail(node: LRUNode): void { - node.prev = this.tail; - node.next = null; - - if (this.tail) { - this.tail.next = node; - } - - this.tail = node; - - this.head ??= node; - } - - private moveToTail(node: LRUNode): void { - if (node === this.tail) { - return; - } - this.removeNode(node); - this.addToTail(node); - } } diff --git a/frontend/test-support/README.md b/frontend/test-support/README.md index 824714ab..6b5d3cb8 100644 --- a/frontend/test-support/README.md +++ b/frontend/test-support/README.md @@ -13,7 +13,8 @@ Test-only code shared by the deterministic and seeded clients. | `restart-reconciliation.test.ts` | Interrupted file application followed by user edits, two-client/server convergence and a second restart | | `server-restore.test.ts` | History checkpoints, server backup restores and interrupted metadata resets | | `content-limits.test.ts`, `protocol-audit.test.ts` | Content validation and API invariants | -| `real-disk.test.ts` | Host filesystem exclusive create/rename, unsafe links, and rename cycles | +| `real-disk.test.ts` | Production Node adapter exclusive create/move, unsafe links, and rename cycles | +| `production-adapters.test.ts` | Actual CLI processes, raw watchers, Obsidian adapter, complete metadata and restart after SIGKILL | The filesystem interruption tests assert the supported contract: complete metadata remains usable, scans observe the current files, and a second scan is stable. @@ -30,9 +31,12 @@ after replacement. Abandoned filesystem sessions are fenced against later writes `crash-worker.ts` runs a complete client in a worker, which the parent terminates at accepted-response checkpoints without graceful shutdown. A new runtime receives -the saved disk and metadata. The separate real-disk fixture uses a test-only native -exclusive-rename helper and ordinary writes without flushes. It is not a production -adapter or a sandbox against hostile concurrent changes to ancestor directories. +the saved disk and metadata. Real-disk tests exercise the production Node adapter. +The production integration suite launches built CLI processes and an Obsidian +filesystem adapter against a real server. Obsidian unit tests exercise unsaved editor +text, selections and mobile storage contracts; they do not launch the Obsidian app. +Filesystem path checks are not a sandbox against hostile concurrent changes to +ancestor directories. Scripted and seeded suites run multiple clients against real servers. They check visible bytes, UUID mappings, canonical server state, content heads and event cursors. @@ -46,7 +50,8 @@ Run from the repository root: E2E_WORKERS=32 E2E_SEED=12 E2E_TIMEOUT_SECONDS=9000 E2E_ITERATIONS=100 scripts/e2e.sh ``` -The command rebuilds peers, runs unit/harness/protocol/scripted suites, and then +The command rebuilds peers and both production integrations, runs core and adapter +unit tests plus harness/protocol/scripted suites, and then runs seeds 12 through 43. `E2E_ARTIFACTS` optionally selects the artifact directory. The runner tests also verify subprocess exit codes, log errors, spawn failures, timeouts, workspace dependency consistency and CI artifact wiring. diff --git a/frontend/test-support/engine-regressions.test.ts b/frontend/test-support/engine-regressions.test.ts index 81995ee6..f75aee62 100644 --- a/frontend/test-support/engine-regressions.test.ts +++ b/frontend/test-support/engine-regressions.test.ts @@ -3,7 +3,7 @@ import { PermanentSyncError } from "../sync-client/src/errors/errors"; import { test } from "node:test"; import { Database, emptyState } from "../sync-client/src/persistence/database"; import { Syncer } from "../sync-client/src/sync-operations/syncer"; -import { SyncClient } from "../sync-client/src/sync-client"; +import { SyncClient, type StoredClient } from "../sync-client/src/sync-client"; import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; import { allocatePortablePath } from "../sync-client/src/utils/portable-path"; import { MemoryDisk, MemoryPersistence } from "./storage"; @@ -83,7 +83,7 @@ test("legacy content notifications preserve the acknowledged namespace prefix on .hash }; initial.lastAppliedLocalChangeId = "consumed-update"; - const saved = { + const saved: StoredClient = { database: initial, settings: { remoteUri: "http://offline.test", vaultName: "test" }, localChangesVaultKey: initial.vaultKey, diff --git a/frontend/test-support/native-fs.c b/frontend/test-support/native-fs.c deleted file mode 100644 index 560c79c6..00000000 --- a/frontend/test-support/native-fs.c +++ /dev/null @@ -1,27 +0,0 @@ -/* Test-only POSIX primitives missing from Node's filesystem API. A preflight - * exists() followed by fs.rename() is not an exclusive rename. */ -#define _GNU_SOURCE -#include -#include -#include -#include -#if defined(__linux__) -#include -#endif - -int main(int argc, char **argv) { - if (argc == 4 && strcmp(argv[1], "rename") == 0) { - int result; -#if defined(__APPLE__) - result = renamex_np(argv[2], argv[3], RENAME_EXCL); -#elif defined(__linux__) - result = syscall(SYS_renameat2, AT_FDCWD, argv[2], AT_FDCWD, argv[3], 1 /* RENAME_NOREPLACE */); -#else -#error "The real filesystem tests require macOS or Linux" -#endif - if (result != 0) { perror("exclusive rename"); return 1; } - return 0; - } - fprintf(stderr, "usage: native-fs rename FROM TO\n"); - return 2; -} diff --git a/frontend/test-support/production-adapters.test.ts b/frontend/test-support/production-adapters.test.ts new file mode 100644 index 00000000..64832b8f --- /dev/null +++ b/frontend/test-support/production-adapters.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import { Logger, SyncClient } from "sync-client"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { ClientPersistence } from "../local-client-cli/src/client-persistence"; +import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesystem"; +import { ObsidianFileSystemOperations } from "../obsidian-plugin/src/obsidian-file-system"; +import { getJson, type CanonicalSnapshot } from "./canonical"; + +const root = path.resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const pause = () => new Promise((resolve) => setTimeout(resolve, 50)); +async function until(check: () => Promise, message: string) { + const deadline = Date.now() + 15_000; + while (!(await check())) { + assert(Date.now() < deadline, message); + await pause(); + } +} +async function contents(file: string) { + try { + return await fs.readFile(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return undefined; + throw error; + } +} +async function stop(child: ChildProcess, signal: NodeJS.Signals) { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill(signal); + await exited; +} + +test( + "production CLI watchers, Obsidian adapter, and full metadata survive process replacement", + { timeout: 90_000 }, + async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-production-") + ); + const server = new ServerControl( + path.join(root, "sync-server/target/release/sync_server"), + path.join(root, "sync-server/config-e2e.yml"), + new Logger() + ); + const children: ChildProcess[] = []; + let obsidian: SyncClient | undefined; + t.after(async () => { + await obsidian?.destroy(); + await Promise.all(children.map((child) => stop(child, "SIGKILL"))); + await server.stop(); + await fs.rm(directory, { recursive: true, force: true }); + }); + await server.start(); + const a = path.join(directory, "a"), + b = path.join(directory, "b"), + c = path.join(directory, "obsidian"); + await Promise.all([a, b, c].map((p) => fs.mkdir(p))); + await fs.writeFile(path.join(a, "note.md"), "initial\n"); + async function launch(vault: string) { + const child = spawn( + process.execPath, + [ + path.join(root, "frontend/local-client-cli/dist/cli.js"), + "-l", + vault, + "-r", + server.remoteUri, + "-t", + token, + "-v", + "production", + "--ignore-pattern", + "**/*.tmp" + ], + { stdio: ["ignore", "pipe", "pipe"] } + ); + children.push(child); + let log = ""; + child.stdout!.on("data", (data: Buffer) => { + log += data.toString(); + }); + child.stderr!.on("data", (data: Buffer) => { + log += data.toString(); + }); + await until(async () => { + assert( + child.exitCode === null && child.signalCode === null, + `CLI exited: ${log}` + ); + return log.includes("File watcher started"); + }, `CLI did not start: ${log}`); + return child; + } + let first = await launch(a); + await launch(b); + await until( + async () => + (await contents(path.join(b, "note.md"))) === "initial\n", + "CLI initial download failed" + ); + const persistence = (vault: string) => + new ClientPersistence( + path.join(vault, ".vaultlink/sync-data.json"), + {} + ); + const original = (await persistence(a).load()).database!; + const [documentId] = Object.keys(original.local!); + assert(documentId); + + const settings = { + remoteUri: server.remoteUri, + token, + vaultName: "production", + isSyncEnabled: true, + enableTelemetry: false, + ignorePatterns: [".vaultlink/**"], + syncIntervalMs: 0 + }; + obsidian = await SyncClient.create({ + fs: new ObsidianFileSystemOperations( + new NodeFileSystemOperations(c), + () => undefined + ), + persistence: new ClientPersistence( + path.join(c, ".vaultlink/sync-data.json"), + settings + ) + }); + await obsidian.start(); + await obsidian.waitUntilFinished(); + assert.equal(await contents(path.join(c, "note.md")), "initial\n"); + await fs.writeFile( + path.join(a, "replacement.tmp"), + "atomic editor save\n" + ); + await fs.rename( + path.join(a, "replacement.tmp"), + path.join(a, "note.md") + ); + await until( + async () => + (await contents(path.join(b, "note.md"))) === + "atomic editor save\n" && + (await contents(path.join(c, "note.md"))) === + "atomic editor save\n", + "Editor replacement did not converge" + ); + assert.equal( + (await persistence(a).load()).database!.local![documentId], + "note.md" + ); + await fs.rename(path.join(b, "note.md"), path.join(b, "renamed.md")); + await until( + async () => + (await contents(path.join(a, "renamed.md"))) === + "atomic editor save\n" && + (await contents(path.join(a, "note.md"))) === undefined, + "CLI rename did not converge" + ); + await until( + async () => + (await persistence(a).load()).database!.local![documentId] === + "renamed.md", + "Rename changed document identity" + ); + const saved = await persistence(a).load(); + assert( + saved.historyCheckpoint?.checkpoint, + "CLI dropped the history checkpoint" + ); + await stop(first, "SIGKILL"); + await fs.writeFile(path.join(a, "renamed.md"), "edited after kill\n"); + first = await launch(a); + await until( + async () => + (await contents(path.join(b, "renamed.md"))) === + "edited after kill\n" && + (await contents(path.join(c, "renamed.md"))) === + "edited after kill\n", + "Restart failed to reconcile current files" + ); + const snapshot = await getJson( + `${server.remoteUri}/vaults/production/vault-snapshot`, + token + ); + assert.deepEqual(snapshot.fileManifest.entries, { + [documentId]: "renamed.md" + }); + await fs.unlink(path.join(b, "renamed.md")); + await until( + async () => + (await contents(path.join(a, "renamed.md"))) === undefined && + (await contents(path.join(c, "renamed.md"))) === undefined, + "Delete did not converge" + ); + await stop(first, "SIGTERM"); + assert.equal(first.exitCode, 0, "CLI shutdown failed"); + } +); diff --git a/frontend/test-support/real-disk.test.ts b/frontend/test-support/real-disk.test.ts index 1819f632..63e5c0de 100644 --- a/frontend/test-support/real-disk.test.ts +++ b/frontend/test-support/real-disk.test.ts @@ -3,9 +3,7 @@ import { test } from "node:test"; import * as fs from "node:fs/promises"; import * as path from "node:path"; import { tmpdir } from "node:os"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { RealDisk } from "./real-disk"; +import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesystem"; import { Database, emptyState } from "../sync-client/src/persistence/database"; import { FileOperations } from "../sync-client/src/file-operations/file-operations"; import type { ServerConfig } from "../sync-client/src/services/server-config"; @@ -21,16 +19,7 @@ test( try { const root = path.join(directory, "vault"); await fs.mkdir(root); - const helper = path.join(directory, "native-fs"); - await promisify(execFile)("cc", [ - "-Wall", - "-Wextra", - "-O2", - path.join(__dirname, "native-fs.c"), - "-o", - helper - ]); - const disk = new RealDisk(root, helper); + const disk = new NodeFileSystemOperations(root); const snapshot = (text: string) => ({ content: new TextEncoder().encode(text) }); @@ -50,6 +39,32 @@ test( ).toString() ) ); + for (const unsafe of [ + "../outside", + "/absolute", + "folder/../outside" + ]) { + await assert.rejects( + disk.write(unsafe, snapshot("blocked")), + /Unsafe path/ + ); + await assert.rejects(disk.readSnapshot(unsafe), /Unsafe path/); + } + // POSIX filenames can contain colons. The engine, not the adapter, + // allocates their portable names for sync. + if (process.platform !== "win32") { + await fs.writeFile( + path.join(root, "local:name.md"), + "local name" + ); + assert.equal( + Buffer.from( + (await disk.readSnapshot("local:name.md"))!.content + ).toString(), + "local name" + ); + await disk.rename("local:name.md", "portable-name.md"); + } await disk.write("a.md", snapshot("A")); await disk.write("b.md", snapshot("B")); // A file in an ancestor position makes descendants absent. diff --git a/frontend/test-support/real-disk.ts b/frontend/test-support/real-disk.ts deleted file mode 100644 index ffe05c40..00000000 --- a/frontend/test-support/real-disk.ts +++ /dev/null @@ -1,171 +0,0 @@ -import assert from "node:assert/strict"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; -import { constants } from "node:fs"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import type { FileSystemOperations, FileSnapshot } from "sync-client"; - -const exec = promisify(execFile); -/** Small integration fixture, NOT a supported application adapter. Safety - * preflights detect static symlinks/hardlinks; they are not an openat sandbox - * against a hostile process replacing ancestors concurrently. */ -export class RealDisk implements FileSystemOperations { - public constructor( - private readonly root: string, - private readonly helper: string - ) {} - private async resolve(relative: string): Promise { - assert( - relative === "" || - relative - .split("/") - .every( - (p) => p && p !== "." && p !== ".." && !p.includes("\\") - ), - "Unsafe path" - ); - let current = this.root; - for (const component of relative.split("/").filter(Boolean)) { - current = path.join(current, component); - const entry = await this.info(current); - if (entry) - assert( - entry.isDirectory() || - (entry.isFile() && entry.nlink === 1), - "Symlink, hardlink or special file" - ); - } - return current; - } - private async info(absolute: string) { - try { - return await fs.lstat(absolute); - } catch (error) { - if ( - ["ENOENT", "ENOTDIR"].includes( - (error as NodeJS.ErrnoException).code ?? "" - ) - ) - return undefined; - throw error; - } - } - public async stat(relative: string) { - const info = await this.info(await this.resolve(relative)); - return info - ? { - kind: info.isFile() - ? ("file" as const) - : ("directory" as const), - size: info.size - } - : undefined; - } - public async exists(relative: string) { - return (await this.stat(relative)) !== undefined; - } - public async readSnapshot( - relative: string - ): Promise { - const absolute = await this.resolve(relative); - let handle; - try { - handle = await fs.open( - absolute, - constants.O_RDONLY | constants.O_NOFOLLOW - ); - } catch (error) { - if ( - ["ENOENT", "ENOTDIR"].includes( - (error as NodeJS.ErrnoException).code ?? "" - ) - ) - return undefined; - throw error; - } - try { - const before = await handle.stat({ bigint: true }); - assert( - before.isFile() && before.nlink === 1n, - "Not an independent regular file" - ); - const content = new Uint8Array(await handle.readFile()); - const after = await handle.stat({ bigint: true }); - const atPath = await fs.lstat(absolute, { bigint: true }); - assert( - before.ino === after.ino && - after.ino === atPath.ino && - before.size === after.size && - before.mtimeNs === after.mtimeNs && - before.ctimeNs === after.ctimeNs, - "File changed during read" - ); - return { content }; - } finally { - await handle.close(); - } - } - public async listFilesRecursively(root = ""): Promise { - const result: string[] = []; - for (const name of await fs.readdir(await this.resolve(root))) { - const relative = root ? `${root}/${name}` : name; - if ((await this.stat(relative))!.kind === "directory") - result.push(...(await this.listFilesRecursively(relative))); - else result.push(relative); - } - return result.sort(); - } - public async createDirectory(relative: string): Promise { - await this.resolve(relative); - let current = this.root; - for (const component of relative.split("/").filter(Boolean)) { - const next = path.join(current, component); - try { - await fs.mkdir(next); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") - throw error; - } - assert((await fs.lstat(next)).isDirectory()); - current = next; - } - } - public async write( - relative: string, - snapshot: FileSnapshot - ): Promise { - assert( - !snapshot.cursors?.length, - "This disk-only fixture has no editor selections" - ); - const destination = await this.resolve(relative); - await fs.writeFile(destination, snapshot.content, { - flag: "wx", - mode: 0o600 - }); - } - public async rename(from: string, to: string): Promise { - const source = await this.resolve(from), - destination = await this.resolve(to); - assert((await this.info(source))?.isFile()); - await exec(this.helper, ["rename", source, destination]); - } - public async delete(relative: string): Promise { - assert(relative, "Cannot prune root"); - const absolute = await this.resolve(relative); - const entry = await this.info(absolute); - if (!entry) return; - assert(entry.isDirectory(), "Cannot delete regular file"); - for (const child of await fs.readdir(absolute)) - await this.delete(`${relative}/${child}`); - await fs.rmdir(absolute); - } - public async deleteFile(relative: string): Promise { - const absolute = await this.resolve(relative); - const entry = await this.info(absolute); - if (!entry) return; - assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); - await fs.unlink(absolute); - } -} diff --git a/frontend/test-support/storage.ts b/frontend/test-support/storage.ts index 09e339a3..a7db333e 100644 --- a/frontend/test-support/storage.ts +++ b/frontend/test-support/storage.ts @@ -2,14 +2,11 @@ import assert from "node:assert/strict"; import type { FileSnapshot, FileSystemOperations, - StoredDatabase, - SyncSettings + StoredDatabase } from "sync-client"; -export type StoredClient = Partial<{ - settings: Partial; - database: Partial; -}>; +import type { StoredClient } from "sync-client"; +export type { StoredClient } from "sync-client"; export type Boundary = (label: string) => void | Promise; type Entry = FileSnapshot | null; // null is a directory; absence is not a directory. export interface DiskImage { diff --git a/scripts/e2e-runner.mjs b/scripts/e2e-runner.mjs index d38fe17b..89b2d342 100644 --- a/scripts/e2e-runner.mjs +++ b/scripts/e2e-runner.mjs @@ -176,7 +176,7 @@ async function main() { }; const frontend = join(root, "frontend"); - // Build only packages in scope and always rebuild the protocol peers. + // Build production adapters as well as the protocol peers and test clients. if ( !(await run( "build-server", @@ -201,6 +201,10 @@ async function main() { "deterministic-tests", "--workspace", "test-client", + "--workspace", + "local-client-cli", + "--workspace", + "vault-link-obsidian-plugin", ], frontend, )) @@ -222,6 +226,7 @@ async function main() { ["run", "test", "--workspace", "sync-client"], frontend, ); + await run("adapter-unit-tests", "npm", ["run", "test", "--workspace", "local-client-cli", "--workspace", "vault-link-obsidian-plugin"], frontend); await run("harness-self-tests", "npm", ["run", "test:harness"], frontend); await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend);