diff --git a/frontend/sync-client/src/persistence/database.test.ts b/frontend/sync-client/src/persistence/database.test.ts index e193801d..ea871f55 100644 --- a/frontend/sync-client/src/persistence/database.test.ts +++ b/frontend/sync-client/src/persistence/database.test.ts @@ -4,7 +4,7 @@ import { Database, createEmptyDatabase } from "./database"; test("current state keeps absent optional fields absent across save and reload", async () => { const initial = createEmptyDatabase("vault"); - initial.local = { local: "local.md", remote: "remote.md" }; + initial.actualFileManifest = { local: "local.md", remote: "remote.md" }; initial.documents = { local: { observedHash: "local" }, remote: { remote: { vaultUpdateId: 1, contentSize: 3 } }, @@ -29,7 +29,7 @@ test("current state keeps absent optional fields absent across save and reload", test("current reset state is restored after an uncertain save", async () => { const saved = createEmptyDatabase("vault"); - saved.local = { note: "note.md" }; + saved.actualFileManifest = { note: "note.md" }; saved.documents.note = { base: { vaultUpdateId: null, hash: "clean" }, observedHash: "clean" @@ -43,6 +43,6 @@ test("current reset state is restored after an uncertain save", async () => { async () => structuredClone(saved) ); await assert.rejects(db.save(), /interrupted/); - await db.recoverPersistence(); + await db.reloadFromSave(); assert.deepEqual(db.state, saved); }); diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 86b2845c..fa543409 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,374 +1,156 @@ -import type { Logger } from "../tracing/logger"; -import { EMPTY_HASH } from "../utils/hash"; -import { CoveredValues } from "../utils/data-structures/min-covered"; -import { awaitAll } from "../utils/await-all"; -import { removeFromArray } from "../utils/remove-from-array"; +import type { PendingRequest } from "./requests"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { FileManifestEntries } from "../types/file-manifest-entries"; export type VaultUpdateId = number; export type DocumentId = string; export type RelativePath = string; -export interface DocumentMetadata { - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath?: RelativePath; +export interface ContentHead { + vaultUpdateId: VaultUpdateId; + contentSize: number; } -export interface StoredDocumentMetadata { - relativePath: RelativePath; - documentId: DocumentId; - parentVersionId: VaultUpdateId; - remoteRelativePath?: RelativePath; - hash: string; +export interface DocumentState { + // A null version retains only the clean hash after a server history reset. + base?: { vaultUpdateId: VaultUpdateId | null; hash: string }; + + // Last observed disk hash; an empty string means a notification awaits scanning. + observedHash?: string; + + // A remote version still to be incorporated, including excluded downloads. + remote?: ContentHead; } +export type { PendingRequest } from "./requests"; + export interface StoredDatabase { - documents: StoredDocumentMetadata[]; - lastSeenUpdateId: VaultUpdateId | undefined; - hasInitialSyncCompleted: boolean; + readonly vaultKey: string; + + // Last incorporated server manifest; the base for the next three-way path merge. + parentFileManifest: FileManifest; + + // Current local document IDs and paths, including files awaiting upload or download. + actualFileManifest: FileManifestEntries; + + // Per-document merge bases and observations; content bytes are not stored here. + documents: Record; + + // Absent until a vault snapshot is applied; also marks a durable history reset. + lastSeenUpdateId?: VaultUpdateId; + + // Exact request to resolve before observing a newer remote base. + pending?: PendingRequest; + + // Commits consumption of the saved notification queue with the identity map. + lastAppliedLocalChangeId?: string; + + // Server path at the start of exclusion, retained as the path-merge base. + // null means the document was absent from the server manifest at that time. + excluded?: Record; } -/** - * Represents a document in the database. - * - * It is mutable and its content should always represent the latest - * state of the document on disk based on the update events we have seen. - */ -export interface DocumentRecord { - relativePath: RelativePath; - documentId: DocumentId; - metadata: DocumentMetadata | undefined; - isDeleted: boolean; - updates: Promise[]; - parallelVersion: number; +export function createEmptyDatabase(vaultKey: string): StoredDatabase { + return { + vaultKey, + parentFileManifest: { fileManifestId: 0, entries: {} }, + actualFileManifest: {}, + documents: {} + }; +} + +function canBindToAnotherVault(state: StoredDatabase): boolean { + return ( + state.lastSeenUpdateId === undefined && + !state.pending && + Object.keys(state.actualFileManifest).length === 0 + ); } export class Database { - private documents: DocumentRecord[]; - private lastSeenUpdateIds: CoveredValues; - private hasInitialSyncCompleted: boolean; + public state: StoredDatabase; + private needsReload = false; public constructor( - private readonly logger: Logger, - initialState: Partial | undefined, - private readonly saveData: (data: StoredDatabase) => Promise + initial: StoredDatabase | undefined, + private readonly saveData: (data: StoredDatabase) => Promise, + vaultKey: string, + private readonly loadData: () => Promise ) { - initialState ??= {}; + const state = + initial === undefined + ? createEmptyDatabase(vaultKey) + : structuredClone(initial); - this.documents = - initialState.documents?.map( - ({ relativePath, documentId, ...metadata }) => ({ - relativePath, - documentId, - metadata, - isDeleted: false, - updates: [], - parallelVersion: 0 - }) - ) ?? []; + if (state.vaultKey !== vaultKey && !canBindToAnotherVault(state)) { + throw new Error( + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." + ); + } - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.length} documents`); - - const { lastSeenUpdateId } = initialState; - this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`); - this.lastSeenUpdateIds = new CoveredValues( - Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1 - ); - - this.documents.forEach((doc) => { - this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId); - }); - - this.hasInitialSyncCompleted = - initialState.hasInitialSyncCompleted ?? false; - this.logger.debug( - `Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}` - ); + this.state = { ...state, vaultKey }; } public get length(): number { - return this.documents.length; + return Object.keys(this.state.actualFileManifest).length; } - public get resolvedDocuments(): DocumentRecord[] { - const paths = new Map(); - this.documents - // eslint-disable-next-line no-restricted-syntax -- Type narrowing, not removing a specific item - .filter(({ metadata }) => metadata !== undefined) - .forEach((record) => - paths.set(record.relativePath, [ - record, - ...(paths.get(record.relativePath) ?? []) - ]) - ); - - return Array.from(paths.values()).map((records) => { - records.sort( - (a, b) => b.parallelVersion - a.parallelVersion // descending - ); - - if ( - records.length > 1 && - records.some((current, i) => - i === 0 - ? false - : records[i - 1].parallelVersion === - current.parallelVersion - ) - ) { - throw new Error( - `Multiple documents with the same parallel version and path at ${records[0].relativePath}` - ); - } - return records[0]; - }); - } - - public updateDocumentMetadata( - metadata: { - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath: RelativePath; - }, - toUpdate: DocumentRecord - ): void { - if (!this.documents.includes(toUpdate)) { - throw new Error("Document not found in database"); - } - - toUpdate.metadata = metadata; - - this.saveInTheBackground(); - } - - public removeDocumentPromise(promise: Promise): void { - const entry = this.documents.find(({ updates }) => - updates.includes(promise) - ); - - if (entry === undefined) { - // This method should be idempotent and tolerant of - // stragglers calling it after the databse has been reset. + // Bind an empty database to the configured vault. + public async bindVault(vaultKey: string): Promise { + if (this.state.vaultKey === vaultKey) { return; } - removeFromArray(entry.updates, promise); - // No need to save as Promises don't get serialized - } - - public removeDocument(find: DocumentRecord): void { - removeFromArray(this.documents, find); - this.saveInTheBackground(); - } - - public getLatestDocumentByRelativePath( - find: RelativePath - ): DocumentRecord | undefined { - const candidates = this.documents.filter( - ({ relativePath }) => relativePath === find - ); - candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending - return candidates[0]; - } - - public async getResolvedDocumentByRelativePath( - relativePath: RelativePath, - promise: Promise - ): Promise { - const entry = this.getLatestDocumentByRelativePath(relativePath); - - if (entry === undefined) { + if (!canBindToAnotherVault(this.state)) { throw new Error( - `Document not found by relative path: ${relativePath}, ${JSON.stringify( - this.documents, - null, - 2 - )}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); } - const currentPromises = entry.updates; - entry.updates = [...currentPromises, promise]; - await awaitAll(currentPromises); - - return entry; + await this.commit({ ...this.state, vaultKey }); } - public createNewPendingDocument( - documentId: DocumentId, - relativePath: RelativePath, - promise: Promise - ): DocumentRecord { - this.logger.debug( - `Creating new pending document: ${relativePath} (${documentId})` + public async commit(next: StoredDatabase): Promise { + const snapshot = structuredClone(next); + + if (this.needsReload) { + throw new Error( + "Reload saved state before retrying an uncertain save" + ); + } + + this.needsReload = true; + await this.saveData(snapshot); + this.state = snapshot; + this.needsReload = false; + } + + public findDocumentId(path: RelativePath): DocumentId | undefined { + return Object.keys(this.state.actualFileManifest).find( + (id) => this.state.actualFileManifest[id] === path ); - const previousEntry = - this.getLatestDocumentByRelativePath(relativePath); - - const entry = { - relativePath, - documentId, - metadata: undefined, - isDeleted: false, - updates: [promise], - parallelVersion: - previousEntry?.parallelVersion === undefined - ? 0 - : previousEntry.parallelVersion + 1 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; } - public createNewEmptyDocument( - documentId: DocumentId, - parentVersionId: VaultUpdateId, - relativePath: RelativePath - ): DocumentRecord { - const entry = { - relativePath, - documentId, - metadata: { - parentVersionId, - hash: EMPTY_HASH, - remoteRelativePath: relativePath - }, - isDeleted: false, - updates: [], - parallelVersion: 0 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public getDocumentByDocumentId( - find: DocumentId - ): DocumentRecord | undefined { - return this.documents.find(({ documentId }) => documentId === find); - } - - public move( - oldRelativePath: RelativePath, - newRelativePath: RelativePath - ): void { - const oldDocument = - this.getLatestDocumentByRelativePath(oldRelativePath); - - if (oldDocument === undefined) { + public async reloadFromSave(): Promise { + if (!this.needsReload) { return; } - const newDocument = - this.getLatestDocumentByRelativePath(newRelativePath); - if (newDocument?.isDeleted === false) { - throw new Error( - `Document already exists at new location: ${newRelativePath}` - ); + const saved = await this.loadData(); + if (saved) { + this.state = structuredClone(saved); } - oldDocument.relativePath = newRelativePath; - // We're in a strange state where the target of the move has just got deleted, - // however, its metadata might already have a bunch of updates queued up for - // the document at the new location. We need to keep these updates. - oldDocument.parallelVersion = - newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - - this.saveInTheBackground(); + this.needsReload = false; } - public delete(relativePath: RelativePath): void { - const candidate = this.getLatestDocumentByRelativePath(relativePath); - if (candidate === undefined) { - throw new Error( - `Document not found by relative path: ${relativePath}` - ); - } - candidate.isDeleted = true; - } - - public getHasInitialSyncCompleted(): boolean { - return this.hasInitialSyncCompleted; - } - - public setHasInitialSyncCompleted(value: boolean): void { - this.hasInitialSyncCompleted = value; - this.saveInTheBackground(); - } - - public getLastSeenUpdateId(): VaultUpdateId { - return this.lastSeenUpdateIds.min; - } - - public addSeenUpdateId(value: number): void { - const previousMin = this.lastSeenUpdateIds.min; - this.lastSeenUpdateIds.add(value); - if (previousMin !== this.lastSeenUpdateIds.min) { - this.saveInTheBackground(); - } - } - - public setLastSeenUpdateId(value: number): void { - this.lastSeenUpdateIds.min = value; - this.saveInTheBackground(); - } - - public reset(): void { - this.documents = []; - this.lastSeenUpdateIds = new CoveredValues( - 0 // the first updateId will be 1 which is the first integer after -1 - ); - this.hasInitialSyncCompleted = false; - this.saveInTheBackground(); + public cloneState(): StoredDatabase { + return structuredClone(this.state); } public async save(): Promise { - return this.saveData({ - documents: this.resolvedDocuments.map( - ({ relativePath, documentId, metadata }) => ({ - documentId, - relativePath, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ...metadata! // `resolvedDocuments` only returns docs with metadata set - }) - ), - lastSeenUpdateId: this.lastSeenUpdateIds.min, - hasInitialSyncCompleted: this.hasInitialSyncCompleted - }); + await this.commit(this.state); } - private ensureConsistency(): void { - const idToPath = new Map(); - this.resolvedDocuments.forEach(({ relativePath, documentId }) => { - idToPath.set(documentId, [ - ...(idToPath.get(documentId) ?? []), - relativePath - ]); - }); - - const duplicates = Array.from(idToPath.entries()) - .filter(([_, paths]) => paths.length > 1) - .map(([id, paths]) => `${id} (${paths.join(", ")})`); - - if (duplicates.length > 0) { - throw new Error( - "Document IDs are not unique, found duplicates: " + - duplicates.join("; ") - ); - } - } - - private saveInTheBackground(): void { - this.ensureConsistency(); - void this.save().catch((error: unknown) => { - this.logger.error(`Error saving data: ${error}`); - }); - } }