.
This commit is contained in:
parent
25f64ec690
commit
ff1f474324
2 changed files with 3271 additions and 2003 deletions
|
|
@ -3,7 +3,7 @@ import type { FileSystemOperations } from "./filesystem-operations";
|
|||
import type {
|
||||
Database,
|
||||
DocumentId,
|
||||
EngineState,
|
||||
StoredDatabase,
|
||||
RelativePath
|
||||
} from "../persistence/database";
|
||||
import {
|
||||
|
|
@ -21,7 +21,7 @@ import { decodeText } from "../utils/decode-text";
|
|||
import type { ServerConfig } from "../services/server-config";
|
||||
import {
|
||||
applyLocalChange,
|
||||
unappliedChanges,
|
||||
getUnappliedChanges,
|
||||
type LocalChange
|
||||
} from "../sync-operations/local-changes";
|
||||
import { LocalChangesDuringReconciliation } from "../errors/errors";
|
||||
|
|
@ -33,9 +33,18 @@ export interface FileWrite {
|
|||
|
||||
type Mutation = (action: () => Promise<void>) => Promise<void>;
|
||||
|
||||
interface FileApplicationContext {
|
||||
current: StoredDatabase;
|
||||
next: StoredDatabase;
|
||||
writes: Partial<Record<DocumentId, FileWrite>>;
|
||||
paths: Set<RelativePath>;
|
||||
guard: () => void;
|
||||
mutate: Mutation;
|
||||
}
|
||||
|
||||
/** Apply files first, then save metadata. Repeating a content merge is idempotent: reconcile(p, a, a) == a */
|
||||
export class FileOperations {
|
||||
private applying?: EngineState;
|
||||
private applying?: StoredDatabase;
|
||||
private readonly paths: FilePaths;
|
||||
|
||||
public constructor(
|
||||
|
|
@ -60,7 +69,7 @@ export class FileOperations {
|
|||
this.paths = new FilePaths(fs, isProtectedFile);
|
||||
}
|
||||
|
||||
public async snapshot(
|
||||
public async readSnapshot(
|
||||
path: RelativePath
|
||||
): Promise<HashedSnapshot | undefined> {
|
||||
const snapshot = await this.fs.read(path);
|
||||
|
|
@ -79,16 +88,19 @@ export class FileOperations {
|
|||
): void {
|
||||
const state = structuredClone(this.applying ?? this.database.state);
|
||||
|
||||
if (state.lastSeenUpdateId === undefined) return;
|
||||
if (state.lastSeenUpdateId === undefined) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const pending of unappliedChanges(state, previous))
|
||||
for (const pending of getUnappliedChanges(state, previous)) {
|
||||
applyLocalChange(state, pending, () => false);
|
||||
}
|
||||
|
||||
applyLocalChange(state, change, () => false);
|
||||
}
|
||||
|
||||
public async apply(
|
||||
next: EngineState,
|
||||
public async applyChanges(
|
||||
next: StoredDatabase,
|
||||
writes: Partial<Record<DocumentId, FileWrite>> = {},
|
||||
validatePlan: () => void = () => undefined
|
||||
): Promise<void> {
|
||||
|
|
@ -96,16 +108,22 @@ export class FileOperations {
|
|||
const changes = this.localChanges.entries().length;
|
||||
let started = false;
|
||||
const guard = (): void => {
|
||||
if (started) return;
|
||||
if (started) {
|
||||
return;
|
||||
}
|
||||
|
||||
validatePlan();
|
||||
if (this.localChanges.entries().length > changes)
|
||||
if (this.localChanges.entries().length > changes) {
|
||||
throw new LocalChangesDuringReconciliation();
|
||||
}
|
||||
};
|
||||
|
||||
const mutate: Mutation = async (action) => {
|
||||
guard();
|
||||
started = true;
|
||||
await action();
|
||||
};
|
||||
|
||||
guard();
|
||||
const current = structuredClone(this.database.state);
|
||||
current.lastSeenUpdateId = next.lastSeenUpdateId;
|
||||
|
|
@ -114,7 +132,7 @@ export class FileOperations {
|
|||
const observeChanges = async (): Promise<void> => {
|
||||
await this.localChanges.flush();
|
||||
guard();
|
||||
for (const change of unappliedChanges(
|
||||
for (const change of getUnappliedChanges(
|
||||
next,
|
||||
this.localChanges.entries()
|
||||
)) {
|
||||
|
|
@ -130,91 +148,23 @@ export class FileOperations {
|
|||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
for (const id of new Set([
|
||||
...Object.keys(current.local),
|
||||
...Object.keys(next.local)
|
||||
])) {
|
||||
await observeChanges();
|
||||
let from =
|
||||
current.documents[id]?.observedHash !== undefined
|
||||
? current.local[id]
|
||||
: undefined;
|
||||
let to = next.local[id];
|
||||
const write = writes[id];
|
||||
if (!write && (from === undefined || from === to)) continue;
|
||||
if (from !== undefined) {
|
||||
paths.add(from);
|
||||
const info = await this.fs.stat(from);
|
||||
guard();
|
||||
if (info?.kind !== "file") {
|
||||
delete next.local[id];
|
||||
continue;
|
||||
}
|
||||
if (this.isProtectedFile(from, info.size)) {
|
||||
next.local[id] = from;
|
||||
next.documents[id] = structuredClone(
|
||||
current.documents[id]
|
||||
);
|
||||
(next.excluded ??= {})[id] =
|
||||
current.fileManifest.entries[id] ?? null;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (to === undefined) {
|
||||
if (from !== undefined) {
|
||||
await mutate(() => this.fs.deleteFile(from!));
|
||||
delete current.local[id];
|
||||
await this.paths.removeEmptyParents(from);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (from !== to) {
|
||||
to = await this.paths.prepareDestination(
|
||||
next,
|
||||
current,
|
||||
id,
|
||||
to,
|
||||
mutate
|
||||
);
|
||||
next.local[id] = to;
|
||||
from =
|
||||
current.documents[id]?.observedHash !== undefined
|
||||
? current.local[id]
|
||||
: undefined;
|
||||
if (from !== undefined) {
|
||||
await mutate(async () => {
|
||||
current.local[id] = to;
|
||||
await this.fs.rename(from!, to);
|
||||
});
|
||||
await this.paths.removeEmptyParents(from);
|
||||
}
|
||||
}
|
||||
if (write) {
|
||||
const snapshot =
|
||||
from === undefined ? undefined : await this.fs.read(to);
|
||||
if (from !== undefined && !snapshot) {
|
||||
delete next.local[id];
|
||||
continue;
|
||||
}
|
||||
const extensions = (await this.serverConfig.getConfig())
|
||||
.mergeableFileExtensions;
|
||||
await mutate(async () => {
|
||||
current.local[id] = to;
|
||||
current.documents[id] = { observedHash: "" };
|
||||
next.documents[id].observedHash =
|
||||
await this.updateContent(
|
||||
to,
|
||||
snapshot,
|
||||
write,
|
||||
extensions
|
||||
);
|
||||
});
|
||||
}
|
||||
paths.add(to);
|
||||
current.local[id] = to;
|
||||
current.documents[id] = structuredClone(next.documents[id]);
|
||||
await this.applyDocument(id, {
|
||||
current,
|
||||
next,
|
||||
writes,
|
||||
paths,
|
||||
guard,
|
||||
mutate
|
||||
});
|
||||
}
|
||||
|
||||
await observeChanges();
|
||||
await this.database.commit(next);
|
||||
} finally {
|
||||
|
|
@ -223,6 +173,105 @@ export class FileOperations {
|
|||
}
|
||||
}
|
||||
|
||||
private async applyDocument(
|
||||
id: DocumentId,
|
||||
context: FileApplicationContext
|
||||
): Promise<void> {
|
||||
const { current, next, writes, paths, guard, mutate } = context;
|
||||
let from =
|
||||
current.documents[id]?.observedHash !== undefined
|
||||
? current.local[id]
|
||||
: undefined;
|
||||
let to = next.local[id];
|
||||
const write = writes[id];
|
||||
if (!write && (from === undefined || from === to)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (from !== undefined) {
|
||||
paths.add(from);
|
||||
const info = await this.fs.stat(from);
|
||||
guard();
|
||||
if (info?.kind !== "file") {
|
||||
Reflect.deleteProperty(next.local, id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.isProtectedFile(from, info.size)) {
|
||||
next.local[id] = from;
|
||||
next.documents[id] = structuredClone(
|
||||
current.documents[id] ?? {}
|
||||
);
|
||||
(next.excluded ??= {})[id] =
|
||||
current.fileManifest.entries[id] ?? null;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (to === undefined) {
|
||||
if (from !== undefined) {
|
||||
const source = from;
|
||||
await mutate(async () => this.fs.deleteFile(source));
|
||||
Reflect.deleteProperty(current.local, id);
|
||||
await this.paths.removeEmptyParents(from);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (from !== to) {
|
||||
to = await this.paths.prepareDestination(
|
||||
next,
|
||||
current,
|
||||
id,
|
||||
to,
|
||||
mutate
|
||||
);
|
||||
next.local[id] = to;
|
||||
from =
|
||||
current.documents[id]?.observedHash !== undefined
|
||||
? current.local[id]
|
||||
: undefined;
|
||||
if (from !== undefined) {
|
||||
const source = from;
|
||||
const destination = to;
|
||||
await mutate(async () => {
|
||||
current.local[id] = destination;
|
||||
await this.fs.rename(source, destination);
|
||||
});
|
||||
await this.paths.removeEmptyParents(from);
|
||||
}
|
||||
}
|
||||
|
||||
if (write) {
|
||||
const snapshot =
|
||||
from === undefined ? undefined : await this.fs.read(to);
|
||||
if (from !== undefined && !snapshot) {
|
||||
Reflect.deleteProperty(next.local, id);
|
||||
return;
|
||||
}
|
||||
|
||||
const extensions = (await this.serverConfig.getConfig())
|
||||
.mergeableFileExtensions;
|
||||
const destination = to;
|
||||
await mutate(async () => {
|
||||
current.local[id] = destination;
|
||||
current.documents[id] = { observedHash: "" };
|
||||
(next.documents[id] ??= {}).observedHash =
|
||||
await this.updateContent(
|
||||
destination,
|
||||
snapshot,
|
||||
write,
|
||||
extensions
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
paths.add(to);
|
||||
current.local[id] = to;
|
||||
current.documents[id] = structuredClone(next.documents[id] ?? {});
|
||||
}
|
||||
|
||||
private async updateContent(
|
||||
path: RelativePath,
|
||||
current: FileSnapshot | undefined,
|
||||
|
|
@ -245,6 +294,7 @@ export class FileOperations {
|
|||
);
|
||||
return hash(new TextEncoder().encode(text));
|
||||
}
|
||||
|
||||
const result =
|
||||
current && base
|
||||
? await mergeContent(
|
||||
|
|
|
|||
|
|
@ -2,11 +2,22 @@ import { v4 as uuid } from "uuid";
|
|||
import type { FileSystemOperations } from "./filesystem-operations";
|
||||
import type {
|
||||
DocumentId,
|
||||
EngineState,
|
||||
StoredDatabase,
|
||||
RelativePath
|
||||
} from "../persistence/database";
|
||||
import { allocatePortablePath, isInternalPath } from "../utils/portable-path";
|
||||
|
||||
interface PathObstruction {
|
||||
path: RelativePath;
|
||||
kind: "file" | "directory";
|
||||
}
|
||||
|
||||
interface Occupant {
|
||||
id: DocumentId;
|
||||
path: RelativePath;
|
||||
}
|
||||
type Mutation = (action: () => Promise<void>) => Promise<void>;
|
||||
|
||||
/** Resolve occupied rename destinations without discarding another file. */
|
||||
export class FilePaths {
|
||||
public constructor(
|
||||
|
|
@ -18,107 +29,169 @@ export class FilePaths {
|
|||
) {}
|
||||
|
||||
public async prepareDestination(
|
||||
next: EngineState,
|
||||
current: EngineState,
|
||||
next: StoredDatabase,
|
||||
current: StoredDatabase,
|
||||
id: DocumentId,
|
||||
path: RelativePath,
|
||||
mutate: (action: () => Promise<void>) => Promise<void>
|
||||
): Promise<RelativePath> {
|
||||
try {
|
||||
for (;;) {
|
||||
const obstruction = await this.obstruction(path);
|
||||
if (!obstruction) break;
|
||||
const children =
|
||||
obstruction.kind === "file"
|
||||
? [obstruction.path]
|
||||
: await this.fs.listFilesRecursively(obstruction.path);
|
||||
const occupants = [];
|
||||
for (const child of children) {
|
||||
const occupant = Object.keys(current.local).find(
|
||||
(key) =>
|
||||
current.documents[key]?.observedHash !==
|
||||
undefined && current.local[key] === child
|
||||
);
|
||||
const info = await this.fs.stat(child);
|
||||
if (
|
||||
!info ||
|
||||
this.isProtectedFile(child, info.size) ||
|
||||
(obstruction.kind === "directory" &&
|
||||
(occupant === undefined ||
|
||||
next.local[occupant] === child))
|
||||
)
|
||||
break;
|
||||
occupants.push({ id: occupant ?? uuid(), path: child });
|
||||
const obstruction = await this.findObstruction(path);
|
||||
if (!obstruction) {
|
||||
break;
|
||||
}
|
||||
if (occupants.length !== children.length) {
|
||||
path = await this.conflictPath(next, id, path);
|
||||
|
||||
const occupants = await this.findMovableOccupants(
|
||||
obstruction,
|
||||
current,
|
||||
next
|
||||
);
|
||||
if (occupants === undefined) {
|
||||
path = await this.allocateConflictPath(next, id, path);
|
||||
continue;
|
||||
}
|
||||
for (const occupant of occupants) {
|
||||
const displaced = await this.conflictPath(
|
||||
next,
|
||||
occupant.id,
|
||||
occupant.path.split("/").at(-1)!
|
||||
);
|
||||
await mutate(async () => {
|
||||
current.local[occupant.id] = displaced;
|
||||
await this.fs.rename(occupant.path, displaced);
|
||||
});
|
||||
if (!current.documents[occupant.id]) {
|
||||
current.documents[occupant.id] = { observedHash: "" };
|
||||
next.documents[occupant.id] = { observedHash: "" };
|
||||
next.local[occupant.id] = displaced;
|
||||
} else if (next.local[occupant.id] === occupant.path)
|
||||
next.local[occupant.id] = displaced;
|
||||
}
|
||||
if (obstruction.kind === "directory")
|
||||
await mutate(() =>
|
||||
|
||||
await this.relocateOccupants(occupants, current, next, mutate);
|
||||
if (obstruction.kind === "directory") {
|
||||
await mutate(async () =>
|
||||
this.fs.deleteDirectory(obstruction.path)
|
||||
);
|
||||
}
|
||||
}
|
||||
await this.ensureParents(path);
|
||||
|
||||
await this.createParentDirectories(path);
|
||||
} catch (error) {
|
||||
const code =
|
||||
typeof error === "object" && error !== null && "code" in error
|
||||
? error.code
|
||||
: undefined;
|
||||
if (
|
||||
![
|
||||
"ENAMETOOLONG",
|
||||
"EINVAL",
|
||||
"ENOTSUP",
|
||||
"EOPNOTSUPP",
|
||||
"EACCES",
|
||||
"EPERM"
|
||||
].includes(String(code))
|
||||
)
|
||||
throw error;
|
||||
const extension =
|
||||
/\.[^.]*$/u.exec(path.split("/").at(-1) ?? "")?.[0] ?? "";
|
||||
path = await this.conflictPath(
|
||||
next,
|
||||
id,
|
||||
`Recovered ${id}${extension}`
|
||||
);
|
||||
await this.ensureParents(path);
|
||||
return this.recoverDestination(next, id, path, error);
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
private async obstruction(
|
||||
public async removeEmptyParents(path: RelativePath): Promise<void> {
|
||||
let parent = path.split("/").slice(0, -1).join("/");
|
||||
while (parent && !isInternalPath(parent)) {
|
||||
if ((await this.fs.listFilesRecursively(parent)).length) {
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.fs.deleteDirectory(parent);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
|
||||
parent = parent.split("/").slice(0, -1).join("/");
|
||||
}
|
||||
}
|
||||
|
||||
private async findMovableOccupants(
|
||||
obstruction: PathObstruction,
|
||||
current: StoredDatabase,
|
||||
next: StoredDatabase
|
||||
): Promise<Occupant[] | undefined> {
|
||||
const children =
|
||||
obstruction.kind === "file"
|
||||
? [obstruction.path]
|
||||
: await this.fs.listFilesRecursively(obstruction.path);
|
||||
const occupants = [];
|
||||
for (const child of children) {
|
||||
const occupant = Object.keys(current.local).find(
|
||||
(key) =>
|
||||
current.documents[key]?.observedHash !== undefined &&
|
||||
current.local[key] === child
|
||||
);
|
||||
const info = await this.fs.stat(child);
|
||||
if (
|
||||
!info ||
|
||||
this.isProtectedFile(child, info.size) ||
|
||||
(obstruction.kind === "directory" &&
|
||||
(occupant === undefined || next.local[occupant] === child))
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
occupants.push({ id: occupant ?? uuid(), path: child });
|
||||
}
|
||||
|
||||
return occupants;
|
||||
}
|
||||
|
||||
private async relocateOccupants(
|
||||
occupants: Occupant[],
|
||||
current: StoredDatabase,
|
||||
next: StoredDatabase,
|
||||
mutate: Mutation
|
||||
): Promise<void> {
|
||||
for (const occupant of occupants) {
|
||||
const displaced = await this.allocateConflictPath(
|
||||
next,
|
||||
occupant.id,
|
||||
occupant.path.split("/").at(-1) ?? occupant.path
|
||||
);
|
||||
await mutate(async () => {
|
||||
current.local[occupant.id] = displaced;
|
||||
await this.fs.rename(occupant.path, displaced);
|
||||
});
|
||||
if (!current.documents[occupant.id]) {
|
||||
current.documents[occupant.id] = { observedHash: "" };
|
||||
next.documents[occupant.id] = { observedHash: "" };
|
||||
next.local[occupant.id] = displaced;
|
||||
} else if (next.local[occupant.id] === occupant.path) {
|
||||
next.local[occupant.id] = displaced;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async recoverDestination(
|
||||
next: StoredDatabase,
|
||||
id: DocumentId,
|
||||
path: RelativePath,
|
||||
error: unknown
|
||||
): Promise<RelativePath> {
|
||||
const code =
|
||||
typeof error === "object" && error !== null && "code" in error
|
||||
? error.code
|
||||
: undefined;
|
||||
if (
|
||||
![
|
||||
"ENAMETOOLONG",
|
||||
"EINVAL",
|
||||
"ENOTSUP",
|
||||
"EOPNOTSUPP",
|
||||
"EACCES",
|
||||
"EPERM"
|
||||
].includes(String(code))
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const extension =
|
||||
/\.[^.]*$/u.exec(path.split("/").at(-1) ?? "")?.[0] ?? "";
|
||||
path = await this.allocateConflictPath(
|
||||
next,
|
||||
id,
|
||||
`Recovered ${id}${extension}`
|
||||
);
|
||||
await this.createParentDirectories(path);
|
||||
return path;
|
||||
}
|
||||
|
||||
private async findObstruction(
|
||||
path: RelativePath
|
||||
): Promise<{ path: RelativePath; kind: "file" | "directory" } | undefined> {
|
||||
): Promise<PathObstruction | undefined> {
|
||||
const parts = path.split("/");
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const prefix = parts.slice(0, i + 1).join("/");
|
||||
const info = await this.fs.stat(prefix);
|
||||
if (info && (info.kind === "file" || i === parts.length - 1))
|
||||
if (info && (info.kind === "file" || i === parts.length - 1)) {
|
||||
return { path: prefix, kind: info.kind };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async conflictPath(
|
||||
next: EngineState,
|
||||
private async allocateConflictPath(
|
||||
next: StoredDatabase,
|
||||
id: DocumentId,
|
||||
path: RelativePath
|
||||
): Promise<RelativePath> {
|
||||
|
|
@ -131,21 +204,10 @@ export class FilePaths {
|
|||
return allocatePortablePath(path, id, [...occupied, ...files]);
|
||||
}
|
||||
|
||||
private async ensureParents(path: RelativePath): Promise<void> {
|
||||
private async createParentDirectories(path: RelativePath): Promise<void> {
|
||||
const parent = path.split("/").slice(0, -1).join("/");
|
||||
if (parent) await this.fs.createDirectory(parent);
|
||||
}
|
||||
|
||||
public async removeEmptyParents(path: RelativePath): Promise<void> {
|
||||
let parent = path.split("/").slice(0, -1).join("/");
|
||||
while (parent && !isInternalPath(parent)) {
|
||||
if ((await this.fs.listFilesRecursively(parent)).length) break;
|
||||
try {
|
||||
await this.fs.deleteDirectory(parent);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
parent = parent.split("/").slice(0, -1).join("/");
|
||||
if (parent) {
|
||||
await this.fs.createDirectory(parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue