Use new reconcile api
This commit is contained in:
parent
3c7f908dcc
commit
49065f5b6d
6 changed files with 127 additions and 86 deletions
41
sync-server/Cargo.lock
generated
41
sync-server/Cargo.lock
generated
|
|
@ -1582,12 +1582,12 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "reconcile-text"
|
||||
version = "0.8.0"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5"
|
||||
checksum = "8eb67d18cd23948a32b940e137128552105970913d5747daa5860a58819a7453"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1907,7 +1907,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"sha2",
|
||||
"smallvec",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
|
|
@ -1991,7 +1991,7 @@ dependencies = [
|
|||
"smallvec",
|
||||
"sqlx-core",
|
||||
"stringprep",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"whoami",
|
||||
|
|
@ -2030,7 +2030,7 @@ dependencies = [
|
|||
"smallvec",
|
||||
"sqlx-core",
|
||||
"stringprep",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
"tracing",
|
||||
"uuid",
|
||||
"whoami",
|
||||
|
|
@ -2056,7 +2056,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_urlencoded",
|
||||
"sqlx-core",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
|
|
@ -2112,6 +2112,17 @@ dependencies = [
|
|||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sync_server"
|
||||
version = "0.14.0"
|
||||
|
|
@ -2136,7 +2147,7 @@ dependencies = [
|
|||
"sha2",
|
||||
"sqlx",
|
||||
"tempfile",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-http",
|
||||
|
|
@ -2197,11 +2208,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.17"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
|
||||
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.17",
|
||||
"thiserror-impl 2.0.20",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2217,13 +2228,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.17"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
|
||||
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.90",
|
||||
"syn 3.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2428,7 +2439,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
|
|||
dependencies = [
|
||||
"chrono",
|
||||
"lazy_static",
|
||||
"thiserror 2.0.17",
|
||||
"thiserror 2.0.20",
|
||||
"ts-rs-macros",
|
||||
"uuid",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ sha2 = "0.10"
|
|||
unicode-normalization = "0.1"
|
||||
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl", "no-serde-warnings"] }
|
||||
base64 = "0.22.1"
|
||||
reconcile-text = { version = "0.8.0", features = ["serde"] }
|
||||
reconcile-text = { version = "0.12.1", features = ["serde"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
# Sync server
|
||||
|
||||
## API v4
|
||||
|
||||
- ban using unwraps
|
||||
|
||||
## API v5
|
||||
|
||||
The server is a compare-and-swap store. Content versions do not contain paths or
|
||||
deletions. A separately versioned file manifest owns the namespace and membership.
|
||||
|
|
@ -8,14 +11,15 @@ All routes below are under `/vaults/:vault_id` and require bearer authentication
|
|||
(except the WebSocket, which authenticates its first message).
|
||||
|
||||
| Endpoint | Purpose |
|
||||
| --- | --- |
|
||||
| ---------------------------------------------------------- | --------------------------------------------------------------------------------------- |
|
||||
| `PUT /documents/:document_id` | CAS an immutable content version. |
|
||||
| `GET /documents/:document_id/metadata` | Current version metadata and size without content bytes. |
|
||||
| `GET /documents/:document_id` | Current content version, including base64 bytes. |
|
||||
| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. |
|
||||
| `GET /file_manifest` | Current file manifest. |
|
||||
| `PUT /file_manifest` | CAS the complete file manifest. |
|
||||
| `GET /vault_snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. |
|
||||
| `GET /events_since?after=N` | Every event after N, in order, with the current watermark. |
|
||||
| `GET /file-manifest` | Current file manifest. |
|
||||
| `PUT /file-manifest` | CAS the complete file manifest. |
|
||||
| `GET /vault-snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. |
|
||||
| `GET /events-since?after=N` | Every event after N, in order, with the current watermark. |
|
||||
| `GET /ws` | Ordered event delivery and ephemeral cursor positions. |
|
||||
|
||||
A content push includes a `Device-Id` header and this JSON body:
|
||||
|
|
@ -54,13 +58,14 @@ UUID-to-path mapping in `file_manifest_entries`. Each version has a header row
|
|||
even when its entries are empty. Entries are unique by document ID and exact path
|
||||
within a version; portable path validation also rejects aliases and directory
|
||||
conflicts. Reads reconstruct the complete map, and vault snapshots join these
|
||||
rows to the latest content versions. The version, entries, and complete file
|
||||
manifest event in `events.event_json` commit atomically. Event replay and retry
|
||||
acknowledgements continue to use `event_json`.
|
||||
rows to the latest content versions. Event replay and retry acknowledgements are
|
||||
also reconstructed from normalized content/manifest rows, without storing a
|
||||
second complete map in the event table.
|
||||
|
||||
Both pushes return a tagged `Accepted` or `StaleBase` response. An accepted file
|
||||
manifest response returns its assigned file manifest ID. A stale content response
|
||||
includes bytes; a stale file manifest response includes the complete map.
|
||||
includes metadata and byte length only; clients fetch immutable bytes after applying
|
||||
their size and exclusion policy. A stale file manifest response includes the complete map.
|
||||
Accepted requests store their unique request ID and fingerprint in the event row,
|
||||
atomically with the version. Retrying the exact request reconstructs the original
|
||||
`Accepted` response from that event before checking the current head, even after
|
||||
|
|
@ -70,8 +75,8 @@ identity is not part of the fingerprint.
|
|||
|
||||
File manifest validation rejects duplicate UUID keys (including aliases), missing
|
||||
content, path aliases, inconsistent directory spelling, and file/ancestor
|
||||
conflicts. Paths use `/`, NFC normalization, at most 240 UTF-8 bytes, and portable
|
||||
Windows filename restrictions. Alias comparison is NFC → Unicode uppercase →
|
||||
conflicts. Paths use `/`, NFC normalization, and portable Windows filename
|
||||
restrictions without imposing a universal total-path byte limit. Alias comparison is NFC → Unicode uppercase →
|
||||
NFC. `.vault-link-sync` and its aliases are reserved. The server never sanitizes
|
||||
or allocates a replacement name.
|
||||
|
||||
|
|
@ -84,14 +89,20 @@ content/file-manifest version IDs are the event IDs that created them. Rejected
|
|||
requests and retries of accepted requests append no events. Timestamps never
|
||||
determine event ordering.
|
||||
|
||||
The WebSocket handshake retains `lastSeenVaultUpdateId`. The server sends
|
||||
`{type: "vaultEvents", headEventId, events}` in increasing order, including events
|
||||
originated by that client. Each connection drains the database through one sender.
|
||||
In-memory notifications only wake it; polling covers missed wakes and cancellation
|
||||
after commit. Reconnect with the last durably processed event ID. The HTTP replay
|
||||
endpoint uses the same history, so losing the final notification is recoverable.
|
||||
A fresh client reads `/vault_snapshot`, applies it durably, then replays after its
|
||||
watermark.
|
||||
WebSocket messages are `{type: "vaultChanged"}` hints. Clients fetch ordered
|
||||
history through HTTP from their last incorporated event ID. The server compares
|
||||
its current history checkpoint after broadcasts and periodically, so missed wakes
|
||||
and requests interrupted after commit still trigger catchup. Reconnecting sends a
|
||||
fresh hint without replaying events over the socket. Cursor messages are unchanged.
|
||||
A fresh client reads `/vault-snapshot`, applies it durably, then replays after its
|
||||
watermark. Successful HTTP responses include `X-Vault-Link-History`, containing
|
||||
an event ID and a random event-incarnation token. Clients persist and echo this
|
||||
checkpoint on later HTTP requests. The server returns HTTP 409 with
|
||||
`X-Vault-Link-History-Mismatch: 1` before executing a request whose checkpoint is
|
||||
absent from the current database. Restored backups retain their original prefix;
|
||||
new events get new tokens even when numeric IDs or request UUIDs are reused.
|
||||
Restore databases while the server is stopped. Live replacement of open SQLite
|
||||
files is not supported.
|
||||
Cursor notifications are ephemeral and do not consume event IDs.
|
||||
Cursor expiry and disconnect broadcast the remaining clients, including an empty
|
||||
list when the last client leaves.
|
||||
|
|
@ -116,8 +127,20 @@ flushes database directory entries; the filesystem must support those durability
|
|||
operations. Content, file manifests, and events (including request fingerprints)
|
||||
are retained indefinitely.
|
||||
|
||||
API v4 requires fresh vault databases and upgraded clients. Existing database
|
||||
migration checksums intentionally reject earlier schemas; there is no automatic
|
||||
erasure or migration. Use a new database directory and preserve old data.
|
||||
Deploy API v5 clients and server together. The v4 database schema migrates by
|
||||
adding event-incarnation tokens without deleting content or receipts. Schemas
|
||||
from before v4 still require a separate database directory and explicit recovery.
|
||||
Vault databases open and migrate independently on first access; one corrupt or
|
||||
incompatible vault does not prevent healthy vaults from serving requests.
|
||||
|
||||
Diffs are limited to 10,000 items and reconstructed in one pass over Unicode
|
||||
scalars on a blocking worker, before taking the SQLite writer transaction. The
|
||||
transaction rechecks request identity and the parent version before committing.
|
||||
Clients send a snapshot when an edit exceeds the diff-operation budget. Decoded
|
||||
content is bounded by `max_body_size_mb` as well as the encoded request limit.
|
||||
|
||||
A missing configuration file is initialized with durably stored credentials.
|
||||
Existing configurations must explicitly supply `users.user_configs`; omitted
|
||||
credentials are rejected instead of generating an unpersisted token at startup.
|
||||
Base64 increases snapshot body size by about one third; request limits apply to
|
||||
the encoded JSON body.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
[toolchain]
|
||||
channel = "1.89.0"
|
||||
channel = "1.94.0"
|
||||
targets = [
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"x86_64-unknown-linux-musl",
|
||||
|
|
|
|||
|
|
@ -242,18 +242,18 @@ async fn manifest_existence_checks_are_independent_of_payload_decoding() {
|
|||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
Database::missing_document(&mut tx, &[id]).await.unwrap(),
|
||||
Database::get_missing_document(&mut tx, &[id]).await.unwrap(),
|
||||
None
|
||||
);
|
||||
let missing = uuid::Uuid::new_v4();
|
||||
assert_eq!(
|
||||
Database::missing_document(&mut tx, &[id, missing])
|
||||
Database::get_missing_document(&mut tx, &[id, missing])
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(missing.to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
Database::missing_document(&mut tx, &[]).await.unwrap(),
|
||||
Database::get_missing_document(&mut tx, &[]).await.unwrap(),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,33 +78,15 @@ pub async fn put_file_content(
|
|||
)));
|
||||
}
|
||||
|
||||
let content = match push.content {
|
||||
PushContent::Snapshot(bytes) => STANDARD
|
||||
.decode(bytes)
|
||||
.map_err(|error| client_error(error.into()))?,
|
||||
let (tx, content) = tokio::task::spawn_blocking(move || {
|
||||
let content = decode_content(parent, push.content);
|
||||
(tx, content)
|
||||
})
|
||||
.await
|
||||
.map_err(|error| server_error(error.into()))?;
|
||||
|
||||
PushContent::Diff(diff) => {
|
||||
if diff
|
||||
.iter()
|
||||
.any(|n| matches!(n, NumberOrText::Number(i64::MIN)))
|
||||
{
|
||||
return Err(client_error(anyhow!("Invalid diff length")));
|
||||
}
|
||||
|
||||
let parent = parent
|
||||
.as_ref()
|
||||
.ok_or_else(|| client_error(anyhow!("Diff requires a parent")))?;
|
||||
|
||||
let text =
|
||||
str::from_utf8(&parent.content).map_err(|error| client_error(error.into()))?;
|
||||
|
||||
EditedText::from_diff(text, diff, &*BuiltinTokenizer::Word)
|
||||
.map_err(|error| client_error(error.into()))?
|
||||
.apply()
|
||||
.text()
|
||||
.into_bytes()
|
||||
}
|
||||
};
|
||||
let mut tx = tx;
|
||||
let content = content.map_err(client_error)?;
|
||||
|
||||
let version = StoredDocumentVersion {
|
||||
vault_update_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint)
|
||||
|
|
@ -132,8 +114,32 @@ pub async fn put_file_content(
|
|||
Ok(Json(response))
|
||||
}
|
||||
|
||||
fn decode_content(
|
||||
parent: Option<StoredDocumentVersion>,
|
||||
content: PushContent,
|
||||
) -> anyhow::Result<Vec<u8>> {
|
||||
let bytes = match content {
|
||||
PushContent::Snapshot(encoded) => STANDARD.decode(encoded)?,
|
||||
PushContent::Diff(diff) => {
|
||||
let parent = parent.ok_or_else(|| anyhow!("Diff requires a parent"))?;
|
||||
let text = str::from_utf8(&parent.content)?;
|
||||
anyhow::ensure!(
|
||||
!diff
|
||||
.iter()
|
||||
.any(|item| matches!(item, NumberOrText::Number(i64::MIN))),
|
||||
"Diff contains an out-of-range length"
|
||||
);
|
||||
let edited = EditedText::from_diff(text, diff, &*BuiltinTokenizer::Character)?;
|
||||
edited.apply().text().into_bytes()
|
||||
}
|
||||
};
|
||||
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use reconcile_text::EditedText;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::*;
|
||||
|
|
@ -148,6 +154,7 @@ mod tests {
|
|||
Config, database_config::DatabaseConfig, server_config::ServerConfig,
|
||||
user_config::VaultAccess,
|
||||
},
|
||||
consts::DEFAULT_EVENTS_PAGE_SIZE,
|
||||
};
|
||||
|
||||
async fn send_document(
|
||||
|
|
@ -363,11 +370,8 @@ mod tests {
|
|||
outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"),
|
||||
};
|
||||
let latest = vault.latest(document).await;
|
||||
assert_eq!(winner.vault_update_id, rejected.metadata.vault_update_id);
|
||||
assert_eq!(
|
||||
STANDARD.decode(rejected.content_base64).unwrap(),
|
||||
latest.content
|
||||
);
|
||||
assert_eq!(winner.vault_update_id, rejected.vault_update_id);
|
||||
assert_eq!(rejected.content_size, latest.content.len());
|
||||
assert!(latest.content == b"left" || latest.content == b"right");
|
||||
let original = vault
|
||||
.state
|
||||
|
|
@ -446,7 +450,7 @@ mod tests {
|
|||
}
|
||||
let events = reopened
|
||||
.database
|
||||
.events_after(&vault.vault, 0)
|
||||
.events_after(&vault.vault, 0, DEFAULT_EVENTS_PAGE_SIZE)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(events.head_event_id, latest.vault_update_id);
|
||||
|
|
@ -473,7 +477,7 @@ mod tests {
|
|||
let events = vault
|
||||
.state
|
||||
.database
|
||||
.events_after(&vault.vault, 0)
|
||||
.events_after(&vault.vault, 0, DEFAULT_EVENTS_PAGE_SIZE)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(events.head_event_id, 1);
|
||||
|
|
@ -510,7 +514,7 @@ mod tests {
|
|||
let events = vault
|
||||
.state
|
||||
.database
|
||||
.events_after(&vault.vault, 0)
|
||||
.events_after(&vault.vault, 0, DEFAULT_EVENTS_PAGE_SIZE)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(events.head_event_id, 0);
|
||||
|
|
@ -534,7 +538,7 @@ mod tests {
|
|||
let events = vault
|
||||
.state
|
||||
.database
|
||||
.events_after(&vault.vault, 0)
|
||||
.events_after(&vault.vault, 0, DEFAULT_EVENTS_PAGE_SIZE)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(events.events.len(), 1);
|
||||
|
|
@ -549,8 +553,11 @@ mod tests {
|
|||
let desired = "Hi beautiful world 🌍";
|
||||
let base = accepted(vault.send(snapshot(document, None, original)).await);
|
||||
let mut update = snapshot(document, Some(base.vault_update_id), "");
|
||||
update.content =
|
||||
PushContent::Diff(EditedText::from_strings(original, &desired.into()).to_diff());
|
||||
update.content = PushContent::Diff(
|
||||
EditedText::from_strings(original, &desired.into())
|
||||
.to_diff()
|
||||
.unwrap(),
|
||||
);
|
||||
let updated = accepted(vault.send(update).await);
|
||||
assert_eq!(vault.latest(document).await.content, desired.as_bytes());
|
||||
|
||||
|
|
@ -577,7 +584,7 @@ mod tests {
|
|||
assert!(matches!(
|
||||
vault.send(foreign_base).await,
|
||||
DocumentUpdateResponse::StaleBase(version)
|
||||
if version.metadata.vault_update_id == updated.vault_update_id
|
||||
if version.vault_update_id == updated.vault_update_id
|
||||
));
|
||||
|
||||
let binary_document = uuid::Uuid::new_v4();
|
||||
|
|
|
|||
Loading…
Reference in a new issue