From 1619c51ad9f4c423d03304ab9cca534617e8fd3e Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Wed, 16 Sep 2026 10:36:20 +0100 Subject: [PATCH 01/48] Migrate server to CAS only --- sync-server/README.md | 33 ++ sync-server/build.rs | 2 +- sync-server/src/app_state/database.rs | 324 +------------- .../database/get_document_version.rs | 41 ++ .../app_state/database/get_latest_document.rs | 43 ++ .../database/get_latest_document_by_path.rs | 46 ++ .../database/get_latest_documents.rs | 56 +++ .../database/get_latest_documents_since.rs | 62 +++ .../database/get_max_update_id_in_vault.rs | 29 ++ .../database/get_push_acknowledgement.rs | 38 ++ .../database/insert_document_version.rs | 79 ++++ .../migrations/20241207143519_bootstrap.sql | 11 +- .../20250522192949_add_provenance_columns.sql | 2 - .../20251019103154_add_has_been_merged.sql | 13 - sync-server/src/app_state/database/models.rs | 2 - sync-server/src/consts.rs | 2 +- sync-server/src/server.rs | 51 +-- sync-server/src/server/create_document.rs | 109 ----- sync-server/src/server/delete_document.rs | 96 ---- sync-server/src/server/endpoints.rs | 7 + .../fetch_document_version_content.rs | 0 .../fetch_latest_document_version.rs | 0 .../endpoints/fetch_latest_documents.rs | 36 ++ .../src/server/{ => endpoints}/index.rs | 2 +- .../src/server/{ => endpoints}/ping.rs | 2 +- .../src/server/endpoints/push_document.rs | 210 +++++++++ .../server/endpoints/push_document/tests.rs | 423 ++++++++++++++++++ .../src/server/{ => endpoints}/websocket.rs | 0 .../src/server/fetch_document_version.rs | 62 --- .../src/server/fetch_latest_documents.rs | 59 --- sync-server/src/server/requests.rs | 56 +-- sync-server/src/server/responses.rs | 26 +- sync-server/src/server/update_document.rs | 271 ----------- sync-server/src/utils.rs | 2 - sync-server/src/utils/is_binary.rs | 26 -- .../src/utils/is_file_type_mergable.rs | 34 -- 36 files changed, 1163 insertions(+), 1092 deletions(-) create mode 100644 sync-server/src/app_state/database/get_document_version.rs create mode 100644 sync-server/src/app_state/database/get_latest_document.rs create mode 100644 sync-server/src/app_state/database/get_latest_document_by_path.rs create mode 100644 sync-server/src/app_state/database/get_latest_documents.rs create mode 100644 sync-server/src/app_state/database/get_latest_documents_since.rs create mode 100644 sync-server/src/app_state/database/get_max_update_id_in_vault.rs create mode 100644 sync-server/src/app_state/database/get_push_acknowledgement.rs create mode 100644 sync-server/src/app_state/database/insert_document_version.rs delete mode 100644 sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql delete mode 100644 sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql delete mode 100644 sync-server/src/server/create_document.rs delete mode 100644 sync-server/src/server/delete_document.rs create mode 100644 sync-server/src/server/endpoints.rs rename sync-server/src/server/{ => endpoints}/fetch_document_version_content.rs (100%) rename sync-server/src/server/{ => endpoints}/fetch_latest_document_version.rs (100%) create mode 100644 sync-server/src/server/endpoints/fetch_latest_documents.rs rename sync-server/src/server/{ => endpoints}/index.rs (68%) rename sync-server/src/server/{ => endpoints}/ping.rs (95%) create mode 100644 sync-server/src/server/endpoints/push_document.rs create mode 100644 sync-server/src/server/endpoints/push_document/tests.rs rename sync-server/src/server/{ => endpoints}/websocket.rs (100%) delete mode 100644 sync-server/src/server/fetch_document_version.rs delete mode 100644 sync-server/src/server/fetch_latest_documents.rs delete mode 100644 sync-server/src/server/update_document.rs delete mode 100644 sync-server/src/utils/is_binary.rs delete mode 100644 sync-server/src/utils/is_file_type_mergable.rs diff --git a/sync-server/README.md b/sync-server/README.md index 16509fb7..9b11a061 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -1,5 +1,38 @@ # Sync server +## Document pushes (API v3) + +The server stores immutable document versions. One compare-and-swap endpoint, +`PUT /vaults/:vault_id/documents/:document_id`, handles every mutation. The client +generates the document UUID; the JSON body contains: + +- `requestId`: a UUID identifying this exact push. +- `parentVersionId`: `null` to create an absent document, otherwise its base version. +- `relativePath`: the desired filename. +- `content`: `{ "type": "Snapshot", "value": "" }`, + `{ "type": "Diff", "value": [3, "inserted text", -2] }`, or `{ "type": "Delete" }`. + +All pushes return one of: + +- `Accepted`: version metadata for the exact snapshot committed. A retry with the + same request ID returns the original acknowledgement, even after a restart or + subsequent updates. Clients must persist the ID with the unchanged request. + A payload fingerprint rejects reusing that ID for a different mutation. +- `StaleBase`: the current version, including `contentBase64`. Nothing was written; + the client merges locally and submits a new push with a new request ID. + +The base check, path allocation, new version, and durable acknowledgement share +one SQLite write transaction. A deleted document stays deleted; recreating a file +uses a new document ID. Occupied paths receive an available numbered filename. +Text updates accept reconcile-text transport diffs against the base version; +the server only reconstructs snapshots and never merges concurrent edits. + +Snapshot requests use base64, adding roughly one third to the raw file size. +Server and reverse-proxy request limits apply to the encoded JSON body. + +API v3 uses a fresh schema. Update clients together with the server and initialise +new vault databases; compatibility with previous releases is not supported. + ## Creating/resetting the Database for development ```sh diff --git a/sync-server/build.rs b/sync-server/build.rs index d5068697..ee539ea3 100644 --- a/sync-server/build.rs +++ b/sync-server/build.rs @@ -1,5 +1,5 @@ // generated by `sqlx migrate build-script` fn main() { // trigger recompilation when a new migration is added - println!("cargo:rerun-if-changed=migrations"); + println!("cargo:rerun-if-changed=src/app_state/database/migrations"); } diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 75ce6df4..216a537d 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -3,21 +3,24 @@ use std::{collections::HashMap, sync::Arc}; use anyhow::{Context as _, Result}; use log::info; -use models::{ - DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId, -}; -use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions, types::chrono::Utc}; - -pub mod models; +use models::VaultId; +use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions}; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; use tokio::sync::Mutex; use tokio::time::Instant; -use uuid::fmt::Hyphenated; -use super::websocket::{ - broadcasts::Broadcasts, - models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate}, -}; +pub mod models; + +mod get_document_version; +mod get_latest_document; +mod get_latest_document_by_path; +mod get_latest_documents; +mod get_latest_documents_since; +mod get_max_update_id_in_vault; +mod get_push_acknowledgement; +mod insert_document_version; + +use super::websocket::broadcasts::Broadcasts; use crate::config::database_config::DatabaseConfig; #[derive(Clone)] @@ -176,305 +179,6 @@ impl Database { Ok(transaction) } - /// Return the latest state of all documents in the vault - pub async fn get_latest_documents( - &self, - vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query!( - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - order by vault_update_id - "#, - ); - - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await - } else { - query - .fetch_all(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest documents") - .map(|rows| { - rows.into_iter() - .map(|row| DocumentVersionWithoutContent { - vault_update_id: row.vault_update_id, - document_id: row.document_id.into(), - relative_path: row.relative_path, - updated_date: row.updated_date, - is_deleted: row.is_deleted, - user_id: row.user_id, - device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), - }) - .collect() - }) - } - - /// Return the latest state of all documents (including deleted) in the - /// vault which have changed since the given update id - pub async fn get_latest_documents_since( - &self, - vault: &VaultId, - vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query!( - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - where vault_update_id > ? - order by vault_update_id - "#, - vault_update_id - ); - - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await - } else { - query - .fetch_all(&self.get_connection_pool(vault).await?) - .await - } - .with_context(|| { - format!("Cannot fetch latest documents since vault_update_id `{vault_update_id}`") - }) - .map(|rows| { - rows.into_iter() - .map(|row| DocumentVersionWithoutContent { - vault_update_id: row.vault_update_id, - document_id: row.document_id.into(), - relative_path: row.relative_path, - updated_date: row.updated_date, - is_deleted: row.is_deleted, - user_id: row.user_id, - device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), - }) - .collect() - }) - } - - pub async fn get_max_update_id_in_vault( - &self, - vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result { - let query = sqlx::query!( - r#" - select coalesce(max(vault_update_id), 0) as max_vault_update_id - from documents - "#, - ); - - if let Some(transaction) = transaction { - query.fetch_one(&mut **transaction).await - } else { - query - .fetch_one(&self.get_connection_pool(vault).await?) - .await - } - .map(|row| row.max_vault_update_id) - .context("Cannot fetch max update id in vault") - } - - pub async fn get_latest_document_by_path( - &self, - vault: &VaultId, - relative_path: &str, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id, - has_been_merged - from latest_document_versions - where relative_path = ? and is_deleted = false - order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however, - -- multiple documents can have the same `relative_path`, if they have been deleted. That's - -- why we only care about the latest version of the document with the given relative path. - limit 1 - "#, - relative_path - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest document version") - } - - pub async fn get_latest_document( - &self, - vault: &VaultId, - document_id: &DocumentId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let document_id = document_id.as_hyphenated(); - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id, - has_been_merged - from latest_document_versions - where document_id = ? - "#, - document_id - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest document version") - } - - pub async fn get_document_version( - &self, - vault: &VaultId, - vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id, - has_been_merged - from documents - where vault_update_id = ?"#, - vault_update_id - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch document version") - } - - // inserting the document must be the last step of the transaction if there's one - pub async fn insert_document_version( - &self, - vault_id: &VaultId, - version: &StoredDocumentVersion, - transaction: Option>, - ) -> Result<()> { - let document_id = version.document_id.as_hyphenated(); - let query = sqlx::query!( - r#" - insert into documents ( - vault_update_id, - document_id, - relative_path, - updated_date, - content, - is_deleted, - user_id, - device_id - ) - values (?, ?, ?, ?, ?, ?, ?, ?) - "#, - version.vault_update_id, - document_id, - version.relative_path, - version.updated_date, - version.content, - version.is_deleted, - version.user_id, - version.device_id - ); - - if let Some(mut transaction) = transaction { - query - .execute(&mut *transaction) - .await - .context("Cannot insert document version")?; - - transaction - .commit() - .await - .context("Failed to commit transaction")?; - } else { - query - .execute(&self.get_connection_pool(vault_id).await?) - .await - .context("Cannot insert document version")?; - } - - self.broadcasts - .send_document_update( - vault_id.clone(), - WebSocketServerMessageWithOrigin::with_origin( - version.device_id.clone(), - WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: vec![version.clone().into()], - is_initial_sync: false, - }), - ), - ) - .await; - - Ok(()) - } - /// Cleanup idle connection pools that haven't been accessed in more than 5 minutes async fn cleanup_idle_pools(&self) { let mut pools = self.connection_pools.lock().await; diff --git a/sync-server/src/app_state/database/get_document_version.rs b/sync-server/src/app_state/database/get_document_version.rs new file mode 100644 index 00000000..c4113d8b --- /dev/null +++ b/sync-server/src/app_state/database/get_document_version.rs @@ -0,0 +1,41 @@ +use anyhow::{Context as _, Result}; +use sqlx::types::chrono::Utc; +use uuid::fmt::Hyphenated; + +use super::models::{StoredDocumentVersion, VaultId, VaultUpdateId}; +use super::{Database, Transaction}; + +impl Database { + pub async fn get_document_version( + &self, + vault: &VaultId, + vault_update_id: VaultUpdateId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query_as!( + StoredDocumentVersion, + r#" + select + vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + content, + is_deleted, + user_id, + device_id + from documents + where vault_update_id = ?"#, + vault_update_id + ); + + if let Some(transaction) = transaction { + query.fetch_optional(&mut **transaction).await + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch document version") + } +} diff --git a/sync-server/src/app_state/database/get_latest_document.rs b/sync-server/src/app_state/database/get_latest_document.rs new file mode 100644 index 00000000..0bb6ecb6 --- /dev/null +++ b/sync-server/src/app_state/database/get_latest_document.rs @@ -0,0 +1,43 @@ +use anyhow::{Context as _, Result}; +use sqlx::types::chrono::Utc; +use uuid::fmt::Hyphenated; + +use super::models::{DocumentId, StoredDocumentVersion, VaultId}; +use super::{Database, Transaction}; + +impl Database { + pub async fn get_latest_document( + &self, + vault: &VaultId, + document_id: &DocumentId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let document_id = document_id.as_hyphenated(); + let query = sqlx::query_as!( + StoredDocumentVersion, + r#" + select + vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + content, + is_deleted, + user_id, + device_id + from latest_document_versions + where document_id = ? + "#, + document_id + ); + + if let Some(transaction) = transaction { + query.fetch_optional(&mut **transaction).await + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch latest document version") + } +} diff --git a/sync-server/src/app_state/database/get_latest_document_by_path.rs b/sync-server/src/app_state/database/get_latest_document_by_path.rs new file mode 100644 index 00000000..95e49875 --- /dev/null +++ b/sync-server/src/app_state/database/get_latest_document_by_path.rs @@ -0,0 +1,46 @@ +use anyhow::{Context as _, Result}; +use sqlx::types::chrono::Utc; +use uuid::fmt::Hyphenated; + +use super::models::{StoredDocumentVersion, VaultId}; +use super::{Database, Transaction}; + +impl Database { + pub async fn get_latest_document_by_path( + &self, + vault: &VaultId, + relative_path: &str, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query_as!( + StoredDocumentVersion, + r#" + select + vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + content, + is_deleted, + user_id, + device_id + from latest_document_versions + where relative_path = ? and is_deleted = false + order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however, + -- multiple documents can have the same `relative_path`, if they have been deleted. That's + -- why we only care about the latest version of the document with the given relative path. + limit 1 + "#, + relative_path + ); + + if let Some(transaction) = transaction { + query.fetch_optional(&mut **transaction).await + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch latest document version") + } +} diff --git a/sync-server/src/app_state/database/get_latest_documents.rs b/sync-server/src/app_state/database/get_latest_documents.rs new file mode 100644 index 00000000..700a505c --- /dev/null +++ b/sync-server/src/app_state/database/get_latest_documents.rs @@ -0,0 +1,56 @@ +use anyhow::{Context as _, Result}; +use sqlx::types::chrono::Utc; +use uuid::fmt::Hyphenated; + +use super::models::{DocumentVersionWithoutContent, VaultId}; +use super::{Database, Transaction}; + +impl Database { + /// Return the latest state of all documents in the vault + pub async fn get_latest_documents( + &self, + vault: &VaultId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query!( + r#" + select + vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + is_deleted, + user_id, + device_id, + length(content) as "content_size: u64" + from latest_document_versions + order by vault_update_id + "#, + ); + + if let Some(transaction) = transaction { + query.fetch_all(&mut **transaction).await + } else { + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await + } + .context("Cannot fetch latest documents") + .map(|rows| { + rows.into_iter() + .map(|row| DocumentVersionWithoutContent { + vault_update_id: row.vault_update_id, + document_id: row.document_id.into(), + relative_path: row.relative_path, + updated_date: row.updated_date, + is_deleted: row.is_deleted, + user_id: row.user_id, + device_id: row.device_id, + content_size: row + .content_size + .expect("Content size can't be null but sqlx can't infer it"), + }) + .collect() + }) + } +} diff --git a/sync-server/src/app_state/database/get_latest_documents_since.rs b/sync-server/src/app_state/database/get_latest_documents_since.rs new file mode 100644 index 00000000..48c2f0e9 --- /dev/null +++ b/sync-server/src/app_state/database/get_latest_documents_since.rs @@ -0,0 +1,62 @@ +use anyhow::{Context as _, Result}; +use sqlx::types::chrono::Utc; +use uuid::fmt::Hyphenated; + +use super::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId}; +use super::{Database, Transaction}; + +impl Database { + /// Return the latest state of all documents (including deleted) in the + /// vault which have changed since the given update id + pub async fn get_latest_documents_since( + &self, + vault: &VaultId, + vault_update_id: VaultUpdateId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query!( + r#" + select + vault_update_id, + document_id as "document_id: Hyphenated", + relative_path, + updated_date as "updated_date: chrono::DateTime", + is_deleted, + user_id, + device_id, + length(content) as "content_size: u64" + from latest_document_versions + where vault_update_id > ? + order by vault_update_id + "#, + vault_update_id + ); + + if let Some(transaction) = transaction { + query.fetch_all(&mut **transaction).await + } else { + query + .fetch_all(&self.get_connection_pool(vault).await?) + .await + } + .with_context(|| { + format!("Cannot fetch latest documents since vault_update_id `{vault_update_id}`") + }) + .map(|rows| { + rows.into_iter() + .map(|row| DocumentVersionWithoutContent { + vault_update_id: row.vault_update_id, + document_id: row.document_id.into(), + relative_path: row.relative_path, + updated_date: row.updated_date, + is_deleted: row.is_deleted, + user_id: row.user_id, + device_id: row.device_id, + content_size: row + .content_size + .expect("Content size can't be null but sqlx can't infer it"), + }) + .collect() + }) + } +} diff --git a/sync-server/src/app_state/database/get_max_update_id_in_vault.rs b/sync-server/src/app_state/database/get_max_update_id_in_vault.rs new file mode 100644 index 00000000..cc4a43c7 --- /dev/null +++ b/sync-server/src/app_state/database/get_max_update_id_in_vault.rs @@ -0,0 +1,29 @@ +use anyhow::{Context as _, Result}; + +use super::models::VaultId; +use super::{Database, Transaction}; + +impl Database { + pub async fn get_max_update_id_in_vault( + &self, + vault: &VaultId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result { + let query = sqlx::query!( + r#" + select coalesce(max(vault_update_id), 0) as max_vault_update_id + from documents + "#, + ); + + if let Some(transaction) = transaction { + query.fetch_one(&mut **transaction).await + } else { + query + .fetch_one(&self.get_connection_pool(vault).await?) + .await + } + .map(|row| row.max_vault_update_id) + .context("Cannot fetch max update id in vault") + } +} diff --git a/sync-server/src/app_state/database/get_push_acknowledgement.rs b/sync-server/src/app_state/database/get_push_acknowledgement.rs new file mode 100644 index 00000000..cd8cd593 --- /dev/null +++ b/sync-server/src/app_state/database/get_push_acknowledgement.rs @@ -0,0 +1,38 @@ +use anyhow::{Context as _, Result}; + +use super::models::{StoredDocumentVersion, VaultId}; +use super::{Database, Transaction}; + +impl Database { + /// Look up the immutable version acknowledged by a previous push. + pub async fn get_push_acknowledgement( + &self, + vault_id: &VaultId, + request_id: uuid::Uuid, + transaction: &mut Transaction<'_>, + ) -> Result, StoredDocumentVersion)>> { + let request_id = request_id.hyphenated().to_string(); + let acknowledgement = sqlx::query!( + "select vault_update_id, request_fingerprint from push_acknowledgements where request_id = ?", + request_id + ) + .fetch_optional(&mut **transaction) + .await + .context("Cannot fetch push acknowledgement")?; + + match acknowledgement { + Some(acknowledgement) => { + let version = self + .get_document_version( + vault_id, + acknowledgement.vault_update_id, + Some(transaction), + ) + .await? + .context("Acknowledged document version is missing")?; + Ok(Some((acknowledgement.request_fingerprint, version))) + } + None => Ok(None), + } + } +} diff --git a/sync-server/src/app_state/database/insert_document_version.rs b/sync-server/src/app_state/database/insert_document_version.rs new file mode 100644 index 00000000..16752471 --- /dev/null +++ b/sync-server/src/app_state/database/insert_document_version.rs @@ -0,0 +1,79 @@ +use anyhow::{Context as _, Result}; + +use super::models::{StoredDocumentVersion, VaultId}; +use super::{Database, Transaction}; +use crate::app_state::websocket::models::{ + WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate, +}; + +impl Database { + /// Commit the snapshot and acknowledgement atomically, then notify subscribers. + pub async fn insert_document_version( + &self, + vault_id: &VaultId, + version: &StoredDocumentVersion, + request_id: uuid::Uuid, + request_fingerprint: &[u8], + mut transaction: Transaction<'_>, + ) -> Result<()> { + let document_id = version.document_id.as_hyphenated(); + sqlx::query!( + r#" + insert into documents ( + vault_update_id, + document_id, + relative_path, + updated_date, + content, + is_deleted, + user_id, + device_id + ) + values (?, ?, ?, ?, ?, ?, ?, ?) + "#, + version.vault_update_id, + document_id, + version.relative_path, + version.updated_date, + version.content, + version.is_deleted, + version.user_id, + version.device_id + ) + .execute(&mut *transaction) + .await + .context("Cannot insert document version")?; + + let request_id = request_id.hyphenated().to_string(); + sqlx::query!( + "insert into push_acknowledgements (request_id, request_fingerprint, vault_update_id) values (?, ?, ?)", + request_id, + request_fingerprint, + version.vault_update_id + ) + .execute(&mut *transaction) + .await + .context("Cannot save push acknowledgement")?; + + transaction + .commit() + .await + .context("Failed to commit push")?; + + // notifying the listeners here means that no update can circumvent this step + self.broadcasts + .send_document_update( + vault_id.clone(), + WebSocketServerMessageWithOrigin::with_origin( + version.device_id.clone(), + WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { + documents: vec![version.clone().into()], + is_initial_sync: false, + }), + ), + ) + .await; + + Ok(()) + } +} diff --git a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql index 4a9f31ba..0b3076c6 100644 --- a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql +++ b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql @@ -4,7 +4,9 @@ CREATE TABLE IF NOT EXISTS documents ( relative_path TEXT NOT NULL, updated_date TIMESTAMP NOT NULL, content BLOB NOT NULL, - is_deleted BOOLEAN NOT NULL + is_deleted BOOLEAN NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT NOT NULL ); CREATE VIEW IF NOT EXISTS latest_document_versions AS @@ -19,3 +21,10 @@ ON d.vault_update_id = max_versions.max_version_id; CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path ON documents (relative_path); + +-- The immutable version and its acknowledgement commit in one transaction. +CREATE TABLE push_acknowledgements ( + request_id TEXT NOT NULL PRIMARY KEY, + request_fingerprint BLOB NOT NULL, + vault_update_id INTEGER NOT NULL REFERENCES documents(vault_update_id) +); diff --git a/sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql b/sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql deleted file mode 100644 index 06860174..00000000 --- a/sync-server/src/app_state/database/migrations/20250522192949_add_provenance_columns.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE documents ADD COLUMN user_id TEXT NOT NULL DEFAULT ""; -ALTER TABLE documents ADD COLUMN device_id TEXT NOT NULL DEFAULT ""; diff --git a/sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql b/sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql deleted file mode 100644 index 5259c175..00000000 --- a/sync-server/src/app_state/database/migrations/20251019103154_add_has_been_merged.sql +++ /dev/null @@ -1,13 +0,0 @@ -ALTER TABLE documents ADD COLUMN has_been_merged BOOLEAN NOT NULL DEFAULT False; - -DROP VIEW latest_document_versions; - -CREATE VIEW IF NOT EXISTS latest_document_versions AS --recreate view as it now includes one more field -SELECT d.* -FROM documents d -INNER JOIN ( - SELECT MAX(vault_update_id) AS max_version_id - FROM documents - GROUP BY document_id -) max_versions -ON d.vault_update_id = max_versions.max_version_id; diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index a216125a..24c0c370 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -20,8 +20,6 @@ pub struct StoredDocumentVersion { pub is_deleted: bool, pub user_id: UserId, pub device_id: DeviceId, - #[allow(dead_code)] // This is for manual analysis - pub has_been_merged: bool, } impl PartialEq for StoredDocumentVersion { diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index 98ed1c1f..9e9890c0 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -20,4 +20,4 @@ pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; -pub const SUPPORTED_API_VERSION: u32 = 2; +pub const SUPPORTED_API_VERSION: u32 = 3; diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 01b09cf6..b8304a6c 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -1,17 +1,8 @@ pub mod auth; -mod create_document; -mod delete_document; mod device_id_header; -mod fetch_document_version; -mod fetch_document_version_content; -mod fetch_latest_document_version; -mod fetch_latest_documents; -mod index; -mod ping; +mod endpoints; mod requests; mod responses; -mod update_document; -mod websocket; use anyhow::{Context as _, Result, anyhow}; use auth::auth_middleware; @@ -21,7 +12,7 @@ use axum::{ http::{self, HeaderValue, Method}, middleware, response::IntoResponse, - routing::{IntoMakeService, delete, get, post, put}, + routing::{IntoMakeService, get}, }; use device_id_header::DEVICE_ID_HEADER_NAME; use log::info; @@ -53,9 +44,12 @@ pub async fn create_server(config: Config) -> Result<()> { let app = Router::new() .nest("/", get_authed_routes(app_state.clone())) - .route("/", get(index::index)) - .route("/vaults/:vault_id/ping", get(ping::ping)) - .route("/vaults/:vault_id/ws", get(websocket::websocket_handler)) + .route("/", get(endpoints::index::index)) + .route("/vaults/:vault_id/ping", get(endpoints::ping::ping)) + .route( + "/vaults/:vault_id/ws", + get(endpoints::websocket::websocket_handler), + ) .layer(DefaultBodyLimit::disable()) .layer(RequestBodyLimitLayer::new( app_state.config.server.max_body_size_mb * 1024 * 1024, @@ -69,7 +63,7 @@ pub async fn create_server(config: Config) -> Result<()> { http::header::AUTHORIZATION, DEVICE_ID_HEADER_NAME.clone(), ]) - .allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]), + .allow_methods([Method::GET, Method::PUT]), ) .layer( TraceLayer::new_for_http() @@ -102,35 +96,16 @@ fn get_authed_routes(app_state: AppState) -> Router { Router::new() .route( "/vaults/:vault_id/documents", - get(fetch_latest_documents::fetch_latest_documents), - ) - .route( - "/vaults/:vault_id/documents", - post(create_document::create_document), + get(endpoints::fetch_latest_documents::fetch_latest_documents), ) .route( "/vaults/:vault_id/documents/:document_id", - get(fetch_latest_document_version::fetch_latest_document_version), - ) - .route( - "/vaults/:vault_id/documents/:document_id/binary", - put(update_document::update_binary), - ) - .route( - "/vaults/:vault_id/documents/:document_id/text", - put(update_document::update_text), - ) - .route( - "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id", - get(fetch_document_version::fetch_document_version), + get(endpoints::fetch_latest_document_version::fetch_latest_document_version) + .put(endpoints::push_document::push_document), ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id/content", - get(fetch_document_version_content::fetch_document_version_content), - ) - .route( - "/vaults/:vault_id/documents/:document_id", - delete(delete_document::delete_document), + get(endpoints::fetch_document_version_content::fetch_document_version_content), ) .layer(middleware::from_fn_with_state(app_state, auth_middleware)) } diff --git a/sync-server/src/server/create_document.rs b/sync-server/src/server/create_document.rs deleted file mode 100644 index 859c0db4..00000000 --- a/sync-server/src/server/create_document.rs +++ /dev/null @@ -1,109 +0,0 @@ -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use axum_typed_multipart::TypedMultipart; -use log::{debug, info}; -use serde::Deserialize; - -use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion}; -use crate::{ - app_state::{ - AppState, - database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, - }, - config::user_config::User, - errors::{SyncServerError, client_error, server_error}, - utils::{ - find_first_available_path::find_first_available_path, normalize::normalize, - sanitize_path::sanitize_path, - }, -}; - -#[derive(Deserialize)] -pub struct CreateDocumentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} - -/// Create a new document in case a document with the same doesn't exist -/// already. If a document with the same path exists, a new version is created -/// with their content merged. -#[axum::debug_handler] -pub async fn create_document( - Path(CreateDocumentPathParams { vault_id }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - TypedMultipart(request): TypedMultipart, -) -> Result, SyncServerError> { - debug!("Creating document in vault `{vault_id}`"); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; - - let document_id = match request.document_id { - Some(document_id) => { - let existing_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - if existing_version.is_some() { - return Err(client_error(anyhow::anyhow!( - "Document with the same ID `{document_id}` already exists" - ))); - } - - document_id - } - None => uuid::Uuid::new_v4(), - }; - - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - let sanitized_relative_path = sanitize_path(&request.relative_path); - let deduped_path = find_first_available_path( - &vault_id, - &sanitized_relative_path, - &state.database, - &mut transaction, - ) - .await - .map_err(server_error)?; - - if deduped_path != sanitized_relative_path { - info!( - "Document already exists at new location: `{sanitized_relative_path}` when trying to create it in vault `{vault_id}`, deconflicting by creating at `{deduped_path}`" - ); - } - - let new_version = StoredDocumentVersion { - vault_update_id: last_update_id + 1, - document_id, - relative_path: deduped_path, - content: request.content.contents.to_vec(), - updated_date: chrono::Utc::now(), - is_deleted: false, - user_id: user.name, - device_id: device_id.0, - has_been_merged: false, - }; - - state - .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) - .await - .map_err(server_error)?; - - Ok(Json(new_version.into())) -} diff --git a/sync-server/src/server/delete_document.rs b/sync-server/src/server/delete_document.rs deleted file mode 100644 index e126d6b5..00000000 --- a/sync-server/src/server/delete_document.rs +++ /dev/null @@ -1,96 +0,0 @@ -use anyhow::Context; -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use log::{debug, info}; -use serde::Deserialize; - -use super::{device_id_header::DeviceIdHeader, requests::DeleteDocumentVersion}; -use crate::{ - app_state::{ - AppState, - database::models::{ - DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, - }, - }, - config::user_config::User, - errors::{SyncServerError, server_error}, - utils::{normalize::normalize, sanitize_path::sanitize_path}, -}; - -#[derive(Deserialize)] -pub struct DeleteDocumentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, -} - -#[axum::debug_handler] -pub async fn delete_document( - Path(DeleteDocumentPathParams { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - Json(request): Json, -) -> Result, SyncServerError> { - debug!("Deleting document `{document_id}` in vault `{vault_id}`"); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; - - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - let latest_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - if let Some(latest_version) = &latest_version - && latest_version.is_deleted - { - transaction - .rollback() - .await - .context("Failed to roll back transaction") - .map_err(server_error)?; - - info!("Document `{document_id}` has already been deleted",); - return Ok(Json(latest_version.clone().into())); - } - - let latest_content = latest_version.map_or_else(Vec::new, |version| version.content); // in case the document has never existed before deleting it - - let new_version = StoredDocumentVersion { - vault_update_id: last_update_id + 1, - document_id, - relative_path: sanitize_path(&request.relative_path), - content: latest_content, // copy the content from the latest version - updated_date: chrono::Utc::now(), - is_deleted: true, - user_id: user.name, - device_id: device_id.0, - has_been_merged: false, - }; - - state - .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) - .await - .map_err(server_error)?; - - Ok(Json(new_version.into())) -} diff --git a/sync-server/src/server/endpoints.rs b/sync-server/src/server/endpoints.rs new file mode 100644 index 00000000..243cf7fa --- /dev/null +++ b/sync-server/src/server/endpoints.rs @@ -0,0 +1,7 @@ +pub mod fetch_document_version_content; +pub mod fetch_latest_document_version; +pub mod fetch_latest_documents; +pub mod index; +pub mod ping; +pub mod push_document; +pub mod websocket; diff --git a/sync-server/src/server/fetch_document_version_content.rs b/sync-server/src/server/endpoints/fetch_document_version_content.rs similarity index 100% rename from sync-server/src/server/fetch_document_version_content.rs rename to sync-server/src/server/endpoints/fetch_document_version_content.rs diff --git a/sync-server/src/server/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs similarity index 100% rename from sync-server/src/server/fetch_latest_document_version.rs rename to sync-server/src/server/endpoints/fetch_latest_document_version.rs diff --git a/sync-server/src/server/endpoints/fetch_latest_documents.rs b/sync-server/src/server/endpoints/fetch_latest_documents.rs new file mode 100644 index 00000000..eee16441 --- /dev/null +++ b/sync-server/src/server/endpoints/fetch_latest_documents.rs @@ -0,0 +1,36 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use serde::Deserialize; + +use crate::{ + app_state::{ + AppState, + database::models::{DocumentVersionWithoutContent, VaultId}, + }, + errors::{SyncServerError, server_error}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct FetchLatestDocumentsPathParams { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[axum::debug_handler] +pub async fn fetch_latest_documents( + Path(FetchLatestDocumentsPathParams { vault_id }): Path, + State(state): State, +) -> Result>, SyncServerError> { + debug!("Fetching latest documents in {vault_id}"); + + state + .database + .get_latest_documents(&vault_id, None) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/index.rs b/sync-server/src/server/endpoints/index.rs similarity index 68% rename from sync-server/src/server/index.rs rename to sync-server/src/server/endpoints/index.rs index 64b053f7..982b0b88 100644 --- a/sync-server/src/server/index.rs +++ b/sync-server/src/server/endpoints/index.rs @@ -1,7 +1,7 @@ use axum::response::{Html, IntoResponse}; pub async fn index() -> impl IntoResponse { - const HTML_CONTENT: &str = include_str!("./assets/index.html"); + const HTML_CONTENT: &str = include_str!("../assets/index.html"); let html_content = HTML_CONTENT; Html(html_content) } diff --git a/sync-server/src/server/ping.rs b/sync-server/src/server/endpoints/ping.rs similarity index 95% rename from sync-server/src/server/ping.rs rename to sync-server/src/server/endpoints/ping.rs index 31aa8acd..c42219be 100644 --- a/sync-server/src/server/ping.rs +++ b/sync-server/src/server/endpoints/ping.rs @@ -9,7 +9,7 @@ use axum_extra::{ use log::debug; use serde::Deserialize; -use super::{auth::auth, responses::PingResponse}; +use crate::server::{auth::auth, responses::PingResponse}; use crate::{ app_state::{AppState, database::models::VaultId}, consts::SUPPORTED_API_VERSION, diff --git a/sync-server/src/server/endpoints/push_document.rs b/sync-server/src/server/endpoints/push_document.rs new file mode 100644 index 00000000..676e8baa --- /dev/null +++ b/sync-server/src/server/endpoints/push_document.rs @@ -0,0 +1,210 @@ +use anyhow::{Context as _, anyhow}; +use axum::{ + Extension, Json, + extract::{Path, State}, +}; +use axum_extra::TypedHeader; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use reconcile_text::{BuiltinTokenizer, EditedText, NumberOrText}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use crate::server::{ + device_id_header::DeviceIdHeader, + requests::{PushContent, PushDocument}, + responses::DocumentUpdateResponse, +}; +use crate::{ + app_state::{ + AppState, + database::{ + Database, + models::{DocumentId, StoredDocumentVersion, VaultId}, + }, + }, + config::user_config::User, + errors::{SyncServerError, client_error, not_found_error, server_error}, + utils::{ + find_first_available_path::find_first_available_path, normalize::normalize, + sanitize_path::sanitize_path, + }, +}; + +#[derive(Deserialize)] +pub struct PushDocumentPath { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, + + document_id: DocumentId, +} + +#[axum::debug_handler] +pub async fn push_document( + Path(PushDocumentPath { + vault_id, + document_id, + }): Path, + Extension(user): Extension, + TypedHeader(device_id): TypedHeader, + State(state): State, + Json(push): Json, +) -> Result, SyncServerError> { + accept_push( + &state.database, + &vault_id, + document_id, + user.name, + device_id.0, + push, + ) + .await + .map(Json) +} + +/// A push is a compare-and-swap of one document's immutable head. The write lock +/// covers idempotency lookup, base validation, path allocation, and the commit. +async fn accept_push( + database: &Database, + vault_id: &VaultId, + document_id: DocumentId, + user_id: String, + device_id: String, + push: PushDocument, +) -> Result { + let fingerprint = fingerprint(document_id, &push); + + let mut transaction = database + .create_write_transaction(vault_id) + .await + .map_err(server_error)?; + + if let Some((stored_fingerprint, acknowledged)) = database + .get_push_acknowledgement(vault_id, push.request_id, &mut transaction) + .await + .map_err(server_error)? + { + if stored_fingerprint != fingerprint { + return Err(client_error(anyhow!( + "Request ID was reused with a different payload" + ))); + } + return Ok(DocumentUpdateResponse::Accepted(acknowledged.into())); + } + + let latest = database + .get_latest_document(vault_id, &document_id, Some(&mut transaction)) + .await + .map_err(server_error)?; + + match &latest { + Some(latest) + if latest.is_deleted || Some(latest.vault_update_id) != push.parent_version_id => + { + return Ok(DocumentUpdateResponse::StaleBase(latest.clone().into())); + } + + None if push.parent_version_id.is_some() => { + return Err(not_found_error(anyhow!( + "Document with id `{document_id}` not found" + ))); + } + + _ => {} + } + + let is_deleted = matches!(&push.content, PushContent::Delete); + let (content, requested_path) = match push.content { + PushContent::Snapshot(content) => ( + STANDARD + .decode(content) + .context("Invalid base64 snapshot") + .map_err(client_error)?, + sanitize_path(&push.relative_path), + ), + PushContent::Diff(diff) => ( + reconstruct_diff(latest.as_ref(), diff)?, + sanitize_path(&push.relative_path), + ), + PushContent::Delete => { + let latest = latest + .as_ref() + .ok_or_else(|| client_error(anyhow!("Cannot delete a missing document")))?; + (latest.content.clone(), latest.relative_path.clone()) + } + }; + + let relative_path = if latest + .as_ref() + .is_some_and(|version| version.relative_path == requested_path) + { + requested_path + } else { + find_first_available_path(vault_id, &requested_path, database, &mut transaction) + .await + .map_err(server_error)? + }; + + let new_version = StoredDocumentVersion { + vault_update_id: database + .get_max_update_id_in_vault(vault_id, Some(&mut transaction)) + .await + .map_err(server_error)? + + 1, + document_id, + relative_path, + content, + updated_date: chrono::Utc::now(), + is_deleted, + user_id, + device_id, + }; + + database + .insert_document_version( + vault_id, + &new_version, + push.request_id, + &fingerprint, + transaction, + ) + .await + .map_err(server_error)?; + Ok(DocumentUpdateResponse::Accepted(new_version.into())) +} + +fn reconstruct_diff( + parent: Option<&StoredDocumentVersion>, + diff: Vec, +) -> Result, SyncServerError> { + // reconcile-text 0.8 negates deletion lengths; i64::MIN would overflow. + if diff + .iter() + .any(|item| matches!(item, NumberOrText::Number(i64::MIN))) + { + return Err(client_error(anyhow!("Invalid diff length"))); + } + let parent = parent.ok_or_else(|| client_error(anyhow!("A diff requires a parent version")))?; + let parent_text = str::from_utf8(&parent.content) + .context("Parent is not UTF-8 text") + .map_err(client_error)?; + // Transport decoding only: reconciliation belongs exclusively to clients. + Ok( + EditedText::from_diff(parent_text, diff, &*BuiltinTokenizer::Word) + .context("Failed to apply diff to parent document") + .map_err(client_error)? + .apply() + .text() + .into_bytes(), + ) +} + +/// Session/device IDs may change across retries; the requested mutation may not. +fn fingerprint(document_id: DocumentId, push: &PushDocument) -> Vec { + Sha256::digest( + serde_json::to_vec(&(document_id, push)).expect("Document pushes are serializable"), + ) + .to_vec() +} + +#[cfg(test)] +mod tests; diff --git a/sync-server/src/server/endpoints/push_document/tests.rs b/sync-server/src/server/endpoints/push_document/tests.rs new file mode 100644 index 00000000..291c2def --- /dev/null +++ b/sync-server/src/server/endpoints/push_document/tests.rs @@ -0,0 +1,423 @@ +use std::path::PathBuf; + +use super::*; +use crate::{ + app_state::{ + database::models::{DocumentVersionWithoutContent, VaultUpdateId}, + websocket::broadcasts::Broadcasts, + }, + config::{database_config::DatabaseConfig, server_config::ServerConfig}, +}; + +struct Push { + document_id: DocumentId, + request_id: uuid::Uuid, + parent_version_id: Option, + relative_path: String, + content: PushContent, + user_id: String, + device_id: String, +} + +impl Push { + async fn send( + self, + database: &Database, + vault: &VaultId, + ) -> Result { + accept_push( + database, + vault, + self.document_id, + self.user_id, + self.device_id, + PushDocument { + request_id: self.request_id, + parent_version_id: self.parent_version_id, + relative_path: self.relative_path, + content: self.content, + }, + ) + .await + } +} + +struct TestVault { + directory: PathBuf, + database: Database, + vault: VaultId, +} + +impl TestVault { + async fn new() -> Self { + let directory = + std::env::temp_dir().join(format!("vaultlink-cas-{}", uuid::Uuid::new_v4())); + let database = Self::open(&directory).await; + Self { + directory, + database, + vault: "test".to_owned(), + } + } + + async fn open(directory: &std::path::Path) -> Database { + Database::try_new( + &DatabaseConfig { + databases_directory_path: directory.to_owned(), + ..Default::default() + }, + &Broadcasts::new(&ServerConfig { + max_clients_per_vault: 16, + ..Default::default() + }), + ) + .await + .unwrap() + } + + async fn send(&self, push: Push) -> DocumentUpdateResponse { + push.send(&self.database, &self.vault).await.unwrap() + } + + async fn latest(&self, document: DocumentId) -> StoredDocumentVersion { + self.database + .get_latest_document(&self.vault, &document, None) + .await + .unwrap() + .unwrap() + } +} + +impl Drop for TestVault { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.directory); + } +} + +fn snapshot(document_id: DocumentId, parent: Option, text: &str) -> Push { + Push { + document_id, + request_id: uuid::Uuid::new_v4(), + parent_version_id: parent, + relative_path: "note.md".to_owned(), + content: PushContent::Snapshot(STANDARD.encode(text)), + user_id: "user".to_owned(), + device_id: "device".to_owned(), + } +} + +fn accepted(response: DocumentUpdateResponse) -> DocumentVersionWithoutContent { + match response { + DocumentUpdateResponse::Accepted(version) => version, + response @ DocumentUpdateResponse::StaleBase(_) => { + panic!("Expected Accepted, got {response:?}") + } + } +} + +#[tokio::test] +async fn json_snapshot_preserves_bytes_and_rejects_invalid_base64_without_a_version() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let content = [0, 255, 13, 10, 42]; + let request: PushDocument = serde_json::from_value(serde_json::json!({ + "requestId": uuid::Uuid::new_v4(), + "parentVersionId": null, + "relativePath": "snapshot.bin", + "content": { "type": "Snapshot", "value": STANDARD.encode(content) }, + })) + .unwrap(); + let base = accepted( + accept_push( + &vault.database, + &vault.vault, + document, + "user".to_owned(), + "device".to_owned(), + request, + ) + .await + .unwrap(), + ); + assert_eq!(vault.latest(document).await.content, content); + + let invalid: PushDocument = serde_json::from_value(serde_json::json!({ + "requestId": uuid::Uuid::new_v4(), + "parentVersionId": base.vault_update_id, + "relativePath": "snapshot.bin", + "content": { "type": "Snapshot", "value": "not base64!" }, + })) + .unwrap(); + assert!(matches!( + accept_push( + &vault.database, + &vault.vault, + document, + "user".to_owned(), + "device".to_owned(), + invalid + ) + .await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(document).await.vault_update_id, + base.vault_update_id + ); +} + +#[tokio::test] +async fn concurrent_writers_only_accept_one_snapshot_and_preserve_the_base() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted(vault.send(snapshot(document, None, "base")).await); + let (left, right) = tokio::join!( + vault.send(snapshot(document, Some(base.vault_update_id), "left")), + vault.send(snapshot(document, Some(base.vault_update_id), "right")), + ); + let (winner, rejected) = match (left, right) { + (DocumentUpdateResponse::Accepted(winner), DocumentUpdateResponse::StaleBase(rejected)) + | (DocumentUpdateResponse::StaleBase(rejected), DocumentUpdateResponse::Accepted(winner)) => { + (winner, rejected) + } + outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"), + }; + let latest = vault.latest(document).await; + assert_eq!(winner.vault_update_id, rejected.vault_update_id); + assert_eq!( + STANDARD.decode(rejected.content_base64).unwrap(), + latest.content + ); + assert!(latest.content == b"left" || latest.content == b"right"); + let original = vault + .database + .get_document_version(&vault.vault, base.vault_update_id, None) + .await + .unwrap() + .unwrap(); + assert_eq!(original.content, b"base"); + assert_eq!( + vault + .database + .get_max_update_id_in_vault(&vault.vault, None) + .await + .unwrap(), + 2 + ); +} + +#[tokio::test] +async fn acknowledgements_survive_reopening_and_return_the_original_version() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let create = snapshot(document, None, "created"); + let create_id = create.request_id; + let base = accepted(vault.send(create).await); + let update = snapshot(document, Some(base.vault_update_id), "updated"); + let update_id = update.request_id; + let updated = accepted(vault.send(update).await); + let latest = accepted( + vault + .send(snapshot(document, Some(updated.vault_update_id), "newest")) + .await, + ); + + let reopened = TestVault::open(&vault.directory).await; + for (request_id, parent, text, original) in [ + (create_id, None, "created", base), + (update_id, Some(1), "updated", updated), + ] { + let mut retry = snapshot(document, parent, text); + retry.request_id = request_id; + retry.device_id = "new-session-after-restart".to_owned(); + let acknowledged = accepted(retry.send(&reopened, &vault.vault).await.unwrap()); + assert_eq!( + serde_json::to_value(acknowledged).unwrap(), + serde_json::to_value(original).unwrap() + ); + } + assert_eq!( + vault.latest(document).await.vault_update_id, + latest.vault_update_id + ); + + let mut reused = snapshot(uuid::Uuid::new_v4(), None, "wrong document"); + reused.request_id = create_id; + assert!(matches!( + reused.send(&reopened, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + + let mut changed_path = snapshot(document, None, "created"); + changed_path.relative_path = "other.md".to_owned(); + let mut changed_operation = snapshot(document, None, "created"); + changed_operation.content = PushContent::Delete; + for mut changed_payload in [ + snapshot(document, None, "different content"), + snapshot(document, Some(1), "created"), + changed_path, + changed_operation, + ] { + changed_payload.request_id = create_id; + assert!(matches!( + changed_payload.send(&reopened, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + } +} + +#[tokio::test] +async fn deletes_compare_the_base_and_replay_without_resurrecting_documents() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted(vault.send(snapshot(document, None, "base")).await); + let updated = accepted( + vault + .send(snapshot(document, Some(base.vault_update_id), "edited")) + .await, + ); + let mut delete = snapshot(document, Some(base.vault_update_id), ""); + delete.content = PushContent::Delete; + assert!(matches!( + vault.send(delete).await, + DocumentUpdateResponse::StaleBase(_) + )); + assert!(!vault.latest(document).await.is_deleted); + + let mut delete = snapshot(document, Some(updated.vault_update_id), ""); + delete.content = PushContent::Delete; + let request_id = delete.request_id; + let deleted = accepted(vault.send(delete).await); + assert!(deleted.is_deleted); + assert_eq!(vault.latest(document).await.content, b"edited"); + let mut retry = snapshot(document, Some(updated.vault_update_id), ""); + retry.content = PushContent::Delete; + retry.request_id = request_id; + assert_eq!( + accepted(vault.send(retry).await).vault_update_id, + deleted.vault_update_id + ); + + let terminal = vault + .send(snapshot( + document, + Some(deleted.vault_update_id), + "resurrect", + )) + .await; + assert!(matches!(terminal, DocumentUpdateResponse::StaleBase(version) if version.is_deleted)); + let recreated = accepted( + vault + .send(snapshot(uuid::Uuid::new_v4(), None, "new identity")) + .await, + ); + assert_eq!(recreated.relative_path, "note.md"); +} + +#[tokio::test] +async fn deleting_a_missing_document_without_a_parent_is_a_client_error() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let mut delete = snapshot(document, None, ""); + delete.content = PushContent::Delete; + assert!(matches!( + delete.send(&vault.database, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault + .database + .get_latest_document(&vault.vault, &document, None) + .await + .unwrap(), + None + ); +} + +#[tokio::test] +async fn transport_diffs_reconstruct_exact_snapshots_and_reject_invalid_bases() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let original = "Hello world 🌍"; + let desired = "Hi beautiful world 🌍"; + let base = accepted(vault.send(snapshot(document, None, original)).await); + let mut update = snapshot(document, Some(base.vault_update_id), ""); + update.content = + PushContent::Diff(EditedText::from_strings(original, &desired.into()).to_diff()); + let updated = accepted(vault.send(update).await); + assert_eq!(vault.latest(document).await.content, desired.as_bytes()); + + for invalid_length in [i64::MIN, i64::MAX] { + let mut malformed = snapshot(document, Some(updated.vault_update_id), ""); + malformed.content = PushContent::Diff(vec![NumberOrText::Number(invalid_length)]); + assert!(matches!( + malformed.send(&vault.database, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + } + assert_eq!( + vault.latest(document).await.vault_update_id, + updated.vault_update_id + ); + + let another = accepted( + vault + .send(snapshot(uuid::Uuid::new_v4(), None, "another document")) + .await, + ); + let mut foreign_base = snapshot(document, Some(another.vault_update_id), ""); + foreign_base.content = PushContent::Diff(vec![]); + assert!( + matches!(vault.send(foreign_base).await, DocumentUpdateResponse::StaleBase(version) if version.vault_update_id == updated.vault_update_id) + ); + + let binary_document = uuid::Uuid::new_v4(); + let mut binary = snapshot(binary_document, None, ""); + binary.content = PushContent::Snapshot(STANDARD.encode([0xff])); + let binary_base = accepted(vault.send(binary).await); + let mut invalid_text = snapshot(binary_document, Some(binary_base.vault_update_id), ""); + invalid_text.content = PushContent::Diff(vec![]); + assert!(matches!( + invalid_text.send(&vault.database, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(binary_document).await.vault_update_id, + binary_base.vault_update_id + ); +} + +#[tokio::test] +async fn renames_allocate_paths_atomically_without_rewriting_history() { + let vault = TestVault::new().await; + let first = uuid::Uuid::new_v4(); + let second = uuid::Uuid::new_v4(); + let base = accepted(vault.send(snapshot(first, None, "first")).await); + let collision = accepted(vault.send(snapshot(second, None, "second")).await); + assert_ne!(base.relative_path, collision.relative_path); + + let mut rename = snapshot(first, Some(base.vault_update_id), "first"); + rename.relative_path = "renamed.md".to_owned(); + let renamed = accepted(vault.send(rename).await); + let mut stale_rename = snapshot(first, Some(base.vault_update_id), "first"); + stale_rename.relative_path = "stale.md".to_owned(); + assert!( + matches!(vault.send(stale_rename).await, DocumentUpdateResponse::StaleBase(version) if version.relative_path == renamed.relative_path) + ); + + let mut move_into_occupied = snapshot(second, Some(collision.vault_update_id), "second"); + move_into_occupied.relative_path = renamed.relative_path.clone(); + let allocated = accepted(vault.send(move_into_occupied).await); + assert_ne!(allocated.relative_path, renamed.relative_path); + assert_eq!(vault.latest(first).await.relative_path, "renamed.md"); + assert_eq!( + vault + .database + .get_document_version(&vault.vault, base.vault_update_id, None) + .await + .unwrap() + .unwrap() + .relative_path, + "note.md" + ); +} diff --git a/sync-server/src/server/websocket.rs b/sync-server/src/server/endpoints/websocket.rs similarity index 100% rename from sync-server/src/server/websocket.rs rename to sync-server/src/server/endpoints/websocket.rs diff --git a/sync-server/src/server/fetch_document_version.rs b/sync-server/src/server/fetch_document_version.rs deleted file mode 100644 index c30f1d76..00000000 --- a/sync-server/src/server/fetch_document_version.rs +++ /dev/null @@ -1,62 +0,0 @@ -use anyhow::anyhow; -use axum::{ - Json, - extract::{Path, State}, -}; -use log::debug; -use serde::Deserialize; - -use crate::{ - app_state::{ - AppState, - database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId}, - }, - errors::{SyncServerError, not_found_error, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct FetchDocumentVersionPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, - vault_update_id: VaultUpdateId, -} - -#[axum::debug_handler] -pub async fn fetch_document_version( - Path(FetchDocumentVersionPathParams { - vault_id, - document_id, - vault_update_id, - }): Path, - State(state): State, -) -> Result, SyncServerError> { - debug!( - "Fetching document version `{vault_update_id}` for document `{document_id}` in vault `{vault_id}`" - ); - - let result = state - .database - .get_document_version(&vault_id, vault_update_id, None) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Document with vault update id `{vault_update_id}` not found", - ))) - }, - Ok, - )?; - - if result.document_id != document_id { - return Err(not_found_error(anyhow!( - "Document with document id `{document_id}` does not have a version with id \ - `{vault_update_id}`", - ))); - } - - Ok(Json(result.into())) -} diff --git a/sync-server/src/server/fetch_latest_documents.rs b/sync-server/src/server/fetch_latest_documents.rs deleted file mode 100644 index 209374ce..00000000 --- a/sync-server/src/server/fetch_latest_documents.rs +++ /dev/null @@ -1,59 +0,0 @@ -use axum::{ - Json, - extract::{Path, Query, State}, -}; -use log::debug; -use serde::Deserialize; - -use super::responses::FetchLatestDocumentsResponse; -use crate::{ - app_state::{ - AppState, - database::models::{VaultId, VaultUpdateId}, - }, - errors::{SyncServerError, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct FetchLatestDocumentsPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} - -#[derive(Deserialize)] -pub struct QueryParams { - since_update_id: Option, -} - -#[axum::debug_handler] -pub async fn fetch_latest_documents( - Path(FetchLatestDocumentsPathParams { vault_id }): Path, - Query(QueryParams { since_update_id }): Query, - State(state): State, -) -> Result, SyncServerError> { - debug!("Fetching latest documents in vault `{vault_id}` since update ID `{since_update_id:?}`"); - - let documents = if let Some(since_update_id) = since_update_id { - state - .database - .get_latest_documents_since(&vault_id, since_update_id, None) - .await - .map_err(server_error) - } else { - state - .database - .get_latest_documents(&vault_id, None) - .await - .map_err(server_error) - }?; - - Ok(Json(FetchLatestDocumentsResponse { - last_update_id: documents - .iter() - .map(|doc| doc.vault_update_id) - .max() - .unwrap_or(since_update_id.unwrap_or(0)), - latest_documents: documents, - })) -} diff --git a/sync-server/src/server/requests.rs b/sync-server/src/server/requests.rs index 119ad467..7b814559 100644 --- a/sync-server/src/server/requests.rs +++ b/sync-server/src/server/requests.rs @@ -1,51 +1,23 @@ -use axum::body::Bytes; -use axum_typed_multipart::{FieldData, TryFromMultipart}; +use crate::app_state::database::models::VaultUpdateId; use reconcile_text::NumberOrText; -use serde::{self, Deserialize}; +use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::app_state::database::models::{DocumentId, VaultUpdateId}; - -#[derive(TS, Debug, TryFromMultipart)] -#[ts(export)] -pub struct CreateDocumentVersion { - /// The client can decide the document id (if it wishes to) in order - /// to help with syncing. If the client does not provide a document id, - /// the server will generate one. If the client provides a document id - /// it must not already exist in the database. - pub document_id: Option, - pub relative_path: String, - - #[ts(as = "Vec")] - #[form_data(limit = "unlimited")] - pub content: FieldData, +#[derive(TS, Debug, Deserialize, Serialize)] +#[serde(tag = "type", content = "value")] +pub enum PushContent { + Snapshot(String), + Diff(#[ts(type = "Array")] Vec), + Delete, } -#[derive(Debug, TryFromMultipart)] -pub struct UpdateBinaryDocumentVersion { - pub parent_version_id: VaultUpdateId, - pub relative_path: String, - - #[form_data(limit = "unlimited")] - pub content: FieldData, -} - -#[derive(TS, Debug, Deserialize)] +#[derive(TS, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] #[ts(export)] -pub struct UpdateTextDocumentVersion { - #[ts(as = "i32")] - pub parent_version_id: VaultUpdateId, - - pub relative_path: String, - - #[ts(type = "Array")] - pub content: Vec, -} - -#[derive(TS, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub struct DeleteDocumentVersion { +pub struct PushDocument { + pub request_id: uuid::Uuid, + #[ts(as = "Option")] + pub parent_version_id: Option, pub relative_path: String, + pub content: PushContent, } diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index a8b3fcd7..ed469949 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -1,9 +1,7 @@ use serde::{self, Serialize}; use ts_rs::TS; -use crate::app_state::database::models::{ - DocumentVersion, DocumentVersionWithoutContent, VaultUpdateId, -}; +use crate::app_state::database::models::{DocumentVersion, DocumentVersionWithoutContent}; /// Response to a ping request. #[derive(TS, Debug, Clone, Serialize)] @@ -25,28 +23,14 @@ pub struct PingResponse { pub supported_api_version: u32, } -/// Response to a fetch latest documents request. -#[derive(TS, Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -#[ts(export)] -pub struct FetchLatestDocumentsResponse { - pub latest_documents: Vec, - - /// The update ID of the latest document in the response. - pub last_update_id: VaultUpdateId, -} - /// Response to an update document request. #[derive(TS, Debug, Clone, Serialize)] #[serde(tag = "type")] #[ts(export)] pub enum DocumentUpdateResponse { - /// Returned when the created/updated document's content is the same as was - /// sent in the create/update request and thus the response doesn't contain - /// the content because the client must already have it. - FastForwardUpdate(DocumentVersionWithoutContent), + /// The exact snapshot was committed, or this is its original acknowledgement. + Accepted(DocumentVersionWithoutContent), - /// Returned when the created/updated document's content is different from - /// what was sent in the create/update request. - MergingUpdate(DocumentVersion), + /// Nothing was written. The client must incorporate this version before retrying. + StaleBase(DocumentVersion), } diff --git a/sync-server/src/server/update_document.rs b/sync-server/src/server/update_document.rs deleted file mode 100644 index 00fbd008..00000000 --- a/sync-server/src/server/update_document.rs +++ /dev/null @@ -1,271 +0,0 @@ -use anyhow::{Context as _, anyhow}; -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use axum_typed_multipart::TypedMultipart; -use log::{debug, info}; -use reconcile_text::{BuiltinTokenizer, EditedText, reconcile}; -use serde::Deserialize; - -use super::{ - device_id_header::DeviceIdHeader, requests::UpdateTextDocumentVersion, - responses::DocumentUpdateResponse, -}; -use crate::{ - app_state::{ - AppState, - database::models::{DocumentId, StoredDocumentVersion, VaultId, VaultUpdateId}, - }, - config::user_config::User, - errors::{SyncServerError, client_error, not_found_error, server_error}, - server::requests::UpdateBinaryDocumentVersion, - utils::{ - find_first_available_path::find_first_available_path, is_binary::is_binary, - is_file_type_mergable::is_file_type_mergable, normalize::normalize, - sanitize_path::sanitize_path, - }, -}; - -#[derive(Deserialize)] -pub struct UpdateDocumentPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - document_id: DocumentId, -} - -#[axum::debug_handler] -pub async fn update_binary( - Path(UpdateDocumentPathParams { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - TypedMultipart(request): TypedMultipart, -) -> Result, SyncServerError> { - let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?; - let content = request.content.contents.to_vec(); - - update_document( - parent_document, - vault_id, - document_id, - user, - device_id, - state, - &request.relative_path, - content, - ) - .await -} - -#[axum::debug_handler] -#[allow(clippy::too_many_lines)] -pub async fn update_text( - Path(UpdateDocumentPathParams { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - Json(request): Json, -) -> Result, SyncServerError> { - let parent_document = get_parent_document(&state, &vault_id, request.parent_version_id).await?; - - let edited_text = EditedText::from_diff( - str::from_utf8(&parent_document.content) - .expect("parent must be valid UTF-8 because it's a text document"), - request.content, - &*BuiltinTokenizer::Word, - ) - .context("Failed to apply given diff to parent document") - .map_err(client_error)?; - - let content = edited_text.apply().text().into_bytes(); - - update_document( - parent_document, - vault_id, - document_id, - user, - device_id, - state, - &request.relative_path, - content, - ) - .await -} - -async fn get_parent_document( - state: &AppState, - vault_id: &VaultId, - parent_version_id: VaultUpdateId, -) -> Result { - state - .database - .get_document_version(vault_id, parent_version_id, None) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Parent version with id `{parent_version_id}` not found" - ))) - }, - Ok, - ) -} - -#[allow(clippy::too_many_lines, clippy::too_many_arguments)] -async fn update_document( - parent_document: StoredDocumentVersion, - vault_id: VaultId, - document_id: DocumentId, - user: User, - device_id: DeviceIdHeader, - state: AppState, - relative_path: &str, - content: Vec, -) -> Result, SyncServerError> { - debug!("Updating document `{document_id}` in vault `{vault_id}`"); - - let sanitized_relative_path = sanitize_path(relative_path); - - let mut transaction = state - .database - .create_write_transaction(&vault_id) - .await - .map_err(server_error)?; - - let last_update_id = state - .database - .get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - let latest_version = state - .database - .get_latest_document(&vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)? - .map_or_else( - || { - Err(not_found_error(anyhow!( - "Document with id `{document_id}` not found", - ))) - }, - Ok, - )?; - - if latest_version.is_deleted { - transaction - .rollback() - .await - .context("Failed to roll back transaction") - .map_err(server_error)?; - - info!("Document `{document_id}` has been deleted, ignoring update to it",); - return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( - latest_version.into(), - ))); - } - - // Return the latest version if the content and path are the same as the latest - // version - if content == latest_version.content && sanitized_relative_path == latest_version.relative_path - { - info!( - "Document content is the same as the latest version for `{document_id}`, skipping update" - ); - transaction - .rollback() - .await - .context("Failed to roll back transaction") - .map_err(server_error)?; - - return Ok(Json(DocumentUpdateResponse::FastForwardUpdate( - latest_version.into(), - ))); - } - - let are_all_participants_mergable = is_file_type_mergable( - &sanitized_relative_path, - &state.config.server.mergeable_file_extensions, - ) && !is_binary(&parent_document.content) - && !is_binary(&latest_version.content) - && !is_binary(&content); - - let merged_content = if are_all_participants_mergable { - info!("Merging changes for document `{document_id}` in vault `{vault_id}`"); - reconcile( - str::from_utf8(&parent_document.content) - .expect("parent must be valid UTF-8 because it's not binary"), - &str::from_utf8(&latest_version.content) - .expect("latest_version must be valid UTF-8 because it's not binary") - .into(), - &str::from_utf8(&content) - .expect("content must be valid UTF-8 because it's not binary") - .into(), - &*BuiltinTokenizer::Word, - ) - .apply() - .text() - .into_bytes() - } else { - content.clone() - }; - - let is_different_from_request_content = merged_content != content; - - // We can only update the relative path if we're the first one to do so - let new_relative_path = if parent_document.relative_path == latest_version.relative_path - && latest_version.relative_path != sanitized_relative_path - { - let new_path = find_first_available_path( - &vault_id, - &sanitized_relative_path, - &state.database, - &mut transaction, - ) - .await - .map_err(server_error)?; - - if new_path != sanitized_relative_path { - info!( - "Document already exists at new location: `{sanitized_relative_path}` when trying to update it in vault `{vault_id}`, deconflicting by creating at `{new_path}`" - ); - } - - new_path - } else { - latest_version.relative_path.clone() - }; - - let new_version = StoredDocumentVersion { - document_id, - vault_update_id: last_update_id + 1, - relative_path: new_relative_path, - content: merged_content, - updated_date: chrono::Utc::now(), - is_deleted: false, - user_id: user.name, - device_id: device_id.0, - has_been_merged: are_all_participants_mergable && is_different_from_request_content, - }; - - state - .database - .insert_document_version(&vault_id, &new_version, Some(transaction)) - .await - .map_err(server_error)?; - - Ok(Json(if is_different_from_request_content { - DocumentUpdateResponse::MergingUpdate(new_version.into()) - } else { - DocumentUpdateResponse::FastForwardUpdate(new_version.into()) - })) -} diff --git a/sync-server/src/utils.rs b/sync-server/src/utils.rs index b501ecb2..35d6a340 100644 --- a/sync-server/src/utils.rs +++ b/sync-server/src/utils.rs @@ -1,7 +1,5 @@ pub mod dedup_paths; pub mod find_first_available_path; -pub mod is_binary; -pub mod is_file_type_mergable; pub mod log_level; pub mod normalize; pub mod rotating_file_writer; diff --git a/sync-server/src/utils/is_binary.rs b/sync-server/src/utils/is_binary.rs deleted file mode 100644 index 09bfcf94..00000000 --- a/sync-server/src/utils/is_binary.rs +++ /dev/null @@ -1,26 +0,0 @@ -/// Heuristically determine if the given data is a binary or a text file's -/// content. -/// -/// Only text inputs can be reconciled using the crate's functions. -#[must_use] -pub fn is_binary(data: &[u8]) -> bool { - if data.contains(&0) { - // Even though the NUL character is valid in UTF-8, it's highly suspicious in - // human-readable text. - return true; - } - - std::str::from_utf8(data).is_err() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_binary() { - assert!(is_binary(&[0, 159, 146, 150])); - assert!(is_binary(&[0, 12])); - assert!(!is_binary(b"hello")); - } -} diff --git a/sync-server/src/utils/is_file_type_mergable.rs b/sync-server/src/utils/is_file_type_mergable.rs deleted file mode 100644 index 7aabb393..00000000 --- a/sync-server/src/utils/is_file_type_mergable.rs +++ /dev/null @@ -1,34 +0,0 @@ -pub fn is_file_type_mergable(path_or_file_name: &str, mergeable_extensions: &[String]) -> bool { - let file_extension = path_or_file_name.split('.').next_back().unwrap_or_default(); - let file_extension_lower = file_extension.to_lowercase(); - - mergeable_extensions - .iter() - .any(|ext| ext.to_lowercase() == file_extension_lower) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_file_type_mergable() { - let mergeable = vec!["md".to_owned(), "txt".to_owned()]; - - assert!(is_file_type_mergable(".md", &mergeable)); - assert!(is_file_type_mergable("hi.md", &mergeable)); - assert!(is_file_type_mergable( - "my/path/to/my/document.md", - &mergeable - )); - assert!(is_file_type_mergable("hi.MD", &mergeable)); - assert!(is_file_type_mergable( - "my/path/to/my/DOCUMENT.MD", - &mergeable - )); - - assert!(!is_file_type_mergable(".json", &mergeable)); - assert!(!is_file_type_mergable("HELLO.JSON", &mergeable)); - assert!(!is_file_type_mergable("my/config.yml", &mergeable)); - } -} -- 2.54.0 From 98da8062531d3baae531a51e827e4a713d089724 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:16:47 +0100 Subject: [PATCH 02/48] Move errors --- frontend/sync-client/src/errors/errors.ts | 34 +++++++++++++++++++ .../file-operations/file-not-found-error.ts | 9 ----- .../src/services/authentication-error.ts | 6 ---- .../services/server-version-mismatch-error.ts | 6 ---- .../src/services/sync-reset-error.ts | 6 ---- 5 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 frontend/sync-client/src/errors/errors.ts delete mode 100644 frontend/sync-client/src/file-operations/file-not-found-error.ts delete mode 100644 frontend/sync-client/src/services/authentication-error.ts delete mode 100644 frontend/sync-client/src/services/server-version-mismatch-error.ts delete mode 100644 frontend/sync-client/src/services/sync-reset-error.ts diff --git a/frontend/sync-client/src/errors/errors.ts b/frontend/sync-client/src/errors/errors.ts new file mode 100644 index 00000000..7dc48a3a --- /dev/null +++ b/frontend/sync-client/src/errors/errors.ts @@ -0,0 +1,34 @@ +export class SyncResetError extends Error { + public constructor() { + super("SyncClient has been reset, cleaning up"); + this.name = "SyncResetError"; + } +} + +export class ServerVersionMismatchError extends Error { + public constructor(message: string) { + super(message); + this.name = "ServerVersionMismatchError"; + } +} + +export class AuthenticationError extends Error { + public constructor(message: string) { + super(message); + this.name = "AuthenticationError"; + } +} + +export class FileNotFoundError extends Error { + public constructor( + message: string, + public readonly filePath: string + ) { + super(message); + this.name = "FileNotFoundError"; + } +} + +export class PermanentSyncError extends Error { } + +export class LocalChangesDuringReconciliation extends Error { } diff --git a/frontend/sync-client/src/file-operations/file-not-found-error.ts b/frontend/sync-client/src/file-operations/file-not-found-error.ts deleted file mode 100644 index b8acd265..00000000 --- a/frontend/sync-client/src/file-operations/file-not-found-error.ts +++ /dev/null @@ -1,9 +0,0 @@ -export class FileNotFoundError extends Error { - public constructor( - message: string, - public readonly filePath: string - ) { - super(message); - this.name = "FileNotFoundError"; - } -} diff --git a/frontend/sync-client/src/services/authentication-error.ts b/frontend/sync-client/src/services/authentication-error.ts deleted file mode 100644 index 6be4af24..00000000 --- a/frontend/sync-client/src/services/authentication-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class AuthenticationError extends Error { - public constructor(message: string) { - super(message); - this.name = "AuthenticationError"; - } -} diff --git a/frontend/sync-client/src/services/server-version-mismatch-error.ts b/frontend/sync-client/src/services/server-version-mismatch-error.ts deleted file mode 100644 index 0b9960ea..00000000 --- a/frontend/sync-client/src/services/server-version-mismatch-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class ServerVersionMismatchError extends Error { - public constructor(message: string) { - super(message); - this.name = "ServerVersionMismatchError"; - } -} diff --git a/frontend/sync-client/src/services/sync-reset-error.ts b/frontend/sync-client/src/services/sync-reset-error.ts deleted file mode 100644 index 7b74e0b9..00000000 --- a/frontend/sync-client/src/services/sync-reset-error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export class SyncResetError extends Error { - public constructor() { - super("SyncClient has been reset, cleaning up"); - this.name = "SyncResetError"; - } -} -- 2.54.0 From 19502cb967caa68bdd45a9c29407986a63235cb5 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:19:48 +0100 Subject: [PATCH 03/48] Update TS bindings --- .cargo/config.toml | 2 ++ .gitignore | 1 - .../services/types/CreateDocumentVersion.ts | 13 ---------- .../types/CursorPositionFromServer.ts | 4 +--- .../src/services/types/CursorSpan.ts | 5 +--- .../services/types/DocumentUpdateResponse.ts | 4 ++-- .../src/services/types/DocumentVersion.ts | 5 ++-- .../types/DocumentVersionWithoutContent.ts | 2 -- .../src/services/types/EventBatch.ts | 4 ++++ .../src/services/types/EventRecord.ts | 8 +++++++ .../types/FetchLatestDocumentsResponse.ts | 13 ---------- .../{DeleteDocumentVersion.ts => Manifest.ts} | 4 +--- .../services/types/ManifestUpdateResponse.ts | 6 +++++ .../src/services/types/PingResponse.ts | 20 ++++++++-------- .../src/services/types/PushContent.ts | 5 ++++ .../src/services/types/PushDocument.ts | 8 +++++++ .../src/services/types/PushManifest.ts | 7 ++++++ .../services/types/UpdateDocumentVersion.ts | 7 ------ .../types/UpdateTextDocumentVersion.ts | 7 ------ .../src/services/types/VaultEvent.ts | 7 ++++++ ...bSocketVaultUpdate.ts => VaultSnapshot.ts} | 6 +++-- .../services/types/WebSocketServerMessage.ts | 4 ++-- scripts/update-api-types.sh | 24 +++++++++---------- 23 files changed, 81 insertions(+), 85 deletions(-) create mode 100644 .cargo/config.toml delete mode 100644 frontend/sync-client/src/services/types/CreateDocumentVersion.ts create mode 100644 frontend/sync-client/src/services/types/EventBatch.ts create mode 100644 frontend/sync-client/src/services/types/EventRecord.ts delete mode 100644 frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts rename frontend/sync-client/src/services/types/{DeleteDocumentVersion.ts => Manifest.ts} (57%) create mode 100644 frontend/sync-client/src/services/types/ManifestUpdateResponse.ts create mode 100644 frontend/sync-client/src/services/types/PushContent.ts create mode 100644 frontend/sync-client/src/services/types/PushDocument.ts create mode 100644 frontend/sync-client/src/services/types/PushManifest.ts delete mode 100644 frontend/sync-client/src/services/types/UpdateDocumentVersion.ts delete mode 100644 frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts create mode 100644 frontend/sync-client/src/services/types/VaultEvent.ts rename frontend/sync-client/src/services/types/{WebSocketVaultUpdate.ts => VaultSnapshot.ts} (66%) diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..5b0ee4b0 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[env] +TS_RS_EXPORT_DIR = { value = "frontend/sync-client/src/services/types", relative = true } diff --git a/.gitignore b/.gitignore index 967b2b65..ab97af82 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ frontend/*/dist # Rust build folders sync-server/target sync-server/artifacts -sync-server/bindings/*.ts # build folders sync-server/db.sqlite3* diff --git a/frontend/sync-client/src/services/types/CreateDocumentVersion.ts b/frontend/sync-client/src/services/types/CreateDocumentVersion.ts deleted file mode 100644 index ed921f18..00000000 --- a/frontend/sync-client/src/services/types/CreateDocumentVersion.ts +++ /dev/null @@ -1,13 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export interface CreateDocumentVersion { - /** - * The client can decide the document id (if it wishes to) in order - * to help with syncing. If the client does not provide a document id, - * the server will generate one. If the client provides a document id - * it must not already exist in the database. - */ - document_id: string | null; - relative_path: string; - content: number[]; -} diff --git a/frontend/sync-client/src/services/types/CursorPositionFromServer.ts b/frontend/sync-client/src/services/types/CursorPositionFromServer.ts index 52a24f27..d9be7d82 100644 --- a/frontend/sync-client/src/services/types/CursorPositionFromServer.ts +++ b/frontend/sync-client/src/services/types/CursorPositionFromServer.ts @@ -1,6 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ClientCursors } from "./ClientCursors"; -export interface CursorPositionFromServer { - clients: ClientCursors[]; -} +export interface CursorPositionFromServer { clients: ClientCursors[] } diff --git a/frontend/sync-client/src/services/types/CursorSpan.ts b/frontend/sync-client/src/services/types/CursorSpan.ts index 2cc2b7fc..d2fc9614 100644 --- a/frontend/sync-client/src/services/types/CursorSpan.ts +++ b/frontend/sync-client/src/services/types/CursorSpan.ts @@ -1,6 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface CursorSpan { - start: number; - end: number; -} +export interface CursorSpan { start: number; end: number } diff --git a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts index 7fd06c7a..29f23cfb 100644 --- a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts @@ -6,5 +6,5 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont * Response to an update document request. */ export type DocumentUpdateResponse = - | ({ type: "FastForwardUpdate" } & DocumentVersionWithoutContent) - | ({ type: "MergingUpdate" } & DocumentVersion); + | ({ type: "Accepted" } & DocumentVersionWithoutContent) + | ({ type: "StaleBase" } & DocumentVersion); diff --git a/frontend/sync-client/src/services/types/DocumentVersion.ts b/frontend/sync-client/src/services/types/DocumentVersion.ts index 3b9aa37b..a636e4fb 100644 --- a/frontend/sync-client/src/services/types/DocumentVersion.ts +++ b/frontend/sync-client/src/services/types/DocumentVersion.ts @@ -1,12 +1,11 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. export interface DocumentVersion { + contentBase64: string; vaultUpdateId: number; documentId: string; - relativePath: string; updatedDate: string; - contentBase64: string; - isDeleted: boolean; userId: string; deviceId: string; + contentSize: number; } diff --git a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts index 4b24e7c5..d4e8165a 100644 --- a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts +++ b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts @@ -3,9 +3,7 @@ export interface DocumentVersionWithoutContent { vaultUpdateId: number; documentId: string; - relativePath: string; updatedDate: string; - isDeleted: boolean; userId: string; deviceId: string; contentSize: number; diff --git a/frontend/sync-client/src/services/types/EventBatch.ts b/frontend/sync-client/src/services/types/EventBatch.ts new file mode 100644 index 00000000..11412274 --- /dev/null +++ b/frontend/sync-client/src/services/types/EventBatch.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { EventRecord } from "./EventRecord"; + +export interface EventBatch { headEventId: number; events: EventRecord[] } diff --git a/frontend/sync-client/src/services/types/EventRecord.ts b/frontend/sync-client/src/services/types/EventRecord.ts new file mode 100644 index 00000000..2e50d69a --- /dev/null +++ b/frontend/sync-client/src/services/types/EventRecord.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; +import type { Manifest } from "./Manifest"; + +export type EventRecord = { eventId: number; requestId: string } & ( + | { type: "content"; document: DocumentVersionWithoutContent } + | { type: "manifest"; manifest: Manifest } +); diff --git a/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts b/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts deleted file mode 100644 index 160c9279..00000000 --- a/frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts +++ /dev/null @@ -1,13 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; - -/** - * Response to a fetch latest documents request. - */ -export interface FetchLatestDocumentsResponse { - latestDocuments: DocumentVersionWithoutContent[]; - /** - * The update ID of the latest document in the response. - */ - lastUpdateId: bigint; -} diff --git a/frontend/sync-client/src/services/types/DeleteDocumentVersion.ts b/frontend/sync-client/src/services/types/Manifest.ts similarity index 57% rename from frontend/sync-client/src/services/types/DeleteDocumentVersion.ts rename to frontend/sync-client/src/services/types/Manifest.ts index 99ecc9e7..f32ce231 100644 --- a/frontend/sync-client/src/services/types/DeleteDocumentVersion.ts +++ b/frontend/sync-client/src/services/types/Manifest.ts @@ -1,5 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DeleteDocumentVersion { - relativePath: string; -} +export interface Manifest { manifestId: number; entries: Record } diff --git a/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts b/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts new file mode 100644 index 00000000..db3ea4a1 --- /dev/null +++ b/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts @@ -0,0 +1,6 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { Manifest } from "./Manifest"; + +export type ManifestUpdateResponse = + | ({ type: "Accepted" } & Manifest) + | ({ type: "StaleBase" } & Manifest); diff --git a/frontend/sync-client/src/services/types/PingResponse.ts b/frontend/sync-client/src/services/types/PingResponse.ts index 6db66354..f96520e9 100644 --- a/frontend/sync-client/src/services/types/PingResponse.ts +++ b/frontend/sync-client/src/services/types/PingResponse.ts @@ -5,21 +5,21 @@ */ export interface PingResponse { /** - * Semantic version of the server. - */ + * Semantic version of the server. + */ serverVersion: string; /** - * Whether the client is authenticated based on the sent Authorization - * header. - */ + * Whether the client is authenticated based on the sent Authorization + * header. + */ isAuthenticated: boolean; /** - * List of file extensions that are allowed to be merged. - */ + * List of file extensions that are allowed to be merged. + */ mergeableFileExtensions: string[]; /** - * API version ensuring backwards & forwards compatibility between the client - * and server. - */ + * API version ensuring backwards & forwards compatibility between the client + * and server. + */ supportedApiVersion: number; } diff --git a/frontend/sync-client/src/services/types/PushContent.ts b/frontend/sync-client/src/services/types/PushContent.ts new file mode 100644 index 00000000..3035b567 --- /dev/null +++ b/frontend/sync-client/src/services/types/PushContent.ts @@ -0,0 +1,5 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PushContent = + | { type: "Snapshot"; value: string } + | { type: "Diff"; value: (number | string)[] }; diff --git a/frontend/sync-client/src/services/types/PushDocument.ts b/frontend/sync-client/src/services/types/PushDocument.ts new file mode 100644 index 00000000..8410b96e --- /dev/null +++ b/frontend/sync-client/src/services/types/PushDocument.ts @@ -0,0 +1,8 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { PushContent } from "./PushContent"; + +export interface PushDocument { + requestId: string; + parentVersionId: number | null; + content: PushContent; +} diff --git a/frontend/sync-client/src/services/types/PushManifest.ts b/frontend/sync-client/src/services/types/PushManifest.ts new file mode 100644 index 00000000..ee5e68e4 --- /dev/null +++ b/frontend/sync-client/src/services/types/PushManifest.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export interface PushManifest { + requestId: string; + parentManifestId: number; + entries: Record; +} diff --git a/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts b/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts deleted file mode 100644 index 4e57a297..00000000 --- a/frontend/sync-client/src/services/types/UpdateDocumentVersion.ts +++ /dev/null @@ -1,7 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export interface UpdateDocumentVersion { - parent_version_id: bigint; - relative_path: string; - content: number[]; -} diff --git a/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts b/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts deleted file mode 100644 index 46f36bd0..00000000 --- a/frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts +++ /dev/null @@ -1,7 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export interface UpdateTextDocumentVersion { - parentVersionId: number; - relativePath: string; - content: (number | string)[]; -} diff --git a/frontend/sync-client/src/services/types/VaultEvent.ts b/frontend/sync-client/src/services/types/VaultEvent.ts new file mode 100644 index 00000000..426efd5b --- /dev/null +++ b/frontend/sync-client/src/services/types/VaultEvent.ts @@ -0,0 +1,7 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; +import type { Manifest } from "./Manifest"; + +export type VaultEvent = + | { type: "content"; document: DocumentVersionWithoutContent } + | { type: "manifest"; manifest: Manifest }; diff --git a/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts b/frontend/sync-client/src/services/types/VaultSnapshot.ts similarity index 66% rename from frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts rename to frontend/sync-client/src/services/types/VaultSnapshot.ts index f1ea0f80..3c0ff9a7 100644 --- a/frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts +++ b/frontend/sync-client/src/services/types/VaultSnapshot.ts @@ -1,7 +1,9 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; +import type { Manifest } from "./Manifest"; -export interface WebSocketVaultUpdate { +export interface VaultSnapshot { + headEventId: number; + manifest: Manifest; documents: DocumentVersionWithoutContent[]; - isInitialSync: boolean; } diff --git a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts index fd250b7b..d1d10a8f 100644 --- a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts +++ b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts @@ -1,7 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { CursorPositionFromServer } from "./CursorPositionFromServer"; -import type { WebSocketVaultUpdate } from "./WebSocketVaultUpdate"; +import type { EventBatch } from "./EventBatch"; export type WebSocketServerMessage = - | ({ type: "vaultUpdate" } & WebSocketVaultUpdate) + | ({ type: "vaultEvents" } & EventBatch) | ({ type: "cursorPositions" } & CursorPositionFromServer); diff --git a/scripts/update-api-types.sh b/scripts/update-api-types.sh index 3f4a9e2a..055dd643 100755 --- a/scripts/update-api-types.sh +++ b/scripts/update-api-types.sh @@ -2,21 +2,19 @@ set -e -rm -rf sync-server/bindings - -cd sync-server -cargo test export_bindings -cd - - -# Both target directories contain only generated bindings — wipe and copy +# This directory contains only generated bindings. ts-rs writes here directly, +# as configured in .cargo/config.toml. rm -f frontend/sync-client/src/services/types/*.ts -rm -f frontend/history-ui/src/lib/types/*.ts -cp -r sync-server/bindings/* frontend/sync-client/src/services/types/ -cp -r sync-server/bindings/* frontend/history-ui/src/lib/types/ -cd frontend -npm run lint -cd .. +( + cd sync-server + cargo test export_bindings +) + +( + cd frontend + npm run lint +) # Format all files across the project (frontend and backend) npx -C frontend prettier --write "**/*.{ts,js,json,md,yml,yaml}" -- 2.54.0 From 5624e51f524fcfb4f8fd128c138679e8b59af7c8 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:23:24 +0100 Subject: [PATCH 04/48] Small improvements --- CLAUDE.md | 155 ------------------ frontend/deterministic-tests/src/cli.ts | 2 +- .../src/deterministic-agent.ts | 2 +- .../deterministic-tests/src/server-control.ts | 3 +- .../deterministic-tests/src/test-runner.ts | 2 +- frontend/local-client-cli/src/cli.ts | 8 +- frontend/local-client-cli/src/healthcheck.ts | 2 +- .../src/utils/data-structures/locks.test.ts | 2 +- .../src/utils/data-structures/locks.ts | 2 +- frontend/sync-client/webpack.config.js | 7 +- scripts/build-docs.sh | 14 +- scripts/bump-version.sh | 13 +- scripts/check.sh | 55 +++---- scripts/e2e.sh | 28 ++-- 14 files changed, 76 insertions(+), 219 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 39161e39..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,155 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project shape - -VaultLink is a self-hosted Obsidian file-sync system. Two halves of one repo: - -- `sync-server/` — Rust (axum + sqlx/SQLite). Source of truth for vault state, broadcasts changes via WebSocket. -- `frontend/` — npm workspaces. The sync engine (`sync-client`) is consumed by an Obsidian plugin, a standalone CLI, a fuzz E2E harness, a scripted determinism harness, and a history UI. - -The HTTP/WS API types are generated from Rust (`ts-rs`) and mirrored into the TS workspaces. **Never hand-edit files in `frontend/sync-client/src/services/types/` or `frontend/history-ui/src/lib/types/`** — run `scripts/update-api-types.sh` after changing anything Serde-derived in the server. - -### Frontend workspaces - -- `sync-client` — the sync engine; published to consumers via `dist/`. All other TS workspaces depend on it via `file:../sync-client`. -- `obsidian-plugin` — Obsidian plugin built from `sync-client`. -- `local-client-cli` — same engine wrapped as a standalone CLI. -- `history-ui` — vault-history web UI. -- `test-client` — fuzz E2E harness (random ops across N processes). -- `deterministic-tests` — scripted multi-client tests with an in-memory FS, run against a real server. - -## Common commands - -Pre-push hygiene (formats, lints, runs tests, requires clean git state): - -```sh -scripts/check.sh --fix -``` - -Run the fuzz E2E (N parallel processes): - -```sh -scripts/e2e.sh 12 -# Logs land in logs/log_.log. Clean with scripts/clean-up.sh -``` - -Run deterministic tests (require a release-built server in `sync-server/target/release/sync_server` — they spawn it themselves): - -```sh -cd sync-server && cargo build --release && cd .. -cd frontend -npm run build -w sync-client -w deterministic-tests -node deterministic-tests/dist/cli.js # all -node deterministic-tests/dist/cli.js --filter=rename # subset -node deterministic-tests/dist/cli.js --filter=… -j 4 # cap parallelism -``` - -Run a single sync-client unit test by file: - -```sh -cd frontend/sync-client && npx tsx --test 'src/**/sync-event-queue.test.ts' -``` - -Server: dev runs from `sync-server/` against `config-e2e.yml`: - -```sh -cd sync-server -cargo run config-e2e.yml # dev -cargo build --release # used by both e2e harnesses -cargo test # unit + ts-rs binding export tests -``` - -Frontend dev (sync-client + obsidian-plugin watch in parallel): - -```sh -cd frontend && npm install && npm run dev -``` - -Regenerate TS bindings from Rust types (touches `frontend/{sync-client,history-ui}/src/.../types/`): - -```sh -scripts/update-api-types.sh -``` - -## SQLite / sqlx - -The server uses `sqlx::query!` macros that need a prepared `.sqlx` cache to compile offline. Touching any SQL means regenerating it: - -```sh -cd sync-server -sqlx database create --database-url sqlite://db.sqlite3 -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 -cargo sqlx prepare --workspace -``` - -New migrations: `sqlx migrate add --source src/app_state/database/migrations `. - -## Sync engine architecture - -Read `frontend/sync-client/src/sync-operations/` to follow the sync engine; the rest of `sync-client` is plumbing (filesystem ops, persistence, services, telemetry). - -The engine is **two independent loops with separate invariants**: - -- **Wire loop** (`syncer.ts`) — drains the single-consumer FIFO queue. HTTP and WS handlers update record fields (`remoteRelativePath`, `parentVersionId`, `remoteHash`) and write content to the file at `record.localPath`. They never move files for path placement. -- **Path reconciler** (`reconciler.ts`) — runs after every drained event. Best-effort pass that moves files to make `localPath === remoteRelativePath`. The move graph is topologically sorted; cycles are resolved by reading every file in the cycle into memory and writing each back to its new slot (no tmp files). Records with pending local events are skipped on each pass — the reconciler operates only on settled records. Failures (slot occupied by an untracked file, etc.) are silent skips; the next pass retries. - -**`SyncEventQueue`** (`sync-event-queue.ts`) holds: - -- `byDocId: Map` — primary record store. -- `byLocalPath: Map` — derived index for path lookups, maintained at every mutation point. -- `events: SyncEvent[]` — pending wire ops in FIFO drain order. - -```ts -DocumentRecord = { - documentId, - parentVersionId, - remoteHash?, - remoteRelativePath, - localPath: RelativePath | undefined -} -``` - -`localPath === undefined` means the doc has no local file yet — typically a remote create whose target slot was occupied at receive time; the reconciler will fetch and place when the slot frees (the bytes wait in `pendingPlacementContent`). - -Local FS events from the watcher update `localPath` synchronously at enqueue time via `setLocalPath` / `upsertRecord`. The wire loop never updates it for path placement; only the reconciler does. A user rename onto a tracked slot enqueues a `LocalDelete` for the displaced doc (the OS rename clobbered its content) and clears that doc's `localPath`. - -**Pending creates** use a `Promise` chain to serialize dependent ops (`LocalUpdate`, `LocalDelete`) behind the still-in-flight `LocalCreate`. `resolveCreate` resolves the promise once the server returns a docId, and `replacePendingDocumentId` swaps the resolved id across already-queued events. `findLatestCreateForPath` is the lookup the watcher uses to attach dependents; `updatePendingCreatePath` rewrites a pending create's `event.path` in place when the user renames the file before its create has acked. - -**Watermark.** `lastSeenUpdateId` uses a `MinCovered` (a contiguous-prefix tracker over a stream of integers): we only advance the published min when the next consecutive id has been processed, so out-of-order RemoteChange ids don't fool the WebSocket handshake into requesting a too-recent catch-up. - -**Server catch-up.** The server's WS handshake replays events newer than the client's `last_seen_vault_update_id` from the `latest_document_versions` view (one row per doc, the latest). On those replayed rows `is_new_file` means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`), not "this row is the doc's first version" — necessary because the catch-up only carries the latest version; if a doc was created and updated past the watermark, the client never sees its create otherwise. - -## Edge-case patterns the sync engine has to survive - -The two-loop split defuses most of the old race catalogue (slot-collision stashes, conflict-uuid divergence, `MoveOnConflict.NEW`/`EXISTING` policy choices) by separating wire transport from path placement. What's left: - -**Pending-create docId is a `Promise`, not a string, until the create acks.** Any `LocalUpdate` / `LocalDelete` queued behind a still-in-flight `LocalCreate` carries the create's `resolvers.promise` as its `documentId`. `replacePendingDocumentId` swaps the resolved id across queued events when the create resolves; `===` comparisons against the resolved string elsewhere will silently fail until that swap runs. Anything that walks `events[]` looking for a docId match must either run after the swap or be tolerant of `Promise`-typed ids. - -**`processCreate` reads `event.path` live, not `event.originalPath`.** The watcher rewrites `event.path` in place via `updatePendingCreatePath` when the user renames a pending-create file. `originalPath` was removed from `LocalCreate` events specifically because reading it would send the stale pre-rename path to the server. - -**`record.localPath` mutates in place across awaits.** When the watcher renames a doc while a drain handler is awaiting an HTTP roundtrip, the queue mutates the in-flight event's record so subsequent reads see the new path. Snapshotting `record.localPath` into a local at function entry and using it after an `await` reads/writes a now-vacated slot. Read `record.localPath` live; only snapshot for the deliberate "did it change while I was awaiting" comparison. - -**Reconciler-defer is the wire-loop's contract with the reconciler.** The reconciler skips records where `hasPendingLocalEventsForDocumentId` returns true. Wire-loop handlers can therefore freely write `remoteRelativePath` to whatever the server returned — even if it disagrees with `localPath` — knowing the reconciler won't move the file out from under a queued user rename. - -**Watermark advancement is load-bearing both ways.** Branches that _skip_ a remote event without advancing `lastSeenUpdateId` create permanent gaps that re-deliver forever. Branches that _advance_ without applying the content lose data: the server has no further event to re-deliver, the catch-up only carries the latest version, and any state in between is gone. Don't advance unless the event was actually applied (or deliberately discarded after weighing both halves). - -**`isNewFile` semantics differ between catch-up and real-time.** On WS handshake replay it means _new to this client_ (`creation_vault_update_id > last_seen_vault_update_id`); on real-time broadcasts it means _this version is the create_ (`creation_vault_update_id == vault_update_id`). A handler that decides based on one interpretation will be wrong on the other channel; reasoning about fetch-and-treat-as-new vs. ignore needs to know which channel delivered the event. - -**Pause / disable-sync mid-flight** is the one race the new model doesn't structurally fix. An HTTP that committed server-side but whose response was discarded leaves the server holding a doc the client has no record of. Resume → offline scan → server-side dedupe handles it (the server merges the duplicate create into the existing doc), but if the merge produces a deconflict, the client picks up an extra file. Out of scope for the two-loop split. - -**Cycle reconciliation uses in-memory content swap.** When the move graph contains a cycle, the reconciler reads every file in the cycle into memory and writes each back to its new slot, with no tmp files. A write-ahead marker at `.vaultlink/swap-.json` lists each leg; on startup the reconciler reads the marker, hashes each `from` to determine which legs ran, and replays the rest. The `.vaultlink/**` glob is hard-coded as an internal ignore pattern so swap markers don't get sync'd. - -## Two complementary E2E harnesses - -- **`test-client` (fuzz):** random ops across N parallel processes for many minutes. Used by `scripts/e2e.sh`. Catches bugs nobody thought to write a test for, but reproductions are noisy. -- **`deterministic-tests`:** scripted scenarios with an in-memory FS pinned to a real server. Used to _capture_ a fuzz-discovered bug as a minimal repro before fixing it. See `frontend/deterministic-tests/README.md` for the step grammar (`pause-server`, `pause-websocket`, `barrier`, `assert-consistent`, etc.). - -When a fuzz failure surfaces, the workflow is: root-cause from logs → write a deterministic test that fails on the bug → fix → confirm both the deterministic test and `e2e.sh` pass. - -## Style - -- TS: 4-space indent, no tabs, LF, prettier (`trailingComma: "none"`). YAML/MD use 2-space indent. -- Rust: `rustfmt.toml` enforces 4-space spaces, LF. -- Lint: ESLint for TS, Clippy for Rust, `cargo machete` for unused deps. All wired into `scripts/check.sh`. diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts index 6e15cac0..1f5f4706 100644 --- a/frontend/deterministic-tests/src/cli.ts +++ b/frontend/deterministic-tests/src/cli.ts @@ -42,7 +42,7 @@ function testUsesPauseServer(test: TestDefinition): boolean { */ function findProjectRoot(): string { let dir = path.dirname(__filename); - const root = path.parse(dir).root; + const {root} = path.parse(dir); while (dir !== root) { if ( fs.existsSync(path.join(dir, "sync-server")) && diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts index b32b01c2..fb7fe582 100644 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -304,7 +304,7 @@ export class DeterministicAgent extends debugging.InMemoryFileSystem { }); } - const nextWriteRename = this.nextWriteRename; + const { nextWriteRename } = this; if ( nextWriteRename !== undefined && nextWriteRename.oldPath === path diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts index 9cb4cde0..d3c5387d 100644 --- a/frontend/deterministic-tests/src/server-control.ts +++ b/frontend/deterministic-tests/src/server-control.ts @@ -239,8 +239,7 @@ export class ServerControl { public isRunning(): boolean { const proc = this.process; return ( - proc !== null && - proc.pid !== undefined && + proc?.pid !== undefined && proc.exitCode === null && proc.signalCode === null ); diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts index 411e9b08..cf99f7f2 100644 --- a/frontend/deterministic-tests/src/test-runner.ts +++ b/frontend/deterministic-tests/src/test-runner.ts @@ -190,7 +190,7 @@ export class TestRunner { (entry) => entry.details.type === step.syncType && entry.details.relativePath === step.path, - () => this.serverControl.pause() + () => { this.serverControl.pause(); } ); this.serverControl.resume(); await historySeen; diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index 31c81d5c..e6aabc68 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -1,11 +1,11 @@ import * as path from "path"; import * as fs from "fs/promises"; import * as fsSync from "fs"; -import type { NetworkConnectionStatus } from "sync-client"; +import type { NetworkConnectionStatus , + Logger} from "sync-client"; import { SyncClient, DEFAULT_SETTINGS, - Logger, LogLevel, LogLine, type SyncSettings, @@ -41,7 +41,7 @@ const LOG_LEVEL_ORDER = { function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { return (logLine: LogLine): void => { if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) { - // eslint-disable-next-line no-console + console.log(formatLogLine(logLine)); } }; @@ -279,7 +279,7 @@ async function main(): Promise { } main().catch((error: unknown) => { - // eslint-disable-next-line no-console + console.error( `Unexpected error: ${error instanceof Error ? error.message : String(error)}` ); diff --git a/frontend/local-client-cli/src/healthcheck.ts b/frontend/local-client-cli/src/healthcheck.ts index d7211c88..b2c296a0 100644 --- a/frontend/local-client-cli/src/healthcheck.ts +++ b/frontend/local-client-cli/src/healthcheck.ts @@ -1,5 +1,5 @@ #!/usr/bin/env node -/* eslint-disable no-console */ + /** * Healthcheck script for Docker container diff --git a/frontend/sync-client/src/utils/data-structures/locks.test.ts b/frontend/sync-client/src/utils/data-structures/locks.test.ts index 9beb867a..f7f0acf2 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.test.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.test.ts @@ -5,7 +5,7 @@ import type { RelativePath } from "../../persistence/database"; import { Locks } from "./locks"; import { awaitAll } from "../await-all"; import { sleep } from "../sleep"; -import { SyncResetError } from "../../services/sync-reset-error"; +import { SyncResetError } from "../../errors/errors"; describe("withLock", () => { const testPath: RelativePath = "test/document/path"; diff --git a/frontend/sync-client/src/utils/data-structures/locks.ts b/frontend/sync-client/src/utils/data-structures/locks.ts index e55c76b0..096cacdf 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.ts @@ -1,4 +1,4 @@ -import { SyncResetError } from "../../services/sync-reset-error"; +import { SyncResetError } from "../../errors/errors"; import type { Logger } from "../../tracing/logger"; import { awaitAll } from "../await-all"; diff --git a/frontend/sync-client/webpack.config.js b/frontend/sync-client/webpack.config.js index b7c3a3fd..899790b8 100644 --- a/frontend/sync-client/webpack.config.js +++ b/frontend/sync-client/webpack.config.js @@ -9,7 +9,12 @@ const common = { rules: [ { test: /\.ts$/, - use: ["ts-loader"] + use: [ + { + loader: "ts-loader", + options: { onlyCompileBundledFiles: true } + } + ] }, { test: /\.wasm$/, diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh index c87144a9..756d46ef 100755 --- a/scripts/build-docs.sh +++ b/scripts/build-docs.sh @@ -4,11 +4,11 @@ set -e ./scripts/utils/check-node.sh -cd docs +( + cd docs -npm ci -npm run format:check -npm run spell:check -npm run build - -cd - + npm ci + npm run format:check + npm run spell:check + npm run build +) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index bea3d982..67f9580d 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -25,13 +25,16 @@ else fi echo "Bumping sync-server versions" -cd sync-server -cargo set-version --bump $1 +( + cd sync-server + cargo set-version --bump $1 +) echo "Bumping frontend versions" -cd ../frontend -npm version $1 --workspaces -cd .. +( + cd frontend + npm version $1 --workspaces +) cp frontend/obsidian-plugin/manifest.json manifest.json # for BRAT, otherwise it wouldn't update diff --git a/scripts/check.sh b/scripts/check.sh index 2ee0dd62..9f5ee98c 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -12,43 +12,42 @@ fi echo "Running checks in sync-server" -cd sync-server -which sqlx || cargo install sqlx-cli -sqlx database create --database-url sqlite://db.sqlite3 -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 +( + cd sync-server + which sqlx || cargo install sqlx-cli + sqlx database create --database-url sqlite://db.sqlite3 + sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 -cargo test --verbose + cargo test --verbose -if [[ "$FIX_MODE" == true ]]; then - cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged - cargo fmt --all -else - cargo clippy --all-targets --all-features - cargo fmt --all -- --check -fi + if [[ "$FIX_MODE" == true ]]; then + cargo clippy --all-targets --all-features --fix --allow-dirty --allow-staged + cargo fmt --all + else + cargo clippy --all-targets --all-features + cargo fmt --all -- --check + fi -which cargo-machete || cargo install cargo-machete -cargo machete --with-metadata + which cargo-machete || cargo install cargo-machete + cargo machete --with-metadata +) -cd .. scripts/update-api-types.sh # this will dirty up the git state if not up-to-date echo "Running checks in frontend" -cd frontend +( + cd frontend -if [[ "$FIX_MODE" == true ]]; then - npm install -else - npm ci -fi + if [[ "$FIX_MODE" == true ]]; then + npm install + else + npm ci + fi -cd .. - -cd frontend -npm run build -npm run test -npm run lint -cd .. + npm run build + npm run test + npm run lint +) # Format all files across the project (frontend and backend) # Prettier respects .gitignore by default diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 7ab8d90c..03d6a35f 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -21,8 +21,10 @@ mkdir -p logs # Build and restart the server echo "Building server..." -cd sync-server -cargo build --release +( + cd sync-server + cargo build --release +) # Kill any existing server process echo "Stopping existing server..." @@ -35,8 +37,12 @@ rm -rf /host/tmp/vaultlink-e2e-databases # Start the server in the background echo "Starting server..." -./target/release/sync_server config-e2e.yml & -server_pid=$! +( + cd sync-server + ./target/release/sync_server config-e2e.yml & + echo $! > ../logs/server.pid +) +server_pid=$(cat logs/server.pid) echo "Server started with PID: $server_pid" # Ensure server is killed on script exit @@ -52,22 +58,22 @@ trap cleanup_server EXIT cd .. -cd frontend -npm ci -npm run build +( + cd frontend + npm ci + npm run build +) -../scripts/utils/wait-for-server.sh +./scripts/utils/wait-for-server.sh pids=() for i in $(seq 1 $process_count); do - node test-client/dist/cli.js > "../logs/log_${i}.log" 2>&1 & + node frontend/test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 & pid=$! pids+=($pid) echo "Started process $i with PID: $pid (log: logs/log_${i}.log)" done -cd .. - print_failed_log() { for i in $(seq 1 $process_count); do if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then -- 2.54.0 From 02f70d5f9fecda9a222fce2876770dcc67e17601 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:24:47 +0100 Subject: [PATCH 05/48] Change to SHA256 hashing --- frontend/sync-client/src/utils/hash.test.ts | 7 +++++++ frontend/sync-client/src/utils/hash.ts | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 frontend/sync-client/src/utils/hash.test.ts diff --git a/frontend/sync-client/src/utils/hash.test.ts b/frontend/sync-client/src/utils/hash.test.ts new file mode 100644 index 00000000..9cf80545 --- /dev/null +++ b/frontend/sync-client/src/utils/hash.test.ts @@ -0,0 +1,7 @@ +import assert from "node:assert"; +import { test } from "node:test"; +import { EMPTY_HASH, hash } from "./hash"; + +void test("EMPTY_HASH is the SHA-256 hash of empty content", async () => { + assert.strictEqual(await hash(new Uint8Array()), EMPTY_HASH); +}); diff --git a/frontend/sync-client/src/utils/hash.ts b/frontend/sync-client/src/utils/hash.ts index 906b6fad..508001a8 100644 --- a/frontend/sync-client/src/utils/hash.ts +++ b/frontend/sync-client/src/utils/hash.ts @@ -1,12 +1,13 @@ -// https://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript -export function hash(content: Uint8Array): string { - let result = 0; - // eslint-disable-next-line @typescript-eslint/prefer-for-of - for (let i = 0; i < content.length; i++) { - result = (result << 5) - result + content[i]; - result |= 0; // Convert to 32bit integer - } - return Math.abs(result).toString(16).padStart(8, "0"); +export async function hash(content: Uint8Array): Promise { + const digest = await globalThis.crypto.subtle.digest( + "SHA-256", + new Uint8Array(content) + ); + + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0") + ).join(""); } -export const EMPTY_HASH = hash(new Uint8Array(0)); +export const EMPTY_HASH = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; -- 2.54.0 From 9f84dce6c369c9695a26850f32f03e2965f0c418 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:25:49 +0100 Subject: [PATCH 06/48] Remove no longer necessary functions --- .../database/get_document_version.rs | 41 -- .../app_state/database/get_latest_document.rs | 43 -- .../database/get_latest_document_by_path.rs | 46 -- .../database/get_latest_documents.rs | 56 --- .../database/get_latest_documents_since.rs | 62 --- .../database/get_max_update_id_in_vault.rs | 29 -- .../database/get_push_acknowledgement.rs | 38 -- .../database/insert_document_version.rs | 79 ---- .../endpoints/fetch_latest_documents.rs | 36 -- .../server/endpoints/push_document/tests.rs | 423 ------------------ sync-server/src/utils/dedup_paths.rs | 150 ------- .../src/utils/find_first_available_path.rs | 26 -- sync-server/src/utils/sanitize_path.rs | 34 -- 13 files changed, 1063 deletions(-) delete mode 100644 sync-server/src/app_state/database/get_document_version.rs delete mode 100644 sync-server/src/app_state/database/get_latest_document.rs delete mode 100644 sync-server/src/app_state/database/get_latest_document_by_path.rs delete mode 100644 sync-server/src/app_state/database/get_latest_documents.rs delete mode 100644 sync-server/src/app_state/database/get_latest_documents_since.rs delete mode 100644 sync-server/src/app_state/database/get_max_update_id_in_vault.rs delete mode 100644 sync-server/src/app_state/database/get_push_acknowledgement.rs delete mode 100644 sync-server/src/app_state/database/insert_document_version.rs delete mode 100644 sync-server/src/server/endpoints/fetch_latest_documents.rs delete mode 100644 sync-server/src/server/endpoints/push_document/tests.rs delete mode 100644 sync-server/src/utils/dedup_paths.rs delete mode 100644 sync-server/src/utils/find_first_available_path.rs delete mode 100644 sync-server/src/utils/sanitize_path.rs diff --git a/sync-server/src/app_state/database/get_document_version.rs b/sync-server/src/app_state/database/get_document_version.rs deleted file mode 100644 index c4113d8b..00000000 --- a/sync-server/src/app_state/database/get_document_version.rs +++ /dev/null @@ -1,41 +0,0 @@ -use anyhow::{Context as _, Result}; -use sqlx::types::chrono::Utc; -use uuid::fmt::Hyphenated; - -use super::models::{StoredDocumentVersion, VaultId, VaultUpdateId}; -use super::{Database, Transaction}; - -impl Database { - pub async fn get_document_version( - &self, - vault: &VaultId, - vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id - from documents - where vault_update_id = ?"#, - vault_update_id - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch document version") - } -} diff --git a/sync-server/src/app_state/database/get_latest_document.rs b/sync-server/src/app_state/database/get_latest_document.rs deleted file mode 100644 index 0bb6ecb6..00000000 --- a/sync-server/src/app_state/database/get_latest_document.rs +++ /dev/null @@ -1,43 +0,0 @@ -use anyhow::{Context as _, Result}; -use sqlx::types::chrono::Utc; -use uuid::fmt::Hyphenated; - -use super::models::{DocumentId, StoredDocumentVersion, VaultId}; -use super::{Database, Transaction}; - -impl Database { - pub async fn get_latest_document( - &self, - vault: &VaultId, - document_id: &DocumentId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let document_id = document_id.as_hyphenated(); - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id - from latest_document_versions - where document_id = ? - "#, - document_id - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest document version") - } -} diff --git a/sync-server/src/app_state/database/get_latest_document_by_path.rs b/sync-server/src/app_state/database/get_latest_document_by_path.rs deleted file mode 100644 index 95e49875..00000000 --- a/sync-server/src/app_state/database/get_latest_document_by_path.rs +++ /dev/null @@ -1,46 +0,0 @@ -use anyhow::{Context as _, Result}; -use sqlx::types::chrono::Utc; -use uuid::fmt::Hyphenated; - -use super::models::{StoredDocumentVersion, VaultId}; -use super::{Database, Transaction}; - -impl Database { - pub async fn get_latest_document_by_path( - &self, - vault: &VaultId, - relative_path: &str, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query_as!( - StoredDocumentVersion, - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - content, - is_deleted, - user_id, - device_id - from latest_document_versions - where relative_path = ? and is_deleted = false - order by vault_update_id desc -- `latest_document_versions` only contains a single latest version of each document, however, - -- multiple documents can have the same `relative_path`, if they have been deleted. That's - -- why we only care about the latest version of the document with the given relative path. - limit 1 - "#, - relative_path - ); - - if let Some(transaction) = transaction { - query.fetch_optional(&mut **transaction).await - } else { - query - .fetch_optional(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest document version") - } -} diff --git a/sync-server/src/app_state/database/get_latest_documents.rs b/sync-server/src/app_state/database/get_latest_documents.rs deleted file mode 100644 index 700a505c..00000000 --- a/sync-server/src/app_state/database/get_latest_documents.rs +++ /dev/null @@ -1,56 +0,0 @@ -use anyhow::{Context as _, Result}; -use sqlx::types::chrono::Utc; -use uuid::fmt::Hyphenated; - -use super::models::{DocumentVersionWithoutContent, VaultId}; -use super::{Database, Transaction}; - -impl Database { - /// Return the latest state of all documents in the vault - pub async fn get_latest_documents( - &self, - vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query!( - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - order by vault_update_id - "#, - ); - - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await - } else { - query - .fetch_all(&self.get_connection_pool(vault).await?) - .await - } - .context("Cannot fetch latest documents") - .map(|rows| { - rows.into_iter() - .map(|row| DocumentVersionWithoutContent { - vault_update_id: row.vault_update_id, - document_id: row.document_id.into(), - relative_path: row.relative_path, - updated_date: row.updated_date, - is_deleted: row.is_deleted, - user_id: row.user_id, - device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), - }) - .collect() - }) - } -} diff --git a/sync-server/src/app_state/database/get_latest_documents_since.rs b/sync-server/src/app_state/database/get_latest_documents_since.rs deleted file mode 100644 index 48c2f0e9..00000000 --- a/sync-server/src/app_state/database/get_latest_documents_since.rs +++ /dev/null @@ -1,62 +0,0 @@ -use anyhow::{Context as _, Result}; -use sqlx::types::chrono::Utc; -use uuid::fmt::Hyphenated; - -use super::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId}; -use super::{Database, Transaction}; - -impl Database { - /// Return the latest state of all documents (including deleted) in the - /// vault which have changed since the given update id - pub async fn get_latest_documents_since( - &self, - vault: &VaultId, - vault_update_id: VaultUpdateId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result> { - let query = sqlx::query!( - r#" - select - vault_update_id, - document_id as "document_id: Hyphenated", - relative_path, - updated_date as "updated_date: chrono::DateTime", - is_deleted, - user_id, - device_id, - length(content) as "content_size: u64" - from latest_document_versions - where vault_update_id > ? - order by vault_update_id - "#, - vault_update_id - ); - - if let Some(transaction) = transaction { - query.fetch_all(&mut **transaction).await - } else { - query - .fetch_all(&self.get_connection_pool(vault).await?) - .await - } - .with_context(|| { - format!("Cannot fetch latest documents since vault_update_id `{vault_update_id}`") - }) - .map(|rows| { - rows.into_iter() - .map(|row| DocumentVersionWithoutContent { - vault_update_id: row.vault_update_id, - document_id: row.document_id.into(), - relative_path: row.relative_path, - updated_date: row.updated_date, - is_deleted: row.is_deleted, - user_id: row.user_id, - device_id: row.device_id, - content_size: row - .content_size - .expect("Content size can't be null but sqlx can't infer it"), - }) - .collect() - }) - } -} diff --git a/sync-server/src/app_state/database/get_max_update_id_in_vault.rs b/sync-server/src/app_state/database/get_max_update_id_in_vault.rs deleted file mode 100644 index cc4a43c7..00000000 --- a/sync-server/src/app_state/database/get_max_update_id_in_vault.rs +++ /dev/null @@ -1,29 +0,0 @@ -use anyhow::{Context as _, Result}; - -use super::models::VaultId; -use super::{Database, Transaction}; - -impl Database { - pub async fn get_max_update_id_in_vault( - &self, - vault: &VaultId, - transaction: Option<&mut Transaction<'_>>, - ) -> Result { - let query = sqlx::query!( - r#" - select coalesce(max(vault_update_id), 0) as max_vault_update_id - from documents - "#, - ); - - if let Some(transaction) = transaction { - query.fetch_one(&mut **transaction).await - } else { - query - .fetch_one(&self.get_connection_pool(vault).await?) - .await - } - .map(|row| row.max_vault_update_id) - .context("Cannot fetch max update id in vault") - } -} diff --git a/sync-server/src/app_state/database/get_push_acknowledgement.rs b/sync-server/src/app_state/database/get_push_acknowledgement.rs deleted file mode 100644 index cd8cd593..00000000 --- a/sync-server/src/app_state/database/get_push_acknowledgement.rs +++ /dev/null @@ -1,38 +0,0 @@ -use anyhow::{Context as _, Result}; - -use super::models::{StoredDocumentVersion, VaultId}; -use super::{Database, Transaction}; - -impl Database { - /// Look up the immutable version acknowledged by a previous push. - pub async fn get_push_acknowledgement( - &self, - vault_id: &VaultId, - request_id: uuid::Uuid, - transaction: &mut Transaction<'_>, - ) -> Result, StoredDocumentVersion)>> { - let request_id = request_id.hyphenated().to_string(); - let acknowledgement = sqlx::query!( - "select vault_update_id, request_fingerprint from push_acknowledgements where request_id = ?", - request_id - ) - .fetch_optional(&mut **transaction) - .await - .context("Cannot fetch push acknowledgement")?; - - match acknowledgement { - Some(acknowledgement) => { - let version = self - .get_document_version( - vault_id, - acknowledgement.vault_update_id, - Some(transaction), - ) - .await? - .context("Acknowledged document version is missing")?; - Ok(Some((acknowledgement.request_fingerprint, version))) - } - None => Ok(None), - } - } -} diff --git a/sync-server/src/app_state/database/insert_document_version.rs b/sync-server/src/app_state/database/insert_document_version.rs deleted file mode 100644 index 16752471..00000000 --- a/sync-server/src/app_state/database/insert_document_version.rs +++ /dev/null @@ -1,79 +0,0 @@ -use anyhow::{Context as _, Result}; - -use super::models::{StoredDocumentVersion, VaultId}; -use super::{Database, Transaction}; -use crate::app_state::websocket::models::{ - WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate, -}; - -impl Database { - /// Commit the snapshot and acknowledgement atomically, then notify subscribers. - pub async fn insert_document_version( - &self, - vault_id: &VaultId, - version: &StoredDocumentVersion, - request_id: uuid::Uuid, - request_fingerprint: &[u8], - mut transaction: Transaction<'_>, - ) -> Result<()> { - let document_id = version.document_id.as_hyphenated(); - sqlx::query!( - r#" - insert into documents ( - vault_update_id, - document_id, - relative_path, - updated_date, - content, - is_deleted, - user_id, - device_id - ) - values (?, ?, ?, ?, ?, ?, ?, ?) - "#, - version.vault_update_id, - document_id, - version.relative_path, - version.updated_date, - version.content, - version.is_deleted, - version.user_id, - version.device_id - ) - .execute(&mut *transaction) - .await - .context("Cannot insert document version")?; - - let request_id = request_id.hyphenated().to_string(); - sqlx::query!( - "insert into push_acknowledgements (request_id, request_fingerprint, vault_update_id) values (?, ?, ?)", - request_id, - request_fingerprint, - version.vault_update_id - ) - .execute(&mut *transaction) - .await - .context("Cannot save push acknowledgement")?; - - transaction - .commit() - .await - .context("Failed to commit push")?; - - // notifying the listeners here means that no update can circumvent this step - self.broadcasts - .send_document_update( - vault_id.clone(), - WebSocketServerMessageWithOrigin::with_origin( - version.device_id.clone(), - WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: vec![version.clone().into()], - is_initial_sync: false, - }), - ), - ) - .await; - - Ok(()) - } -} diff --git a/sync-server/src/server/endpoints/fetch_latest_documents.rs b/sync-server/src/server/endpoints/fetch_latest_documents.rs deleted file mode 100644 index eee16441..00000000 --- a/sync-server/src/server/endpoints/fetch_latest_documents.rs +++ /dev/null @@ -1,36 +0,0 @@ -use axum::{ - Json, - extract::{Path, State}, -}; -use log::debug; -use serde::Deserialize; - -use crate::{ - app_state::{ - AppState, - database::models::{DocumentVersionWithoutContent, VaultId}, - }, - errors::{SyncServerError, server_error}, - utils::normalize::normalize, -}; - -#[derive(Deserialize)] -pub struct FetchLatestDocumentsPathParams { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, -} - -#[axum::debug_handler] -pub async fn fetch_latest_documents( - Path(FetchLatestDocumentsPathParams { vault_id }): Path, - State(state): State, -) -> Result>, SyncServerError> { - debug!("Fetching latest documents in {vault_id}"); - - state - .database - .get_latest_documents(&vault_id, None) - .await - .map(Json) - .map_err(server_error) -} diff --git a/sync-server/src/server/endpoints/push_document/tests.rs b/sync-server/src/server/endpoints/push_document/tests.rs deleted file mode 100644 index 291c2def..00000000 --- a/sync-server/src/server/endpoints/push_document/tests.rs +++ /dev/null @@ -1,423 +0,0 @@ -use std::path::PathBuf; - -use super::*; -use crate::{ - app_state::{ - database::models::{DocumentVersionWithoutContent, VaultUpdateId}, - websocket::broadcasts::Broadcasts, - }, - config::{database_config::DatabaseConfig, server_config::ServerConfig}, -}; - -struct Push { - document_id: DocumentId, - request_id: uuid::Uuid, - parent_version_id: Option, - relative_path: String, - content: PushContent, - user_id: String, - device_id: String, -} - -impl Push { - async fn send( - self, - database: &Database, - vault: &VaultId, - ) -> Result { - accept_push( - database, - vault, - self.document_id, - self.user_id, - self.device_id, - PushDocument { - request_id: self.request_id, - parent_version_id: self.parent_version_id, - relative_path: self.relative_path, - content: self.content, - }, - ) - .await - } -} - -struct TestVault { - directory: PathBuf, - database: Database, - vault: VaultId, -} - -impl TestVault { - async fn new() -> Self { - let directory = - std::env::temp_dir().join(format!("vaultlink-cas-{}", uuid::Uuid::new_v4())); - let database = Self::open(&directory).await; - Self { - directory, - database, - vault: "test".to_owned(), - } - } - - async fn open(directory: &std::path::Path) -> Database { - Database::try_new( - &DatabaseConfig { - databases_directory_path: directory.to_owned(), - ..Default::default() - }, - &Broadcasts::new(&ServerConfig { - max_clients_per_vault: 16, - ..Default::default() - }), - ) - .await - .unwrap() - } - - async fn send(&self, push: Push) -> DocumentUpdateResponse { - push.send(&self.database, &self.vault).await.unwrap() - } - - async fn latest(&self, document: DocumentId) -> StoredDocumentVersion { - self.database - .get_latest_document(&self.vault, &document, None) - .await - .unwrap() - .unwrap() - } -} - -impl Drop for TestVault { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.directory); - } -} - -fn snapshot(document_id: DocumentId, parent: Option, text: &str) -> Push { - Push { - document_id, - request_id: uuid::Uuid::new_v4(), - parent_version_id: parent, - relative_path: "note.md".to_owned(), - content: PushContent::Snapshot(STANDARD.encode(text)), - user_id: "user".to_owned(), - device_id: "device".to_owned(), - } -} - -fn accepted(response: DocumentUpdateResponse) -> DocumentVersionWithoutContent { - match response { - DocumentUpdateResponse::Accepted(version) => version, - response @ DocumentUpdateResponse::StaleBase(_) => { - panic!("Expected Accepted, got {response:?}") - } - } -} - -#[tokio::test] -async fn json_snapshot_preserves_bytes_and_rejects_invalid_base64_without_a_version() { - let vault = TestVault::new().await; - let document = uuid::Uuid::new_v4(); - let content = [0, 255, 13, 10, 42]; - let request: PushDocument = serde_json::from_value(serde_json::json!({ - "requestId": uuid::Uuid::new_v4(), - "parentVersionId": null, - "relativePath": "snapshot.bin", - "content": { "type": "Snapshot", "value": STANDARD.encode(content) }, - })) - .unwrap(); - let base = accepted( - accept_push( - &vault.database, - &vault.vault, - document, - "user".to_owned(), - "device".to_owned(), - request, - ) - .await - .unwrap(), - ); - assert_eq!(vault.latest(document).await.content, content); - - let invalid: PushDocument = serde_json::from_value(serde_json::json!({ - "requestId": uuid::Uuid::new_v4(), - "parentVersionId": base.vault_update_id, - "relativePath": "snapshot.bin", - "content": { "type": "Snapshot", "value": "not base64!" }, - })) - .unwrap(); - assert!(matches!( - accept_push( - &vault.database, - &vault.vault, - document, - "user".to_owned(), - "device".to_owned(), - invalid - ) - .await, - Err(SyncServerError::ClientError(_)) - )); - assert_eq!( - vault.latest(document).await.vault_update_id, - base.vault_update_id - ); -} - -#[tokio::test] -async fn concurrent_writers_only_accept_one_snapshot_and_preserve_the_base() { - let vault = TestVault::new().await; - let document = uuid::Uuid::new_v4(); - let base = accepted(vault.send(snapshot(document, None, "base")).await); - let (left, right) = tokio::join!( - vault.send(snapshot(document, Some(base.vault_update_id), "left")), - vault.send(snapshot(document, Some(base.vault_update_id), "right")), - ); - let (winner, rejected) = match (left, right) { - (DocumentUpdateResponse::Accepted(winner), DocumentUpdateResponse::StaleBase(rejected)) - | (DocumentUpdateResponse::StaleBase(rejected), DocumentUpdateResponse::Accepted(winner)) => { - (winner, rejected) - } - outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"), - }; - let latest = vault.latest(document).await; - assert_eq!(winner.vault_update_id, rejected.vault_update_id); - assert_eq!( - STANDARD.decode(rejected.content_base64).unwrap(), - latest.content - ); - assert!(latest.content == b"left" || latest.content == b"right"); - let original = vault - .database - .get_document_version(&vault.vault, base.vault_update_id, None) - .await - .unwrap() - .unwrap(); - assert_eq!(original.content, b"base"); - assert_eq!( - vault - .database - .get_max_update_id_in_vault(&vault.vault, None) - .await - .unwrap(), - 2 - ); -} - -#[tokio::test] -async fn acknowledgements_survive_reopening_and_return_the_original_version() { - let vault = TestVault::new().await; - let document = uuid::Uuid::new_v4(); - let create = snapshot(document, None, "created"); - let create_id = create.request_id; - let base = accepted(vault.send(create).await); - let update = snapshot(document, Some(base.vault_update_id), "updated"); - let update_id = update.request_id; - let updated = accepted(vault.send(update).await); - let latest = accepted( - vault - .send(snapshot(document, Some(updated.vault_update_id), "newest")) - .await, - ); - - let reopened = TestVault::open(&vault.directory).await; - for (request_id, parent, text, original) in [ - (create_id, None, "created", base), - (update_id, Some(1), "updated", updated), - ] { - let mut retry = snapshot(document, parent, text); - retry.request_id = request_id; - retry.device_id = "new-session-after-restart".to_owned(); - let acknowledged = accepted(retry.send(&reopened, &vault.vault).await.unwrap()); - assert_eq!( - serde_json::to_value(acknowledged).unwrap(), - serde_json::to_value(original).unwrap() - ); - } - assert_eq!( - vault.latest(document).await.vault_update_id, - latest.vault_update_id - ); - - let mut reused = snapshot(uuid::Uuid::new_v4(), None, "wrong document"); - reused.request_id = create_id; - assert!(matches!( - reused.send(&reopened, &vault.vault).await, - Err(SyncServerError::ClientError(_)) - )); - - let mut changed_path = snapshot(document, None, "created"); - changed_path.relative_path = "other.md".to_owned(); - let mut changed_operation = snapshot(document, None, "created"); - changed_operation.content = PushContent::Delete; - for mut changed_payload in [ - snapshot(document, None, "different content"), - snapshot(document, Some(1), "created"), - changed_path, - changed_operation, - ] { - changed_payload.request_id = create_id; - assert!(matches!( - changed_payload.send(&reopened, &vault.vault).await, - Err(SyncServerError::ClientError(_)) - )); - } -} - -#[tokio::test] -async fn deletes_compare_the_base_and_replay_without_resurrecting_documents() { - let vault = TestVault::new().await; - let document = uuid::Uuid::new_v4(); - let base = accepted(vault.send(snapshot(document, None, "base")).await); - let updated = accepted( - vault - .send(snapshot(document, Some(base.vault_update_id), "edited")) - .await, - ); - let mut delete = snapshot(document, Some(base.vault_update_id), ""); - delete.content = PushContent::Delete; - assert!(matches!( - vault.send(delete).await, - DocumentUpdateResponse::StaleBase(_) - )); - assert!(!vault.latest(document).await.is_deleted); - - let mut delete = snapshot(document, Some(updated.vault_update_id), ""); - delete.content = PushContent::Delete; - let request_id = delete.request_id; - let deleted = accepted(vault.send(delete).await); - assert!(deleted.is_deleted); - assert_eq!(vault.latest(document).await.content, b"edited"); - let mut retry = snapshot(document, Some(updated.vault_update_id), ""); - retry.content = PushContent::Delete; - retry.request_id = request_id; - assert_eq!( - accepted(vault.send(retry).await).vault_update_id, - deleted.vault_update_id - ); - - let terminal = vault - .send(snapshot( - document, - Some(deleted.vault_update_id), - "resurrect", - )) - .await; - assert!(matches!(terminal, DocumentUpdateResponse::StaleBase(version) if version.is_deleted)); - let recreated = accepted( - vault - .send(snapshot(uuid::Uuid::new_v4(), None, "new identity")) - .await, - ); - assert_eq!(recreated.relative_path, "note.md"); -} - -#[tokio::test] -async fn deleting_a_missing_document_without_a_parent_is_a_client_error() { - let vault = TestVault::new().await; - let document = uuid::Uuid::new_v4(); - let mut delete = snapshot(document, None, ""); - delete.content = PushContent::Delete; - assert!(matches!( - delete.send(&vault.database, &vault.vault).await, - Err(SyncServerError::ClientError(_)) - )); - assert_eq!( - vault - .database - .get_latest_document(&vault.vault, &document, None) - .await - .unwrap(), - None - ); -} - -#[tokio::test] -async fn transport_diffs_reconstruct_exact_snapshots_and_reject_invalid_bases() { - let vault = TestVault::new().await; - let document = uuid::Uuid::new_v4(); - let original = "Hello world 🌍"; - let desired = "Hi beautiful world 🌍"; - let base = accepted(vault.send(snapshot(document, None, original)).await); - let mut update = snapshot(document, Some(base.vault_update_id), ""); - update.content = - PushContent::Diff(EditedText::from_strings(original, &desired.into()).to_diff()); - let updated = accepted(vault.send(update).await); - assert_eq!(vault.latest(document).await.content, desired.as_bytes()); - - for invalid_length in [i64::MIN, i64::MAX] { - let mut malformed = snapshot(document, Some(updated.vault_update_id), ""); - malformed.content = PushContent::Diff(vec![NumberOrText::Number(invalid_length)]); - assert!(matches!( - malformed.send(&vault.database, &vault.vault).await, - Err(SyncServerError::ClientError(_)) - )); - } - assert_eq!( - vault.latest(document).await.vault_update_id, - updated.vault_update_id - ); - - let another = accepted( - vault - .send(snapshot(uuid::Uuid::new_v4(), None, "another document")) - .await, - ); - let mut foreign_base = snapshot(document, Some(another.vault_update_id), ""); - foreign_base.content = PushContent::Diff(vec![]); - assert!( - matches!(vault.send(foreign_base).await, DocumentUpdateResponse::StaleBase(version) if version.vault_update_id == updated.vault_update_id) - ); - - let binary_document = uuid::Uuid::new_v4(); - let mut binary = snapshot(binary_document, None, ""); - binary.content = PushContent::Snapshot(STANDARD.encode([0xff])); - let binary_base = accepted(vault.send(binary).await); - let mut invalid_text = snapshot(binary_document, Some(binary_base.vault_update_id), ""); - invalid_text.content = PushContent::Diff(vec![]); - assert!(matches!( - invalid_text.send(&vault.database, &vault.vault).await, - Err(SyncServerError::ClientError(_)) - )); - assert_eq!( - vault.latest(binary_document).await.vault_update_id, - binary_base.vault_update_id - ); -} - -#[tokio::test] -async fn renames_allocate_paths_atomically_without_rewriting_history() { - let vault = TestVault::new().await; - let first = uuid::Uuid::new_v4(); - let second = uuid::Uuid::new_v4(); - let base = accepted(vault.send(snapshot(first, None, "first")).await); - let collision = accepted(vault.send(snapshot(second, None, "second")).await); - assert_ne!(base.relative_path, collision.relative_path); - - let mut rename = snapshot(first, Some(base.vault_update_id), "first"); - rename.relative_path = "renamed.md".to_owned(); - let renamed = accepted(vault.send(rename).await); - let mut stale_rename = snapshot(first, Some(base.vault_update_id), "first"); - stale_rename.relative_path = "stale.md".to_owned(); - assert!( - matches!(vault.send(stale_rename).await, DocumentUpdateResponse::StaleBase(version) if version.relative_path == renamed.relative_path) - ); - - let mut move_into_occupied = snapshot(second, Some(collision.vault_update_id), "second"); - move_into_occupied.relative_path = renamed.relative_path.clone(); - let allocated = accepted(vault.send(move_into_occupied).await); - assert_ne!(allocated.relative_path, renamed.relative_path); - assert_eq!(vault.latest(first).await.relative_path, "renamed.md"); - assert_eq!( - vault - .database - .get_document_version(&vault.vault, base.vault_update_id, None) - .await - .unwrap() - .unwrap() - .relative_path, - "note.md" - ); -} diff --git a/sync-server/src/utils/dedup_paths.rs b/sync-server/src/utils/dedup_paths.rs deleted file mode 100644 index bc687f6a..00000000 --- a/sync-server/src/utils/dedup_paths.rs +++ /dev/null @@ -1,150 +0,0 @@ -use regex::Regex; - -pub fn dedup_paths(path: &str) -> impl Iterator { - let mut path_parts = path.split('/').collect::>(); - let file_name = path_parts.pop().unwrap().to_owned(); - - let mut directory = path_parts.join("/"); - if !directory.is_empty() { - directory.push('/'); - } - - // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should split as ".config" + ".json" - let is_simple_dotfile = file_name.starts_with('.') && file_name.matches('.').count() == 1; - - let (stem, extension) = if is_simple_dotfile { - (file_name.clone(), String::new()) - } else { - // Regular file or dotfile with extension - let name_parts = file_name.rsplitn(2, '.').collect::>(); - let mut reverse_parts = name_parts.into_iter().rev(); - match (reverse_parts.next(), reverse_parts.next()) { - (Some(stem), maybe_extension) => ( - stem.to_owned(), - maybe_extension - .map(|ext| format!(".{ext}")) - .unwrap_or_default(), - ), - _ => unreachable!("Path must have at least one part"), - } - }; - - let regex = Regex::new(r" \((\d+)\)$").unwrap(); - let start_number = regex - .captures(&stem) - .and_then(|caps| caps.get(1)) - .and_then(|m| m.as_str().parse::().ok()) - .unwrap_or(0); - - let clean_stem = regex.replace(&stem, "").to_string(); - - (start_number..).map(move |dedup_number| { - if dedup_number == 0 { - format!("{directory}{clean_stem}{extension}") - } else { - format!("{directory}{clean_stem} ({dedup_number}){extension}") - } - }) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_dedup_paths() { - let mut deduped = dedup_paths("file.txt"); - assert_eq!(deduped.next(), Some("file.txt".to_owned())); - assert_eq!(deduped.next(), Some("file (1).txt".to_owned())); - assert_eq!(deduped.next(), Some("file (2).txt".to_owned())); - - let mut deduped = dedup_paths("file"); - assert_eq!(deduped.next(), Some("file".to_owned())); - assert_eq!(deduped.next(), Some("file (1)".to_owned())); - assert_eq!(deduped.next(), Some("file (2)".to_owned())); - - let mut deduped = dedup_paths("file (51).md"); - assert_eq!(deduped.next(), Some("file (51).md".to_owned())); - assert_eq!(deduped.next(), Some("file (52).md".to_owned())); - assert_eq!(deduped.next(), Some("file (53).md".to_owned())); - - let mut deduped = dedup_paths("file (5)"); - assert_eq!(deduped.next(), Some("file (5)".to_owned())); - assert_eq!(deduped.next(), Some("file (6)".to_owned())); - assert_eq!(deduped.next(), Some("file (7)".to_owned())); - - let mut deduped = dedup_paths("my/path.with.dots/file (5).md"); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (5).md".to_owned()) - ); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (6).md".to_owned()) - ); - - let mut deduped = dedup_paths("my/path.with.dots/file (5)"); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (5)".to_owned()) - ); - assert_eq!( - deduped.next(), - Some("my/path.with.dots/file (6)".to_owned()) - ); - } - - #[test] - fn test_regex_capturing_group() { - // Single digit in parentheses - let mut deduped = dedup_paths("document (5).md"); - assert_eq!(deduped.next(), Some("document (5).md".to_owned())); - assert_eq!(deduped.next(), Some("document (6).md".to_owned())); - assert_eq!(deduped.next(), Some("document (7).md".to_owned())); - - // Multi-digit number - let mut deduped = dedup_paths("report (123).pdf"); - assert_eq!(deduped.next(), Some("report (123).pdf".to_owned())); - assert_eq!(deduped.next(), Some("report (124).pdf".to_owned())); - assert_eq!(deduped.next(), Some("report (125).pdf".to_owned())); - - // Number without extension - let mut deduped = dedup_paths("folder (99)"); - assert_eq!(deduped.next(), Some("folder (99)".to_owned())); - assert_eq!(deduped.next(), Some("folder (100)".to_owned())); - assert_eq!(deduped.next(), Some("folder (101)".to_owned())); - } - - #[test] - fn test_dedup_dotfiles() { - // Simple dotfile (no extension) - let mut deduped = dedup_paths(".gitignore"); - assert_eq!(deduped.next(), Some(".gitignore".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (1)".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (2)".to_owned())); - - // Dotfile with extension - let mut deduped = dedup_paths(".config.json"); - assert_eq!(deduped.next(), Some(".config.json".to_owned())); - assert_eq!(deduped.next(), Some(".config (1).json".to_owned())); - assert_eq!(deduped.next(), Some(".config (2).json".to_owned())); - - // Dotfile with number - let mut deduped = dedup_paths(".gitignore (5)"); - assert_eq!(deduped.next(), Some(".gitignore (5)".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (6)".to_owned())); - assert_eq!(deduped.next(), Some(".gitignore (7)".to_owned())); - - // Dotfile with extension and number - let mut deduped = dedup_paths(".config (3).json"); - assert_eq!(deduped.next(), Some(".config (3).json".to_owned())); - assert_eq!(deduped.next(), Some(".config (4).json".to_owned())); - assert_eq!(deduped.next(), Some(".config (5).json".to_owned())); - - // Dotfile in subdirectory - let mut deduped = dedup_paths("my/path/.gitignore"); - assert_eq!(deduped.next(), Some("my/path/.gitignore".to_owned())); - assert_eq!(deduped.next(), Some("my/path/.gitignore (1)".to_owned())); - assert_eq!(deduped.next(), Some("my/path/.gitignore (2)".to_owned())); - } -} diff --git a/sync-server/src/utils/find_first_available_path.rs b/sync-server/src/utils/find_first_available_path.rs deleted file mode 100644 index 7629d8f1..00000000 --- a/sync-server/src/utils/find_first_available_path.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::app_state::database::models::VaultId; -use crate::{app_state::database::Transaction, utils::dedup_paths::dedup_paths}; -use anyhow::Result; -use log::{debug, info}; - -pub async fn find_first_available_path( - vault_id: &VaultId, - sanitized_relative_path: &str, - database: &crate::app_state::database::Database, - transaction: &mut Transaction<'_>, -) -> Result { - info!("Finding first available path for `{sanitized_relative_path}` in vault `{vault_id}`"); - for candidate in dedup_paths(sanitized_relative_path) { - debug!("Checking candidate path for deconflicting names: `{candidate}`"); - if database - .get_latest_document_by_path(vault_id, &candidate, Some(transaction)) - .await? - .is_none() - { - info!("Selected available path: `{candidate}`"); - return Ok(candidate); - } - } - - unreachable!("dedup_paths produces infinite paths"); -} diff --git a/sync-server/src/utils/sanitize_path.rs b/sync-server/src/utils/sanitize_path.rs deleted file mode 100644 index 9703225c..00000000 --- a/sync-server/src/utils/sanitize_path.rs +++ /dev/null @@ -1,34 +0,0 @@ -/// Sanitize the document's path to allow all clients to create the same path in -/// their filesystem. If we didn't do this server-side, client's would need to -/// deal with mapping invalid names to valid ones and then back. -pub fn sanitize_path(path: &str) -> String { - let options = sanitize_filename::Options { - truncate: true, - windows: true, // Windows is the lowest common denominator - replacement: "", - }; - - path.split('/') - .map(|part| { - let proposal = sanitize_filename::sanitize_with_options(part, options.clone()); - if !part.is_empty() && proposal.is_empty() { - "_".to_owned() - } else { - proposal - } - }) - .collect::>() - .join("/") -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_sanitize_path() { - assert_eq!(sanitize_path("/my/path/what?"), "/my/path/what"); - assert_eq!(sanitize_path("file (1).md"), "file (1).md"); - assert_eq!(sanitize_path("/my/path/\\\\:?"), "/my/path/_"); - } -} -- 2.54.0 From 4b623c36d982c215b9c1b1454ac3e3c6874c5764 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:27:32 +0100 Subject: [PATCH 07/48] New protocol version --- frontend/sync-client/src/consts.ts | 2 +- sync-server/src/consts.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index da70ba47..edb133fb 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -2,5 +2,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; -export const SUPPORTED_API_VERSION = 2; +export const SUPPORTED_API_VERSION = 4; export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index 9e9890c0..83d99a82 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -20,4 +20,4 @@ pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; -pub const SUPPORTED_API_VERSION: u32 = 3; +pub const SUPPORTED_API_VERSION: u32 = 4; -- 2.54.0 From 985f9ebf3fcfdc6ab4bd6d188da75fe0227034c6 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:52:12 +0100 Subject: [PATCH 08/48] Unformat --- .../src/services/types/ClientCursors.ts | 6 +-- .../types/CursorPositionFromClient.ts | 4 +- .../types/CursorPositionFromServer.ts | 2 +- .../src/services/types/CursorSpan.ts | 2 +- .../services/types/DocumentUpdateResponse.ts | 4 +- .../src/services/types/DocumentVersion.ts | 10 +---- .../types/DocumentVersionWithoutContent.ts | 9 +---- .../src/services/types/DocumentWithCursors.ts | 7 +--- .../src/services/types/EventBatch.ts | 2 +- .../src/services/types/EventRecord.ts | 5 +-- .../src/services/types/Manifest.ts | 2 +- .../services/types/ManifestUpdateResponse.ts | 4 +- .../src/services/types/PingResponse.ts | 39 +++++++++---------- .../src/services/types/PushContent.ts | 4 +- .../src/services/types/PushDocument.ts | 6 +-- .../src/services/types/PushManifest.ts | 6 +-- .../src/services/types/SerializedError.ts | 6 +-- .../src/services/types/VaultEvent.ts | 4 +- .../src/services/types/VaultSnapshot.ts | 6 +-- .../services/types/WebSocketClientMessage.ts | 4 +- .../src/services/types/WebSocketHandshake.ts | 6 +-- .../services/types/WebSocketServerMessage.ts | 4 +- 22 files changed, 40 insertions(+), 102 deletions(-) diff --git a/frontend/sync-client/src/services/types/ClientCursors.ts b/frontend/sync-client/src/services/types/ClientCursors.ts index e8c9b93d..bb629100 100644 --- a/frontend/sync-client/src/services/types/ClientCursors.ts +++ b/frontend/sync-client/src/services/types/ClientCursors.ts @@ -1,8 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentWithCursors } from "./DocumentWithCursors"; -export interface ClientCursors { - userName: string; - deviceId: string; - documentsWithCursors: DocumentWithCursors[]; -} +export type ClientCursors = { userName: string, deviceId: string, documentsWithCursors: Array, }; diff --git a/frontend/sync-client/src/services/types/CursorPositionFromClient.ts b/frontend/sync-client/src/services/types/CursorPositionFromClient.ts index ee937f4e..60b48e5e 100644 --- a/frontend/sync-client/src/services/types/CursorPositionFromClient.ts +++ b/frontend/sync-client/src/services/types/CursorPositionFromClient.ts @@ -1,6 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentWithCursors } from "./DocumentWithCursors"; -export interface CursorPositionFromClient { - documentsWithCursors: DocumentWithCursors[]; -} +export type CursorPositionFromClient = { documentsWithCursors: Array, }; diff --git a/frontend/sync-client/src/services/types/CursorPositionFromServer.ts b/frontend/sync-client/src/services/types/CursorPositionFromServer.ts index d9be7d82..c8444892 100644 --- a/frontend/sync-client/src/services/types/CursorPositionFromServer.ts +++ b/frontend/sync-client/src/services/types/CursorPositionFromServer.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ClientCursors } from "./ClientCursors"; -export interface CursorPositionFromServer { clients: ClientCursors[] } +export type CursorPositionFromServer = { clients: Array, }; diff --git a/frontend/sync-client/src/services/types/CursorSpan.ts b/frontend/sync-client/src/services/types/CursorSpan.ts index d2fc9614..d0bce6ea 100644 --- a/frontend/sync-client/src/services/types/CursorSpan.ts +++ b/frontend/sync-client/src/services/types/CursorSpan.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface CursorSpan { start: number; end: number } +export type CursorSpan = { start: number, end: number, }; diff --git a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts index 29f23cfb..572e2d9d 100644 --- a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts @@ -5,6 +5,4 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutCont /** * Response to an update document request. */ -export type DocumentUpdateResponse = - | ({ type: "Accepted" } & DocumentVersionWithoutContent) - | ({ type: "StaleBase" } & DocumentVersion); +export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersion; diff --git a/frontend/sync-client/src/services/types/DocumentVersion.ts b/frontend/sync-client/src/services/types/DocumentVersion.ts index a636e4fb..930b4899 100644 --- a/frontend/sync-client/src/services/types/DocumentVersion.ts +++ b/frontend/sync-client/src/services/types/DocumentVersion.ts @@ -1,11 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DocumentVersion { - contentBase64: string; - vaultUpdateId: number; - documentId: string; - updatedDate: string; - userId: string; - deviceId: string; - contentSize: number; -} +export type DocumentVersion = { contentBase64: string, vaultUpdateId: number, documentId: string, updatedDate: string, userId: string, deviceId: string, contentSize: number, }; diff --git a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts index d4e8165a..fac3b420 100644 --- a/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts +++ b/frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts @@ -1,10 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface DocumentVersionWithoutContent { - vaultUpdateId: number; - documentId: string; - updatedDate: string; - userId: string; - deviceId: string; - contentSize: number; -} +export type DocumentVersionWithoutContent = { vaultUpdateId: number, documentId: string, updatedDate: string, userId: string, deviceId: string, contentSize: number, }; diff --git a/frontend/sync-client/src/services/types/DocumentWithCursors.ts b/frontend/sync-client/src/services/types/DocumentWithCursors.ts index dcfe6e2d..38857a35 100644 --- a/frontend/sync-client/src/services/types/DocumentWithCursors.ts +++ b/frontend/sync-client/src/services/types/DocumentWithCursors.ts @@ -1,9 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { CursorSpan } from "./CursorSpan"; -export interface DocumentWithCursors { - vault_update_id: number | null; - document_id: string; - relative_path: string; - cursors: CursorSpan[]; -} +export type DocumentWithCursors = { vault_update_id: number | null, document_id: string, relative_path: string, cursors: Array, }; diff --git a/frontend/sync-client/src/services/types/EventBatch.ts b/frontend/sync-client/src/services/types/EventBatch.ts index 11412274..f17f17ed 100644 --- a/frontend/sync-client/src/services/types/EventBatch.ts +++ b/frontend/sync-client/src/services/types/EventBatch.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { EventRecord } from "./EventRecord"; -export interface EventBatch { headEventId: number; events: EventRecord[] } +export type EventBatch = { headEventId: number, events: Array, }; diff --git a/frontend/sync-client/src/services/types/EventRecord.ts b/frontend/sync-client/src/services/types/EventRecord.ts index 2e50d69a..6a1bac34 100644 --- a/frontend/sync-client/src/services/types/EventRecord.ts +++ b/frontend/sync-client/src/services/types/EventRecord.ts @@ -2,7 +2,4 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; import type { Manifest } from "./Manifest"; -export type EventRecord = { eventId: number; requestId: string } & ( - | { type: "content"; document: DocumentVersionWithoutContent } - | { type: "manifest"; manifest: Manifest } -); +export type EventRecord = { eventId: number, requestId: string, } & ({ "type": "content", document: DocumentVersionWithoutContent, } | { "type": "manifest", manifest: Manifest, }); diff --git a/frontend/sync-client/src/services/types/Manifest.ts b/frontend/sync-client/src/services/types/Manifest.ts index f32ce231..9572ad93 100644 --- a/frontend/sync-client/src/services/types/Manifest.ts +++ b/frontend/sync-client/src/services/types/Manifest.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface Manifest { manifestId: number; entries: Record } +export type Manifest = { manifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts b/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts index db3ea4a1..76ac2f7f 100644 --- a/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts @@ -1,6 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Manifest } from "./Manifest"; -export type ManifestUpdateResponse = - | ({ type: "Accepted" } & Manifest) - | ({ type: "StaleBase" } & Manifest); +export type ManifestUpdateResponse = { "type": "Accepted" } & Manifest | { "type": "StaleBase" } & Manifest; diff --git a/frontend/sync-client/src/services/types/PingResponse.ts b/frontend/sync-client/src/services/types/PingResponse.ts index f96520e9..c38845d2 100644 --- a/frontend/sync-client/src/services/types/PingResponse.ts +++ b/frontend/sync-client/src/services/types/PingResponse.ts @@ -3,23 +3,22 @@ /** * Response to a ping request. */ -export interface PingResponse { - /** - * Semantic version of the server. - */ - serverVersion: string; - /** - * Whether the client is authenticated based on the sent Authorization - * header. - */ - isAuthenticated: boolean; - /** - * List of file extensions that are allowed to be merged. - */ - mergeableFileExtensions: string[]; - /** - * API version ensuring backwards & forwards compatibility between the client - * and server. - */ - supportedApiVersion: number; -} +export type PingResponse = { +/** + * Semantic version of the server. + */ +serverVersion: string, +/** + * Whether the client is authenticated based on the sent Authorization + * header. + */ +isAuthenticated: boolean, +/** + * List of file extensions that are allowed to be merged. + */ +mergeableFileExtensions: Array, +/** + * API version ensuring backwards & forwards compatibility between the client + * and server. + */ +supportedApiVersion: number, }; diff --git a/frontend/sync-client/src/services/types/PushContent.ts b/frontend/sync-client/src/services/types/PushContent.ts index 3035b567..6a4820ac 100644 --- a/frontend/sync-client/src/services/types/PushContent.ts +++ b/frontend/sync-client/src/services/types/PushContent.ts @@ -1,5 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type PushContent = - | { type: "Snapshot"; value: string } - | { type: "Diff"; value: (number | string)[] }; +export type PushContent = { "type": "Snapshot", "value": string } | { "type": "Diff", "value": Array }; diff --git a/frontend/sync-client/src/services/types/PushDocument.ts b/frontend/sync-client/src/services/types/PushDocument.ts index 8410b96e..a23b3bc4 100644 --- a/frontend/sync-client/src/services/types/PushDocument.ts +++ b/frontend/sync-client/src/services/types/PushDocument.ts @@ -1,8 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { PushContent } from "./PushContent"; -export interface PushDocument { - requestId: string; - parentVersionId: number | null; - content: PushContent; -} +export type PushDocument = { requestId: string, parentVersionId: number | null, content: PushContent, }; diff --git a/frontend/sync-client/src/services/types/PushManifest.ts b/frontend/sync-client/src/services/types/PushManifest.ts index ee5e68e4..743ad219 100644 --- a/frontend/sync-client/src/services/types/PushManifest.ts +++ b/frontend/sync-client/src/services/types/PushManifest.ts @@ -1,7 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface PushManifest { - requestId: string; - parentManifestId: number; - entries: Record; -} +export type PushManifest = { requestId: string, parentManifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/SerializedError.ts b/frontend/sync-client/src/services/types/SerializedError.ts index ec1c4503..5e3fa9b9 100644 --- a/frontend/sync-client/src/services/types/SerializedError.ts +++ b/frontend/sync-client/src/services/types/SerializedError.ts @@ -1,7 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface SerializedError { - errorType: string; - message: string; - causes: string[]; -} +export type SerializedError = { errorType: string, message: string, causes: Array, }; diff --git a/frontend/sync-client/src/services/types/VaultEvent.ts b/frontend/sync-client/src/services/types/VaultEvent.ts index 426efd5b..a96d5a2d 100644 --- a/frontend/sync-client/src/services/types/VaultEvent.ts +++ b/frontend/sync-client/src/services/types/VaultEvent.ts @@ -2,6 +2,4 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; import type { Manifest } from "./Manifest"; -export type VaultEvent = - | { type: "content"; document: DocumentVersionWithoutContent } - | { type: "manifest"; manifest: Manifest }; +export type VaultEvent = { "type": "content", document: DocumentVersionWithoutContent, } | { "type": "manifest", manifest: Manifest, }; diff --git a/frontend/sync-client/src/services/types/VaultSnapshot.ts b/frontend/sync-client/src/services/types/VaultSnapshot.ts index 3c0ff9a7..95a192f9 100644 --- a/frontend/sync-client/src/services/types/VaultSnapshot.ts +++ b/frontend/sync-client/src/services/types/VaultSnapshot.ts @@ -2,8 +2,4 @@ import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; import type { Manifest } from "./Manifest"; -export interface VaultSnapshot { - headEventId: number; - manifest: Manifest; - documents: DocumentVersionWithoutContent[]; -} +export type VaultSnapshot = { headEventId: number, manifest: Manifest, documents: Array, }; diff --git a/frontend/sync-client/src/services/types/WebSocketClientMessage.ts b/frontend/sync-client/src/services/types/WebSocketClientMessage.ts index 9608f3af..5765a0d0 100644 --- a/frontend/sync-client/src/services/types/WebSocketClientMessage.ts +++ b/frontend/sync-client/src/services/types/WebSocketClientMessage.ts @@ -2,6 +2,4 @@ import type { CursorPositionFromClient } from "./CursorPositionFromClient"; import type { WebSocketHandshake } from "./WebSocketHandshake"; -export type WebSocketClientMessage = - | ({ type: "handshake" } & WebSocketHandshake) - | ({ type: "cursorPositions" } & CursorPositionFromClient); +export type WebSocketClientMessage = { "type": "handshake" } & WebSocketHandshake | { "type": "cursorPositions" } & CursorPositionFromClient; diff --git a/frontend/sync-client/src/services/types/WebSocketHandshake.ts b/frontend/sync-client/src/services/types/WebSocketHandshake.ts index a2910f49..85c2cf0d 100644 --- a/frontend/sync-client/src/services/types/WebSocketHandshake.ts +++ b/frontend/sync-client/src/services/types/WebSocketHandshake.ts @@ -1,7 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export interface WebSocketHandshake { - token: string; - deviceId: string; - lastSeenVaultUpdateId: number | null; -} +export type WebSocketHandshake = { token: string, deviceId: string, lastSeenVaultUpdateId: number | null, }; diff --git a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts index d1d10a8f..2a7a7813 100644 --- a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts +++ b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts @@ -2,6 +2,4 @@ import type { CursorPositionFromServer } from "./CursorPositionFromServer"; import type { EventBatch } from "./EventBatch"; -export type WebSocketServerMessage = - | ({ type: "vaultEvents" } & EventBatch) - | ({ type: "cursorPositions" } & CursorPositionFromServer); +export type WebSocketServerMessage = { "type": "vaultEvents" } & EventBatch | { "type": "cursorPositions" } & CursorPositionFromServer; -- 2.54.0 From 5d533f6da9f818376e16547696acc87eebbc27d5 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:52:45 +0100 Subject: [PATCH 09/48] Extract endpoints and wire up new ones --- sync-server/src/server.rs | 29 ++++++++----------- .../server/endpoints/method_not_allowed.rs | 9 ++++++ sync-server/src/server/endpoints/not_found.rs | 9 ++++++ 3 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 sync-server/src/server/endpoints/method_not_allowed.rs create mode 100644 sync-server/src/server/endpoints/not_found.rs diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index b8304a6c..ecc01de9 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -1,17 +1,16 @@ pub mod auth; mod device_id_header; mod endpoints; -mod requests; -mod responses; +pub(crate) mod requests; +pub(crate) mod responses; -use anyhow::{Context as _, Result, anyhow}; +use anyhow::{Context as _, Result}; use auth::auth_middleware; use axum::{ Router, extract::{DefaultBodyLimit, Request}, http::{self, HeaderValue, Method}, middleware, - response::IntoResponse, routing::{IntoMakeService, get}, }; use device_id_header::DEVICE_ID_HEADER_NAME; @@ -32,7 +31,6 @@ use tracing::{Level, info_span}; use crate::{ app_state::AppState, config::{Config, server_config::ServerConfig}, - errors::{client_error, not_found_error}, }; pub async fn create_server(config: Config) -> Result<()> { @@ -85,8 +83,8 @@ pub async fn create_server(config: Config) -> Result<()> { .on_failure(DefaultOnFailure::new().level(Level::ERROR)), ) .with_state(app_state) - .fallback(handle_404) - .fallback(handle_405) + .fallback(endpoints::not_found::not_found) + .method_not_allowed_fallback(endpoints::method_not_allowed::method_not_allowed) .into_make_service(); start_server(app, &server_config).await @@ -95,9 +93,14 @@ pub async fn create_server(config: Config) -> Result<()> { fn get_authed_routes(app_state: AppState) -> Router { Router::new() .route( - "/vaults/:vault_id/documents", - get(endpoints::fetch_latest_documents::fetch_latest_documents), + "/vaults/:vault_id/manifest", + get(endpoints::get_manifest::get_manifest).put(endpoints::put_manifest::put_manifest), ) + .route( + "/vaults/:vault_id/state", + get(endpoints::snapshot::snapshot), + ) + .route("/vaults/:vault_id/events", get(endpoints::events::events)) .route( "/vaults/:vault_id/documents/:document_id", get(endpoints::fetch_latest_document_version::fetch_latest_document_version) @@ -153,11 +156,3 @@ async fn shutdown_signal() { () = terminate => {}, } } - -async fn handle_404() -> impl IntoResponse { - not_found_error(anyhow!("Page not found")) -} - -async fn handle_405() -> impl IntoResponse { - client_error(anyhow!("Method not allowed")) -} diff --git a/sync-server/src/server/endpoints/method_not_allowed.rs b/sync-server/src/server/endpoints/method_not_allowed.rs new file mode 100644 index 00000000..65d9c8c9 --- /dev/null +++ b/sync-server/src/server/endpoints/method_not_allowed.rs @@ -0,0 +1,9 @@ +use anyhow::anyhow; +use axum::response::IntoResponse; + +use crate::errors::client_error; + +#[axum::debug_handler] +pub async fn method_not_allowed() -> impl IntoResponse { + client_error(anyhow!("Method not allowed")) +} diff --git a/sync-server/src/server/endpoints/not_found.rs b/sync-server/src/server/endpoints/not_found.rs new file mode 100644 index 00000000..7c760fa9 --- /dev/null +++ b/sync-server/src/server/endpoints/not_found.rs @@ -0,0 +1,9 @@ +use anyhow::anyhow; +use axum::response::IntoResponse; + +use crate::errors::not_found_error; + +#[axum::debug_handler] +pub async fn not_found() -> impl IntoResponse { + not_found_error(anyhow!("Page not found")) +} -- 2.54.0 From 945984adfa8852112146c7a1e9ab90de26b5e1ea Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 15:53:30 +0100 Subject: [PATCH 10/48] Update deps --- sync-server/Cargo.lock | 12 ++---------- sync-server/Cargo.toml | 5 +++-- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/sync-server/Cargo.lock b/sync-server/Cargo.lock index b3da1486..810b4b93 100644 --- a/sync-server/Cargo.lock +++ b/sync-server/Cargo.lock @@ -1679,15 +1679,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" -[[package]] -name = "sanitize-filename" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" -dependencies = [ - "regex", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2139,10 +2130,10 @@ dependencies = [ "rand 0.9.0", "reconcile-text", "regex", - "sanitize-filename", "serde", "serde_json", "serde_yaml", + "sha2", "sqlx", "thiserror 2.0.17", "tokio", @@ -2150,6 +2141,7 @@ dependencies = [ "tracing", "tracing-subscriber", "ts-rs", + "unicode-normalization", "uuid", ] diff --git a/sync-server/Cargo.toml b/sync-server/Cargo.toml index fac06efa..a333545b 100644 --- a/sync-server/Cargo.toml +++ b/sync-server/Cargo.toml @@ -24,14 +24,15 @@ humantime-serde = "1.1.1" sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] } chrono = { version = "0.4.41", features = ["serde"] } rand = "0.9.0" -sanitize-filename = "0.6.0" regex = "1.12.2" clap = { version = "4.5.38", features = ["derive"] } futures = "0.3.31" serde_yaml = "0.9.34" serde_json = "1.0.140" bimap = "0.6.3" -ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] } +sha2 = "0.10" +unicode-normalization = "0.1" +ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl", "no-serde-warnings"] } base64 = "0.22.1" reconcile-text = { version = "0.8.0", features = ["serde"] } -- 2.54.0 From ce5f3bbd7d749b51e34ee723714556b0661a74e8 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 17:05:50 +0100 Subject: [PATCH 11/48] Add new data model --- .../migrations/20241207143519_bootstrap.sql | 46 +++--- sync-server/src/app_state/database/models.rs | 140 +++++++++++------- .../src/app_state/database/mutations.rs | 68 +++++++++ sync-server/src/app_state/database/queries.rs | 140 ++++++++++++++++++ 4 files changed, 320 insertions(+), 74 deletions(-) create mode 100644 sync-server/src/app_state/database/mutations.rs create mode 100644 sync-server/src/app_state/database/queries.rs diff --git a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql index 0b3076c6..504f89ba 100644 --- a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql +++ b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql @@ -1,30 +1,34 @@ -CREATE TABLE IF NOT EXISTS documents ( - vault_update_id INTEGER NOT NULL PRIMARY KEY, +CREATE TABLE events ( + event_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(event_id <= 9007199254740991), -- JavaScript’s Number.MAX_SAFE_INTEGER + request_id TEXT NOT NULL UNIQUE, + request_fingerprint BLOB NOT NULL, + event_json TEXT NOT NULL +); + +CREATE TABLE documents ( + vault_update_id INTEGER PRIMARY KEY REFERENCES events(event_id), document_id TEXT NOT NULL, - relative_path TEXT NOT NULL, - updated_date TIMESTAMP NOT NULL, + updated_date TEXT NOT NULL, content BLOB NOT NULL, - is_deleted BOOLEAN NOT NULL, user_id TEXT NOT NULL, device_id TEXT NOT NULL ); -CREATE VIEW IF NOT EXISTS latest_document_versions AS -SELECT d.* -FROM documents d -INNER JOIN ( - SELECT MAX(vault_update_id) AS max_version_id - FROM documents - GROUP BY document_id -) max_versions -ON d.vault_update_id = max_versions.max_version_id; +CREATE INDEX documents_by_id ON documents(document_id); -CREATE INDEX IF NOT EXISTS idx_documents_vault_id_relative_path -ON documents (relative_path); +CREATE VIEW latest_document_versions AS + SELECT d.* FROM documents d JOIN ( + SELECT document_id, MAX(vault_update_id) AS version FROM documents GROUP BY document_id + ) heads ON d.vault_update_id = heads.version; --- The immutable version and its acknowledgement commit in one transaction. -CREATE TABLE push_acknowledgements ( - request_id TEXT NOT NULL PRIMARY KEY, - request_fingerprint BLOB NOT NULL, - vault_update_id INTEGER NOT NULL REFERENCES documents(vault_update_id) +CREATE TABLE file_manifests ( + file_manifest_id INTEGER PRIMARY KEY REFERENCES events(event_id) +); + +CREATE TABLE file_manifest_entries ( + file_manifest_id INTEGER NOT NULL REFERENCES file_manifests(file_manifest_id), + document_id TEXT NOT NULL, + path TEXT NOT NULL, + PRIMARY KEY (file_manifest_id, document_id), + UNIQUE (file_manifest_id, path) ); diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index 24c0c370..81cb6343 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -1,91 +1,125 @@ use base64::{Engine as _, engine::general_purpose::STANDARD}; use chrono::{DateTime, Utc}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use ts_rs::TS; pub type VaultId = String; pub type VaultUpdateId = i64; - pub type DocumentId = uuid::Uuid; pub type UserId = String; pub type DeviceId = String; +pub type FileManifestEntries = BTreeMap; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, sqlx::FromRow)] pub struct StoredDocumentVersion { pub vault_update_id: VaultUpdateId, + #[sqlx(try_from = "uuid::fmt::Hyphenated")] pub document_id: DocumentId, - pub relative_path: String, pub updated_date: DateTime, pub content: Vec, - pub is_deleted: bool, pub user_id: UserId, pub device_id: DeviceId, } -impl PartialEq for StoredDocumentVersion { - fn eq(&self, other: &Self) -> bool { - self.vault_update_id == other.vault_update_id - } -} - -#[derive(TS, Debug, Clone, Serialize)] +#[derive(TS, Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] +#[ts(export)] pub struct DocumentVersionWithoutContent { - #[ts(as = "i32")] + #[ts(as = "f64")] pub vault_update_id: VaultUpdateId, - pub document_id: DocumentId, - pub relative_path: String, pub updated_date: DateTime, - pub is_deleted: bool, pub user_id: UserId, pub device_id: DeviceId, - - #[ts(as = "i32")] - pub content_size: u64, + pub content_size: usize, } -impl From for DocumentVersionWithoutContent { - fn from(value: StoredDocumentVersion) -> Self { +impl From<&StoredDocumentVersion> for DocumentVersionWithoutContent { + fn from(v: &StoredDocumentVersion) -> Self { Self { - vault_update_id: value.vault_update_id, - document_id: value.document_id, - relative_path: value.relative_path, - updated_date: value.updated_date, - is_deleted: value.is_deleted, - user_id: value.user_id, - device_id: value.device_id, - content_size: value.content.len() as u64, + vault_update_id: v.vault_update_id, + document_id: v.document_id, + updated_date: v.updated_date, + user_id: v.user_id.clone(), + device_id: v.device_id.clone(), + content_size: v.content.len(), } } } +impl From for DocumentVersionWithoutContent { + fn from(v: StoredDocumentVersion) -> Self { + Self::from(&v) + } +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct DocumentVersion { + #[serde(flatten)] + pub metadata: DocumentVersionWithoutContent, + pub content_base64: String, +} +impl From for DocumentVersion { + fn from(v: StoredDocumentVersion) -> Self { + Self { + metadata: (&v).into(), + content_base64: STANDARD.encode(&v.content), + } + } +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct FileManifest { + #[ts(as = "f64")] + pub file_manifest_id: VaultUpdateId, + #[ts(type = "Record")] + pub entries: FileManifestEntries, +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(export)] +pub enum VaultEvent { + Content { + document: DocumentVersionWithoutContent, + }, + FileManifest { + #[serde(rename = "fileManifest")] + #[ts(rename = "fileManifest")] + file_manifest: FileManifest, + }, +} + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct EventRecord { + #[ts(as = "f64")] + pub event_id: VaultUpdateId, + pub request_id: uuid::Uuid, + #[serde(flatten)] + pub event: VaultEvent, +} #[derive(TS, Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct DocumentVersion { - #[ts(as = "i32")] - pub vault_update_id: VaultUpdateId, - - pub document_id: DocumentId, - pub relative_path: String, - pub updated_date: DateTime, - pub content_base64: String, - pub is_deleted: bool, - pub user_id: UserId, - pub device_id: DeviceId, +#[ts(export)] +pub struct VaultSnapshot { + #[ts(as = "f64")] + pub head_event_id: VaultUpdateId, + pub file_manifest: FileManifest, + pub documents: Vec, } -impl From for DocumentVersion { - fn from(value: StoredDocumentVersion) -> Self { - Self { - vault_update_id: value.vault_update_id, - document_id: value.document_id, - relative_path: value.relative_path, - updated_date: value.updated_date, - content_base64: STANDARD.encode(&value.content), - is_deleted: value.is_deleted, - user_id: value.user_id, - device_id: value.device_id, - } - } +#[derive(TS, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[ts(export)] +pub struct EventBatch { + #[ts(as = "f64")] + pub head_event_id: VaultUpdateId, + pub events: Vec, } diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs new file mode 100644 index 00000000..59383789 --- /dev/null +++ b/sync-server/src/app_state/database/mutations.rs @@ -0,0 +1,68 @@ +use super::{Database, Transaction, models::*}; +use anyhow::Result; + +impl Database { + pub async fn allocate_event( + tx: &mut Transaction<'_>, + request_id: uuid::Uuid, + fingerprint: &[u8], + ) -> Result { + Ok(sqlx::query( + "INSERT INTO events(request_id, request_fingerprint, event_json) VALUES (?, ?, '')", + ) + .bind(request_id.to_string()) + .bind(fingerprint) + .execute(&mut **tx) + .await? + .last_insert_rowid()) + } + + pub async fn insert_document_version( + tx: &mut Transaction<'_>, + version: &StoredDocumentVersion, + ) -> Result<()> { + sqlx::query("INSERT INTO documents(vault_update_id, document_id, updated_date, content, user_id, device_id) VALUES (?, ?, ?, ?, ?, ?)") + .bind(version.vault_update_id) + .bind(version.document_id.hyphenated()) + .bind(version.updated_date) + .bind(&version.content) + .bind(&version.user_id) + .bind(&version.device_id) + .execute(&mut **tx) + .await?; + + Ok(()) + } + + pub async fn insert_file_manifest( + tx: &mut Transaction<'_>, + file_manifest: &FileManifest, + ) -> Result<()> { + sqlx::query("INSERT INTO file_manifests(file_manifest_id) VALUES (?)") + .bind(file_manifest.file_manifest_id) + .execute(&mut **tx) + .await?; + + for (document_id, path) in &file_manifest.entries { + sqlx::query( + "INSERT INTO file_manifest_entries(file_manifest_id, document_id, path) VALUES (?, ?, ?)", + ) + .bind(file_manifest.file_manifest_id) + .bind(document_id.hyphenated()) + .bind(path) + .execute(&mut **tx) + .await?; + } + + Ok(()) + } + + pub async fn write_event(tx: &mut Transaction<'_>, event: &EventRecord) -> Result<()> { + sqlx::query("UPDATE events SET event_json = ? WHERE event_id = ?") + .bind(serde_json::to_string(event)?) + .bind(event.event_id) + .execute(&mut **tx) + .await?; + Ok(()) + } +} diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs new file mode 100644 index 00000000..edc77801 --- /dev/null +++ b/sync-server/src/app_state/database/queries.rs @@ -0,0 +1,140 @@ +use super::{Database, Transaction, models::*}; +use anyhow::{Result, ensure}; + +impl Database { + pub async fn get_request_event( + tx: &mut Transaction<'_>, + request_id: uuid::Uuid, + ) -> Result, String)>> { + Ok(sqlx::query_as( + "SELECT request_fingerprint, event_json FROM events WHERE request_id = ?", + ) + .bind(request_id.to_string()) + .fetch_optional(&mut **tx) + .await?) + } + + pub async fn get_latest_document_version( + &self, + vault: &VaultId, + id: &DocumentId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query_as::<_, StoredDocumentVersion>( + "SELECT * FROM documents WHERE document_id = ? ORDER BY vault_update_id DESC LIMIT 1", + ) + .bind(id.hyphenated()); + + Ok(if let Some(tx) = transaction { + query.fetch_optional(&mut **tx).await? + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await? + }) + } + + pub async fn get_document_version( + &self, + vault: &VaultId, + version: VaultUpdateId, + transaction: Option<&mut Transaction<'_>>, + ) -> Result> { + let query = sqlx::query_as::<_, StoredDocumentVersion>( + "SELECT * FROM documents WHERE vault_update_id = ?", + ) + .bind(version); + + Ok(if let Some(tx) = transaction { + query.fetch_optional(&mut **tx).await? + } else { + query + .fetch_optional(&self.get_connection_pool(vault).await?) + .await? + }) + } + + pub async fn current_file_manifest(tx: &mut Transaction<'_>) -> Result { + let Some(file_manifest_id) = sqlx::query_scalar::<_, VaultUpdateId>( + "SELECT file_manifest_id FROM file_manifests ORDER BY file_manifest_id DESC LIMIT 1", + ) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(FileManifest::default()); + }; + + let entries = sqlx::query_as::<_, (uuid::fmt::Hyphenated, String)>( + "SELECT document_id, path FROM file_manifest_entries WHERE file_manifest_id = ?", + ) + .bind(file_manifest_id) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|(document_id, path)| (document_id.into_uuid(), path)) + .collect(); + + Ok(FileManifest { + file_manifest_id, + entries, + }) + } + + pub async fn latest_event_id(tx: &mut Transaction<'_>) -> Result { + Ok( + sqlx::query_scalar("SELECT COALESCE(MAX(event_id), 0) FROM events") + .fetch_one(&mut **tx) + .await?, + ) + } + + pub async fn vault_snapshot(&self, vault: &VaultId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let head_event_id = Self::latest_event_id(&mut tx).await?; + let file_manifest = Self::current_file_manifest(&mut tx).await?; + + let versions = sqlx::query_as::<_, StoredDocumentVersion>( + "SELECT d.* FROM latest_document_versions d + JOIN file_manifest_entries m ON m.document_id = d.document_id + WHERE m.file_manifest_id = ? ORDER BY d.vault_update_id", + ) + .bind(file_manifest.file_manifest_id) + .fetch_all(&mut *tx) + .await?; + + let documents = versions.into_iter().map(Into::into).collect(); + + Ok(VaultSnapshot { + head_event_id, + file_manifest, + documents, + }) + } + + pub async fn events_after(&self, vault: &VaultId, after: VaultUpdateId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let head_event_id = Self::latest_event_id(&mut tx).await?; + + ensure!( + (0..=head_event_id).contains(&after), + "Event cursor is outside this vault's history" + ); + + let rows: Vec = sqlx::query_scalar( + "SELECT event_json FROM events WHERE event_id > ? ORDER BY event_id", + ) + .bind(after) + .fetch_all(&mut *tx) + .await?; + + let events = rows + .iter() + .map(|json| serde_json::from_str(json)) + .collect::>()?; + + Ok(EventBatch { + head_event_id, + events, + }) + } +} -- 2.54.0 From 69a1e91d4ba8a723da1f7c68a95505e5672a3f21 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 17:06:31 +0100 Subject: [PATCH 12/48] Add new endpoints --- sync-server/src/app_state/cursors.rs | 17 +- sync-server/src/app_state/database.rs | 20 +- .../src/app_state/websocket/broadcasts.rs | 84 +-- sync-server/src/app_state/websocket/models.rs | 35 +- sync-server/src/app_state/websocket/utils.rs | 25 +- sync-server/src/server/endpoints.rs | 10 +- sync-server/src/server/endpoints/events.rs | 49 ++ .../fetch_latest_document_version.rs | 2 +- .../src/server/endpoints/get_file_manifest.rs | 43 ++ sync-server/src/server/endpoints/index.rs | 1 + .../src/server/endpoints/push_document.rs | 210 ------ .../src/server/endpoints/put_file_content.rs | 624 ++++++++++++++++++ .../src/server/endpoints/put_file_manifest.rs | 122 ++++ sync-server/src/server/endpoints/utils.rs | 32 + .../src/server/endpoints/vault_snapshot.rs | 36 + sync-server/src/server/endpoints/websocket.rs | 238 +++---- sync-server/src/server/requests.rs | 142 +++- sync-server/src/server/responses.rs | 20 +- 18 files changed, 1199 insertions(+), 511 deletions(-) create mode 100644 sync-server/src/server/endpoints/events.rs create mode 100644 sync-server/src/server/endpoints/get_file_manifest.rs delete mode 100644 sync-server/src/server/endpoints/push_document.rs create mode 100644 sync-server/src/server/endpoints/put_file_content.rs create mode 100644 sync-server/src/server/endpoints/put_file_manifest.rs create mode 100644 sync-server/src/server/endpoints/utils.rs create mode 100644 sync-server/src/server/endpoints/vault_snapshot.rs diff --git a/sync-server/src/app_state/cursors.rs b/sync-server/src/app_state/cursors.rs index d083e1ac..2c57a49d 100644 --- a/sync-server/src/app_state/cursors.rs +++ b/sync-server/src/app_state/cursors.rs @@ -6,11 +6,8 @@ use tokio::sync::Mutex; use super::{ database::models::{DeviceId, VaultId}, websocket::{ - broadcasts::Broadcasts, - models::{ - ClientCursors, CursorPositionFromServer, WebSocketServerMessage, - WebSocketServerMessageWithOrigin, - }, + broadcasts::{Broadcasts, Notification}, + models::{ClientCursors, CursorPositionFromServer}, }, }; use crate::{ @@ -91,13 +88,11 @@ impl Cursors { for (vault_id, cursors) in vault_to_cursors.iter() { self.broadcasts - .send_document_update( + .send( vault_id.clone(), - WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions( - CursorPositionFromServer { - clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), - }, - )), + Notification::Cursors(CursorPositionFromServer { + clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), + }), ) .await; } diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 216a537d..3bd5e9c0 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -11,16 +11,9 @@ use tokio::time::Instant; pub mod models; -mod get_document_version; -mod get_latest_document; -mod get_latest_document_by_path; -mod get_latest_documents; -mod get_latest_documents_since; -mod get_max_update_id_in_vault; -mod get_push_acknowledgement; -mod insert_document_version; +mod mutations; +mod queries; -use super::websocket::broadcasts::Broadcasts; use crate::config::database_config::DatabaseConfig; #[derive(Clone)] @@ -41,14 +34,13 @@ impl std::fmt::Debug for PoolWithTimestamp { #[derive(Clone, Debug)] pub struct Database { config: DatabaseConfig, - broadcasts: Broadcasts, connection_pools: Arc>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; impl Database { - pub async fn try_new(config: &DatabaseConfig, broadcasts: &Broadcasts) -> Result { + pub async fn try_new(config: &DatabaseConfig) -> Result { tokio::fs::create_dir_all(&config.databases_directory_path) .await .with_context(|| { @@ -86,7 +78,6 @@ impl Database { let database = Self { config: config.clone(), connection_pools: Arc::new(Mutex::new(connection_pools)), - broadcasts: broadcasts.clone(), }; // Start background task to cleanup idle connection pools @@ -109,6 +100,9 @@ impl Database { .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Full) .busy_timeout(Duration::from_secs(3600)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .synchronous(sqlx::sqlite::SqliteSynchronous::Full) + .pragma("fullfsync", "ON") + .foreign_keys(true) .log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30)); let pool = SqlitePoolOptions::new() @@ -172,7 +166,7 @@ impl Database { let mut transaction = self.create_readonly_transaction(vault).await?; // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 - sqlx::query!("END; BEGIN IMMEDIATE;") + sqlx::query("END; BEGIN IMMEDIATE;") .execute(&mut *transaction) .await?; diff --git a/sync-server/src/app_state/websocket/broadcasts.rs b/sync-server/src/app_state/websocket/broadcasts.rs index 60ae0219..1b157b13 100644 --- a/sync-server/src/app_state/websocket/broadcasts.rs +++ b/sync-server/src/app_state/websocket/broadcasts.rs @@ -1,69 +1,47 @@ +use super::models::CursorPositionFromServer; +use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig}; use std::{collections::HashMap, sync::Arc}; - -use anyhow::Context; -use log::{debug, warn}; use tokio::sync::{Mutex, broadcast}; -use super::models::WebSocketServerMessageWithOrigin; -use crate::{ - app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error, -}; +#[derive(Debug, Clone)] +pub enum Notification { + VaultUpdate, + Cursors(CursorPositionFromServer), +} #[derive(Debug, Clone)] pub struct Broadcasts { - max_clients_per_vault: usize, - tx: Arc>>>, + capacity: usize, + senders: Arc>>>, } impl Broadcasts { - pub fn new(server_config: &ServerConfig) -> Self { + pub fn new(config: &ServerConfig) -> Self { Self { - max_clients_per_vault: server_config.max_clients_per_vault, - tx: Arc::new(Mutex::new(HashMap::new())), + capacity: config.max_clients_per_vault.max(1), + senders: Arc::default(), } } - pub async fn get_receiver( - &self, - vault: VaultId, - ) -> broadcast::Receiver { - let tx = self.get_or_create(vault).await; - - tx.subscribe() - } - - /// Notify all clients (who are subscribed to the vault) about an update. - /// We only log failures and don't propagate them. - pub async fn send_document_update( - &self, - vault: VaultId, - document: WebSocketServerMessageWithOrigin, - ) { - let tx = self.get_or_create(vault.clone()).await; - - if tx.receiver_count() == 0 { - debug!("Skipping broadcast, no clients connected for vault `{vault}`"); - return; - } - - let result = tx - .send(document) - .context("Cannot broadcast server message to websocket listeners") - .map_err(server_error); - - if result.is_err() { - warn!("Failed to send message: {result:?}"); - } - } - - async fn get_or_create( - &self, - vault: VaultId, - ) -> broadcast::Sender { - let mut tx = self.tx.lock().await; - - tx.entry(vault) - .or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone()) + async fn sender(&self, vault: VaultId) -> broadcast::Sender { + self.senders + .lock() + .await + .entry(vault) + .or_insert_with(|| broadcast::channel(self.capacity).0) .clone() } + + pub async fn get_receiver(&self, vault: VaultId) -> broadcast::Receiver { + self.sender(vault).await.subscribe() + } + + pub async fn send(&self, vault: VaultId, notification: Notification) { + // No listeners (or lagging listeners) is normal; they replay the log. + let _ = self.sender(vault).await.send(notification); + } + + pub async fn notify_about_vault_update(&self, vault: VaultId) { + self.send(vault, Notification::VaultUpdate).await; + } } diff --git a/sync-server/src/app_state/websocket/models.rs b/sync-server/src/app_state/websocket/models.rs index e037fb7e..32d97966 100644 --- a/sync-server/src/app_state/websocket/models.rs +++ b/sync-server/src/app_state/websocket/models.rs @@ -1,9 +1,7 @@ use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::app_state::database::models::{ - DeviceId, DocumentId, DocumentVersionWithoutContent, VaultUpdateId, -}; +use crate::app_state::database::models::{DeviceId, DocumentId, EventBatch, VaultUpdateId}; #[derive(TS, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] @@ -57,13 +55,6 @@ pub struct CursorPositionFromServer { pub clients: Vec, } -#[derive(TS, Serialize, Clone, Debug)] -#[serde(rename_all = "camelCase")] -pub struct WebSocketVaultUpdate { - pub documents: Vec, - pub is_initial_sync: bool, -} - #[derive(TS, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase", tag = "type")] #[ts(export)] @@ -76,28 +67,6 @@ pub enum WebSocketClientMessage { #[serde(rename_all = "camelCase", tag = "type")] #[ts(export)] pub enum WebSocketServerMessage { - VaultUpdate(WebSocketVaultUpdate), + VaultEvents(EventBatch), CursorPositions(CursorPositionFromServer), } - -#[derive(Clone, Debug)] -pub struct WebSocketServerMessageWithOrigin { - pub origin_device_id: Option, - pub message: WebSocketServerMessage, -} - -impl WebSocketServerMessageWithOrigin { - pub fn new(message: WebSocketServerMessage) -> Self { - Self { - origin_device_id: None, - message, - } - } - - pub fn with_origin(origin_device_id: DeviceId, message: WebSocketServerMessage) -> Self { - Self { - origin_device_id: Some(origin_device_id), - message, - } - } -} diff --git a/sync-server/src/app_state/websocket/utils.rs b/sync-server/src/app_state/websocket/utils.rs index 1e0dd243..e72a3775 100644 --- a/sync-server/src/app_state/websocket/utils.rs +++ b/sync-server/src/app_state/websocket/utils.rs @@ -4,10 +4,7 @@ use futures::{sink::SinkExt, stream::SplitSink}; use super::models::{WebSocketClientMessage, WebSocketHandshake, WebSocketServerMessage}; use crate::{ - app_state::{ - AppState, - database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId}, - }, + app_state::{AppState, database::models::VaultId}, config::user_config::User, errors::{SyncServerError, server_error, unauthenticated_error}, server::auth::auth, @@ -44,26 +41,6 @@ pub fn get_authenticated_handshake( } } -pub async fn get_unseen_documents( - state: &AppState, - vault_id: &VaultId, - last_seen_vault_update_id: Option, -) -> Result, SyncServerError> { - if let Some(update_id) = last_seen_vault_update_id { - state - .database - .get_latest_documents_since(vault_id, update_id, None) - .await - .map_err(server_error) - } else { - state - .database - .get_latest_documents(vault_id, None) - .await - .map_err(server_error) - } -} - pub async fn send_update_over_websocket( update: &WebSocketServerMessage, sender: &mut SplitSink, diff --git a/sync-server/src/server/endpoints.rs b/sync-server/src/server/endpoints.rs index 243cf7fa..ed873473 100644 --- a/sync-server/src/server/endpoints.rs +++ b/sync-server/src/server/endpoints.rs @@ -1,7 +1,13 @@ +pub mod events; pub mod fetch_document_version_content; pub mod fetch_latest_document_version; -pub mod fetch_latest_documents; +pub mod get_file_manifest; pub mod index; +pub mod method_not_allowed; +pub mod not_found; pub mod ping; -pub mod push_document; +pub mod put_file_content; +pub mod put_file_manifest; +mod utils; +pub mod vault_snapshot; pub mod websocket; diff --git a/sync-server/src/server/endpoints/events.rs b/sync-server/src/server/endpoints/events.rs new file mode 100644 index 00000000..ef113864 --- /dev/null +++ b/sync-server/src/server/endpoints/events.rs @@ -0,0 +1,49 @@ +use axum::{ + Json, + extract::{Path, Query, State}, +}; +use log::debug; +use serde::Deserialize; + +use crate::{ + app_state::{ + AppState, + database::models::{EventBatch, VaultId}, + }, + errors::{SyncServerError, client_error, server_error}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct EventsPath { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[derive(Deserialize)] +pub struct EventsQuery { + pub after: i64, +} + +#[axum::debug_handler] +pub async fn events( + Path(path): Path, + Query(query): Query, + State(state): State, +) -> Result, SyncServerError> { + debug!( + "Fetching events after `{}` for vault `{}`", + query.after, path.vault_id + ); + + if query.after < 0 { + return Err(client_error(anyhow::anyhow!("Invalid event cursor"))); + } + + state + .database + .events_after(&path.vault_id, query.after) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/endpoints/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs index a9973606..a6b662e9 100644 --- a/sync-server/src/server/endpoints/fetch_latest_document_version.rs +++ b/sync-server/src/server/endpoints/fetch_latest_document_version.rs @@ -35,7 +35,7 @@ pub async fn fetch_latest_document_version( let latest_version = state .database - .get_latest_document(&vault_id, &document_id, None) + .get_latest_document_version(&vault_id, &document_id, None) .await .map_err(server_error)? .map_or_else( diff --git a/sync-server/src/server/endpoints/get_file_manifest.rs b/sync-server/src/server/endpoints/get_file_manifest.rs new file mode 100644 index 00000000..cfafda02 --- /dev/null +++ b/sync-server/src/server/endpoints/get_file_manifest.rs @@ -0,0 +1,43 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use serde::Deserialize; + +use crate::{ + app_state::{ + AppState, + database::{ + Database, + models::{FileManifest, VaultId}, + }, + }, + errors::{SyncServerError, server_error}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct GetFileManifestPath { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[axum::debug_handler] +pub async fn get_file_manifest( + Path(path): Path, + State(state): State, +) -> Result, SyncServerError> { + debug!("Fetching file manifest for vault `{}`", path.vault_id); + + let mut transaction = state + .database + .create_readonly_transaction(&path.vault_id) + .await + .map_err(server_error)?; + + Database::current_file_manifest(&mut transaction) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/endpoints/index.rs b/sync-server/src/server/endpoints/index.rs index 982b0b88..f6ea022d 100644 --- a/sync-server/src/server/endpoints/index.rs +++ b/sync-server/src/server/endpoints/index.rs @@ -1,5 +1,6 @@ use axum::response::{Html, IntoResponse}; +#[axum::debug_handler] pub async fn index() -> impl IntoResponse { const HTML_CONTENT: &str = include_str!("../assets/index.html"); let html_content = HTML_CONTENT; diff --git a/sync-server/src/server/endpoints/push_document.rs b/sync-server/src/server/endpoints/push_document.rs deleted file mode 100644 index 676e8baa..00000000 --- a/sync-server/src/server/endpoints/push_document.rs +++ /dev/null @@ -1,210 +0,0 @@ -use anyhow::{Context as _, anyhow}; -use axum::{ - Extension, Json, - extract::{Path, State}, -}; -use axum_extra::TypedHeader; -use base64::{Engine as _, engine::general_purpose::STANDARD}; -use reconcile_text::{BuiltinTokenizer, EditedText, NumberOrText}; -use serde::Deserialize; -use sha2::{Digest, Sha256}; - -use crate::server::{ - device_id_header::DeviceIdHeader, - requests::{PushContent, PushDocument}, - responses::DocumentUpdateResponse, -}; -use crate::{ - app_state::{ - AppState, - database::{ - Database, - models::{DocumentId, StoredDocumentVersion, VaultId}, - }, - }, - config::user_config::User, - errors::{SyncServerError, client_error, not_found_error, server_error}, - utils::{ - find_first_available_path::find_first_available_path, normalize::normalize, - sanitize_path::sanitize_path, - }, -}; - -#[derive(Deserialize)] -pub struct PushDocumentPath { - #[serde(deserialize_with = "normalize")] - vault_id: VaultId, - - document_id: DocumentId, -} - -#[axum::debug_handler] -pub async fn push_document( - Path(PushDocumentPath { - vault_id, - document_id, - }): Path, - Extension(user): Extension, - TypedHeader(device_id): TypedHeader, - State(state): State, - Json(push): Json, -) -> Result, SyncServerError> { - accept_push( - &state.database, - &vault_id, - document_id, - user.name, - device_id.0, - push, - ) - .await - .map(Json) -} - -/// A push is a compare-and-swap of one document's immutable head. The write lock -/// covers idempotency lookup, base validation, path allocation, and the commit. -async fn accept_push( - database: &Database, - vault_id: &VaultId, - document_id: DocumentId, - user_id: String, - device_id: String, - push: PushDocument, -) -> Result { - let fingerprint = fingerprint(document_id, &push); - - let mut transaction = database - .create_write_transaction(vault_id) - .await - .map_err(server_error)?; - - if let Some((stored_fingerprint, acknowledged)) = database - .get_push_acknowledgement(vault_id, push.request_id, &mut transaction) - .await - .map_err(server_error)? - { - if stored_fingerprint != fingerprint { - return Err(client_error(anyhow!( - "Request ID was reused with a different payload" - ))); - } - return Ok(DocumentUpdateResponse::Accepted(acknowledged.into())); - } - - let latest = database - .get_latest_document(vault_id, &document_id, Some(&mut transaction)) - .await - .map_err(server_error)?; - - match &latest { - Some(latest) - if latest.is_deleted || Some(latest.vault_update_id) != push.parent_version_id => - { - return Ok(DocumentUpdateResponse::StaleBase(latest.clone().into())); - } - - None if push.parent_version_id.is_some() => { - return Err(not_found_error(anyhow!( - "Document with id `{document_id}` not found" - ))); - } - - _ => {} - } - - let is_deleted = matches!(&push.content, PushContent::Delete); - let (content, requested_path) = match push.content { - PushContent::Snapshot(content) => ( - STANDARD - .decode(content) - .context("Invalid base64 snapshot") - .map_err(client_error)?, - sanitize_path(&push.relative_path), - ), - PushContent::Diff(diff) => ( - reconstruct_diff(latest.as_ref(), diff)?, - sanitize_path(&push.relative_path), - ), - PushContent::Delete => { - let latest = latest - .as_ref() - .ok_or_else(|| client_error(anyhow!("Cannot delete a missing document")))?; - (latest.content.clone(), latest.relative_path.clone()) - } - }; - - let relative_path = if latest - .as_ref() - .is_some_and(|version| version.relative_path == requested_path) - { - requested_path - } else { - find_first_available_path(vault_id, &requested_path, database, &mut transaction) - .await - .map_err(server_error)? - }; - - let new_version = StoredDocumentVersion { - vault_update_id: database - .get_max_update_id_in_vault(vault_id, Some(&mut transaction)) - .await - .map_err(server_error)? - + 1, - document_id, - relative_path, - content, - updated_date: chrono::Utc::now(), - is_deleted, - user_id, - device_id, - }; - - database - .insert_document_version( - vault_id, - &new_version, - push.request_id, - &fingerprint, - transaction, - ) - .await - .map_err(server_error)?; - Ok(DocumentUpdateResponse::Accepted(new_version.into())) -} - -fn reconstruct_diff( - parent: Option<&StoredDocumentVersion>, - diff: Vec, -) -> Result, SyncServerError> { - // reconcile-text 0.8 negates deletion lengths; i64::MIN would overflow. - if diff - .iter() - .any(|item| matches!(item, NumberOrText::Number(i64::MIN))) - { - return Err(client_error(anyhow!("Invalid diff length"))); - } - let parent = parent.ok_or_else(|| client_error(anyhow!("A diff requires a parent version")))?; - let parent_text = str::from_utf8(&parent.content) - .context("Parent is not UTF-8 text") - .map_err(client_error)?; - // Transport decoding only: reconciliation belongs exclusively to clients. - Ok( - EditedText::from_diff(parent_text, diff, &*BuiltinTokenizer::Word) - .context("Failed to apply diff to parent document") - .map_err(client_error)? - .apply() - .text() - .into_bytes(), - ) -} - -/// Session/device IDs may change across retries; the requested mutation may not. -fn fingerprint(document_id: DocumentId, push: &PushDocument) -> Vec { - Sha256::digest( - serde_json::to_vec(&(document_id, push)).expect("Document pushes are serializable"), - ) - .to_vec() -} - -#[cfg(test)] -mod tests; diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs new file mode 100644 index 00000000..9d757bb8 --- /dev/null +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -0,0 +1,624 @@ +use super::utils::find_already_processed_event; +use crate::{ + app_state::{ + AppState, + database::{ + Database, + models::{DocumentId, EventRecord, StoredDocumentVersion, VaultEvent, VaultId}, + }, + }, + config::user_config::User, + errors::{SyncServerError, client_error, not_found_error, server_error}, + server::{ + device_id_header::DeviceIdHeader, + requests::{PushContent, PutFileContent}, + responses::DocumentUpdateResponse, + }, + utils::normalize::normalize, +}; +use anyhow::anyhow; +use axum::{ + Extension, Json, + extract::{Path, State}, +}; +use axum_extra::TypedHeader; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use log::debug; +use reconcile_text::{BuiltinTokenizer, EditedText, NumberOrText}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +#[derive(Deserialize)] +pub struct PutFileContentPath { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, + document_id: DocumentId, +} + +#[axum::debug_handler] +pub async fn put_file_content( + Path(path): Path, + Extension(user): Extension, + TypedHeader(device): TypedHeader, + State(state): State, + Json(push): Json, +) -> Result, SyncServerError> { + debug!( + "Pushing document `{}` in vault `{}`", + path.document_id, path.vault_id + ); + + let fingerprint = Sha256::digest( + serde_json::to_vec(&("content", path.document_id, &push)) + .map_err(|error| server_error(error.into()))?, + ); + + let mut tx = state + .database + .create_write_transaction(&path.vault_id) + .await + .map_err(server_error)?; + + if let Some(event) = + find_already_processed_event(&mut tx, push.request_id, &fingerprint).await? + { + return match event { + VaultEvent::Content { document } => { + Ok(Json(DocumentUpdateResponse::Accepted(document))) + } + VaultEvent::FileManifest { .. } => Err(server_error(anyhow!( + "Stored event does not match content request" + ))), + }; + } + + let parent = state + .database + .get_latest_document_version(&path.vault_id, &path.document_id, Some(&mut tx)) + .await + .map_err(server_error)?; + + if let Some(parent) = &parent { + if Some(parent.vault_update_id) != push.parent_version_id { + return Ok(Json(DocumentUpdateResponse::StaleBase( + parent.clone().into(), + ))); + } + } else if push.parent_version_id.is_some() { + return Err(not_found_error(anyhow!( + "Document {} does not exist", + path.document_id + ))); + } + + let content = match push.content { + PushContent::Snapshot(bytes) => STANDARD + .decode(bytes) + .map_err(|error| client_error(error.into()))?, + + PushContent::Diff(diff) => { + if diff + .iter() + .any(|n| matches!(n, NumberOrText::Number(i64::MIN))) + { + return Err(client_error(anyhow!("Invalid diff length"))); + } + + let parent = parent + .as_ref() + .ok_or_else(|| client_error(anyhow!("Diff requires a parent")))?; + + let text = + str::from_utf8(&parent.content).map_err(|error| client_error(error.into()))?; + + EditedText::from_diff(text, diff, &*BuiltinTokenizer::Word) + .map_err(|error| client_error(error.into()))? + .apply() + .text() + .into_bytes() + } + }; + + let version = StoredDocumentVersion { + vault_update_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) + .await + .map_err(server_error)?, + document_id: path.document_id, + content, + updated_date: chrono::Utc::now(), + user_id: user.name, + device_id: device.0, + }; + + Database::insert_document_version(&mut tx, &version) + .await + .map_err(server_error)?; + + let response = DocumentUpdateResponse::Accepted((&version).into()); + + let event = EventRecord { + event_id: version.vault_update_id, + request_id: push.request_id, + event: VaultEvent::Content { + document: version.into(), + }, + }; + + Database::write_event(&mut tx, &event) + .await + .map_err(server_error)?; + + tx.commit() + .await + .map_err(|error| server_error(error.into()))?; + + state + .broadcasts + .notify_about_vault_update(path.vault_id) + .await; + + Ok(Json(response)) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::{ + app_state::database::{ + Database, + models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultUpdateId}, + }, + config::{Config, database_config::DatabaseConfig, server_config::ServerConfig}, + }; + + async fn send_document( + state: &AppState, + vault: &VaultId, + document_id: DocumentId, + user_id: String, + device_id: String, + push: PutFileContent, + ) -> Result { + put_file_content( + Path(PutFileContentPath { + vault_id: vault.clone(), + document_id, + }), + Extension(User { + name: user_id, + token: String::new(), + vault_access: Default::default(), + }), + TypedHeader(DeviceIdHeader(device_id)), + State(state.clone()), + Json(push), + ) + .await + .map(|Json(response)| response) + } + + struct Push { + document_id: DocumentId, + request_id: uuid::Uuid, + parent_version_id: Option, + content: PushContent, + user_id: String, + device_id: String, + } + + impl Push { + async fn send( + self, + state: &AppState, + vault: &VaultId, + ) -> Result { + send_document( + state, + vault, + self.document_id, + self.user_id, + self.device_id, + PutFileContent { + request_id: self.request_id, + parent_version_id: self.parent_version_id, + content: self.content, + }, + ) + .await + } + } + + struct TestVault { + directory: PathBuf, + state: AppState, + vault: VaultId, + } + + impl TestVault { + async fn new() -> Self { + let directory = + std::env::temp_dir().join(format!("vaultlink-cas-{}", uuid::Uuid::new_v4())); + let state = Self::open(&directory).await; + Self { + directory, + state, + vault: "test".to_owned(), + } + } + + async fn open(directory: &std::path::Path) -> AppState { + AppState::try_new(Config { + database: DatabaseConfig { + databases_directory_path: directory.to_owned(), + ..Default::default() + }, + server: ServerConfig { + max_clients_per_vault: 16, + ..Default::default() + }, + ..Default::default() + }) + .await + .unwrap() + } + + async fn send(&self, push: Push) -> DocumentUpdateResponse { + push.send(&self.state, &self.vault).await.unwrap() + } + + async fn latest(&self, document: DocumentId) -> StoredDocumentVersion { + self.state + .database + .get_latest_document_version(&self.vault, &document, None) + .await + .unwrap() + .unwrap() + } + } + + impl Drop for TestVault { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.directory); + } + } + + fn snapshot(document_id: DocumentId, parent: Option, text: &str) -> Push { + Push { + document_id, + request_id: uuid::Uuid::new_v4(), + parent_version_id: parent, + content: PushContent::Snapshot(STANDARD.encode(text)), + user_id: "user".to_owned(), + device_id: "device".to_owned(), + } + } + + fn accepted(response: DocumentUpdateResponse) -> DocumentVersionWithoutContent { + match response { + DocumentUpdateResponse::Accepted(version) => version, + response @ DocumentUpdateResponse::StaleBase(_) => { + panic!("Expected Accepted, got {response:?}") + } + } + } + + #[tokio::test] + async fn json_snapshot_preserves_bytes_and_rejects_invalid_base64_without_a_version() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let content = [0, 255, 13, 10, 42]; + let request: PutFileContent = serde_json::from_value(serde_json::json!({ + "requestId": uuid::Uuid::new_v4(), + "parentVersionId": null, + "content": { "type": "Snapshot", "value": STANDARD.encode(content) }, + })) + .unwrap(); + let base = accepted( + send_document( + &vault.state, + &vault.vault, + document, + "user".to_owned(), + "device".to_owned(), + request, + ) + .await + .unwrap(), + ); + assert_eq!(vault.latest(document).await.content, content); + let mut tx = vault + .state + .database + .create_readonly_transaction(&vault.vault) + .await + .unwrap(); + let stored_document_id: String = + sqlx::query_scalar("SELECT document_id FROM documents WHERE vault_update_id = ?") + .bind(base.vault_update_id) + .fetch_one(&mut *tx) + .await + .unwrap(); + assert_eq!(stored_document_id, document.to_string()); + tx.commit().await.unwrap(); + assert!(serde_json::to_value(&base).unwrap().get("mtime").is_none()); + + let invalid: PutFileContent = serde_json::from_value(serde_json::json!({ + "requestId": uuid::Uuid::new_v4(), + "parentVersionId": base.vault_update_id, + "content": { "type": "Snapshot", "value": "not base64!" }, + })) + .unwrap(); + assert!(matches!( + send_document( + &vault.state, + &vault.vault, + document, + "user".to_owned(), + "device".to_owned(), + invalid, + ) + .await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(document).await.vault_update_id, + base.vault_update_id + ); + } + + #[tokio::test] + async fn concurrent_writers_only_accept_one_snapshot_and_preserve_the_base() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted(vault.send(snapshot(document, None, "base")).await); + let (left, right) = tokio::join!( + vault.send(snapshot(document, Some(base.vault_update_id), "left")), + vault.send(snapshot(document, Some(base.vault_update_id), "right")), + ); + let (winner, rejected) = match (left, right) { + ( + DocumentUpdateResponse::Accepted(winner), + DocumentUpdateResponse::StaleBase(rejected), + ) + | ( + DocumentUpdateResponse::StaleBase(rejected), + DocumentUpdateResponse::Accepted(winner), + ) => (winner, rejected), + outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"), + }; + let latest = vault.latest(document).await; + assert_eq!(winner.vault_update_id, rejected.metadata.vault_update_id); + assert_eq!( + STANDARD.decode(rejected.content_base64).unwrap(), + latest.content + ); + assert!(latest.content == b"left" || latest.content == b"right"); + let original = vault + .state + .database + .get_document_version(&vault.vault, base.vault_update_id, None) + .await + .unwrap() + .unwrap(); + assert_eq!(original.content, b"base"); + let mut transaction = vault + .state + .database + .create_readonly_transaction(&vault.vault) + .await + .unwrap(); + assert_eq!( + Database::latest_event_id(&mut transaction).await.unwrap(), + 2 + ); + } + + #[tokio::test] + async fn acknowledgements_survive_reopening_and_return_the_original_version() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let create = snapshot(document, None, "created"); + let create_id = create.request_id; + let base = accepted(vault.send(create).await); + let update = snapshot(document, Some(base.vault_update_id), "updated"); + let update_id = update.request_id; + let updated = accepted(vault.send(update).await); + let latest = accepted( + vault + .send(snapshot(document, Some(updated.vault_update_id), "newest")) + .await, + ); + + let reopened = TestVault::open(&vault.directory).await; + for (request_id, parent, text, original) in [ + (create_id, None, "created", base), + (update_id, Some(1), "updated", updated), + ] { + let mut retry = snapshot(document, parent, text); + retry.request_id = request_id; + retry.device_id = "new-session-after-restart".to_owned(); + let acknowledged = accepted(retry.send(&reopened, &vault.vault).await.unwrap()); + assert_eq!( + serde_json::to_value(acknowledged).unwrap(), + serde_json::to_value(original).unwrap() + ); + } + assert_eq!( + vault.latest(document).await.vault_update_id, + latest.vault_update_id + ); + + let mut reused = snapshot(uuid::Uuid::new_v4(), None, "wrong document"); + reused.request_id = create_id; + assert!(matches!( + reused.send(&reopened, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + + let mut changed_operation = snapshot(document, None, "created"); + changed_operation.content = PushContent::Diff(vec![]); + for mut changed_payload in [ + snapshot(document, None, "different content"), + snapshot(document, Some(1), "created"), + changed_operation, + ] { + changed_payload.request_id = create_id; + assert!(matches!( + changed_payload.send(&reopened, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + } + let events = reopened + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.head_event_id, latest.vault_update_id); + assert_eq!(events.events.len(), 3); + assert_eq!(events.events[0].request_id, create_id); + assert_eq!(events.events[1].request_id, update_id); + } + + #[tokio::test] + async fn concurrent_retries_return_the_same_acknowledgement_and_append_one_event() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let original = snapshot(document, None, "created"); + let request_id = original.request_id; + let mut retry = snapshot(document, None, "created"); + retry.request_id = request_id; + retry.device_id = "reconnected-device".to_owned(); + + let (left, right) = tokio::join!(vault.send(original), vault.send(retry)); + assert_eq!( + serde_json::to_value(accepted(left)).unwrap(), + serde_json::to_value(accepted(right)).unwrap() + ); + let events = vault + .state + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.head_event_id, 1); + assert_eq!(events.events.len(), 1); + assert_eq!(events.events[0].request_id, request_id); + } + + #[tokio::test] + async fn failed_event_commit_rolls_back_content_and_allows_retrying_the_request() { + let vault = TestVault::new().await; + let mut tx = vault + .state + .database + .create_write_transaction(&vault.vault) + .await + .unwrap(); + sqlx::query("CREATE TRIGGER reject_event BEFORE UPDATE ON events BEGIN SELECT RAISE(ABORT, 'simulated event write failure'); END") + .execute(&mut *tx).await.unwrap(); + tx.commit().await.unwrap(); + + let document = uuid::Uuid::new_v4(); + let request = snapshot(document, None, "created"); + let request_id = request.request_id; + assert!(request.send(&vault.state, &vault.vault).await.is_err()); + assert!( + vault + .state + .database + .get_latest_document_version(&vault.vault, &document, None) + .await + .unwrap() + .is_none() + ); + let events = vault + .state + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.head_event_id, 0); + assert!(events.events.is_empty()); + + let mut tx = vault + .state + .database + .create_write_transaction(&vault.vault) + .await + .unwrap(); + sqlx::query("DROP TRIGGER reject_event") + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + let mut retry = snapshot(document, None, "created"); + retry.request_id = request_id; + let version = accepted(vault.send(retry).await); + assert_eq!(version.vault_update_id, 1); + let events = vault + .state + .database + .events_after(&vault.vault, 0) + .await + .unwrap(); + assert_eq!(events.events.len(), 1); + assert_eq!(events.events[0].request_id, request_id); + } + + #[tokio::test] + async fn transport_diffs_reconstruct_exact_snapshots_and_reject_invalid_bases() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let original = "Hello world 🌍"; + let desired = "Hi beautiful world 🌍"; + let base = accepted(vault.send(snapshot(document, None, original)).await); + let mut update = snapshot(document, Some(base.vault_update_id), ""); + update.content = + PushContent::Diff(EditedText::from_strings(original, &desired.into()).to_diff()); + let updated = accepted(vault.send(update).await); + assert_eq!(vault.latest(document).await.content, desired.as_bytes()); + + for invalid_length in [i64::MIN, i64::MAX] { + let mut malformed = snapshot(document, Some(updated.vault_update_id), ""); + malformed.content = PushContent::Diff(vec![NumberOrText::Number(invalid_length)]); + assert!(matches!( + malformed.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + } + assert_eq!( + vault.latest(document).await.vault_update_id, + updated.vault_update_id + ); + + let another = accepted( + vault + .send(snapshot(uuid::Uuid::new_v4(), None, "another document")) + .await, + ); + let mut foreign_base = snapshot(document, Some(another.vault_update_id), ""); + foreign_base.content = PushContent::Diff(vec![]); + assert!(matches!( + vault.send(foreign_base).await, + DocumentUpdateResponse::StaleBase(version) + if version.metadata.vault_update_id == updated.vault_update_id + )); + + let binary_document = uuid::Uuid::new_v4(); + let mut binary = snapshot(binary_document, None, ""); + binary.content = PushContent::Snapshot(STANDARD.encode([0xff])); + let binary_base = accepted(vault.send(binary).await); + let mut invalid_text = snapshot(binary_document, Some(binary_base.vault_update_id), ""); + invalid_text.content = PushContent::Diff(vec![]); + assert!(matches!( + invalid_text.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(binary_document).await.vault_update_id, + binary_base.vault_update_id + ); + } +} diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs new file mode 100644 index 00000000..db99c374 --- /dev/null +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -0,0 +1,122 @@ +use anyhow::anyhow; +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use serde::Deserialize; +use sha2::{Digest, Sha256}; + +use super::utils::find_already_processed_event; +use crate::{ + app_state::{ + AppState, + database::{ + Database, + models::{EventRecord, FileManifest, VaultEvent, VaultId}, + }, + }, + errors::{SyncServerError, client_error, server_error}, + server::{requests::PushFileManifest, responses::FileManifestUpdateResponse}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct PutFileManifestPath { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[axum::debug_handler] +pub async fn put_file_manifest( + Path(path): Path, + State(state): State, + Json(push): Json, +) -> Result, SyncServerError> { + debug!("Pushing file manifest for vault `{}`", path.vault_id); + + // BTreeMap canonicalizes file manifest entries for request fingerprinting. + let fingerprint = Sha256::digest( + serde_json::to_vec(&("file_manifest", &push)) + .map_err(|error| server_error(error.into()))?, + ); + + let mut tx = state + .database + .create_write_transaction(&path.vault_id) + .await + .map_err(server_error)?; + + if let Some(event) = + find_already_processed_event(&mut tx, push.request_id, &fingerprint).await? + { + return match event { + VaultEvent::FileManifest { file_manifest } => { + Ok(Json(FileManifestUpdateResponse::Accepted { + file_manifest_id: file_manifest.file_manifest_id, + })) + } + VaultEvent::Content { .. } => Err(client_error(anyhow!( + "Stored event does not match file manifest request" + ))), + }; + } + + let latest = Database::current_file_manifest(&mut tx) + .await + .map_err(server_error)?; + + if latest.file_manifest_id != push.parent_file_manifest_id { + return Ok(Json(FileManifestUpdateResponse::StaleBase(latest))); + } + + for id in push.entries.keys() { + if state + .database + .get_latest_document_version(&path.vault_id, id, Some(&mut tx)) + .await + .map_err(server_error)? + .is_none() + { + return Err(client_error(anyhow!( + "File manifest references missing content: {id}" + ))); + } + } + + let file_manifest = FileManifest { + file_manifest_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) + .await + .map_err(server_error)?, + entries: push.entries, + }; + + Database::insert_file_manifest(&mut tx, &file_manifest) + .await + .map_err(server_error)?; + + let response = FileManifestUpdateResponse::Accepted { + file_manifest_id: file_manifest.file_manifest_id, + }; + + let event = EventRecord { + event_id: file_manifest.file_manifest_id, + request_id: push.request_id, + event: VaultEvent::FileManifest { file_manifest }, + }; + + Database::write_event(&mut tx, &event) + .await + .map_err(server_error)?; + + tx.commit() + .await + .map_err(|error| server_error(error.into()))?; + + state + .broadcasts + .notify_about_vault_update(path.vault_id) + .await; + + Ok(Json(response)) +} diff --git a/sync-server/src/server/endpoints/utils.rs b/sync-server/src/server/endpoints/utils.rs new file mode 100644 index 00000000..75e08989 --- /dev/null +++ b/sync-server/src/server/endpoints/utils.rs @@ -0,0 +1,32 @@ +use anyhow::anyhow; + +use crate::{ + app_state::database::{ + Database, Transaction, + models::{EventRecord, VaultEvent}, + }, + errors::{SyncServerError, client_error, server_error}, +}; + +pub(super) async fn find_already_processed_event( + tx: &mut Transaction<'_>, + request_id: uuid::Uuid, + fingerprint: &[u8], +) -> Result, SyncServerError> { + let Some((stored_fingerprint, event_json)) = Database::get_request_event(tx, request_id) + .await + .map_err(server_error)? + else { + return Ok(None); + }; + + if stored_fingerprint != fingerprint { + return Err(client_error(anyhow!( + "Request ID was reused with a different payload" + ))); + } + + serde_json::from_str::(&event_json) + .map(|record| Some(record.event)) + .map_err(|error| server_error(error.into())) +} diff --git a/sync-server/src/server/endpoints/vault_snapshot.rs b/sync-server/src/server/endpoints/vault_snapshot.rs new file mode 100644 index 00000000..552517c5 --- /dev/null +++ b/sync-server/src/server/endpoints/vault_snapshot.rs @@ -0,0 +1,36 @@ +use axum::{ + Json, + extract::{Path, State}, +}; +use log::debug; +use serde::Deserialize; + +use crate::{ + app_state::{ + AppState, + database::models::{VaultId, VaultSnapshot}, + }, + errors::{SyncServerError, server_error}, + utils::normalize::normalize, +}; + +#[derive(Deserialize)] +pub struct VaultSnapshotPath { + #[serde(deserialize_with = "normalize")] + vault_id: VaultId, +} + +#[axum::debug_handler] +pub async fn vault_snapshot( + Path(path): Path, + State(state): State, +) -> Result, SyncServerError> { + debug!("Fetching vault snapshot for vault `{}`", path.vault_id); + + state + .database + .vault_snapshot(&path.vault_id) + .await + .map(Json) + .map_err(server_error) +} diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index bb10b49f..97257215 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -1,4 +1,16 @@ -use anyhow::Context; +use crate::{ + app_state::{ + AppState, + database::models::VaultId, + websocket::{ + broadcasts::Notification, + models::{CursorPositionFromServer, WebSocketClientMessage, WebSocketServerMessage}, + utils::{get_authenticated_handshake, send_update_over_websocket}, + }, + }, + errors::{SyncServerError, client_error, server_error}, + utils::normalize::normalize, +}; use axum::{ extract::{ Path, State, @@ -6,27 +18,11 @@ use axum::{ }, response::Response, }; -use futures::stream::StreamExt; +use futures::StreamExt; use log::{debug, info}; use serde::Deserialize; - -use crate::{ - app_state::{ - AppState, - database::models::VaultId, - websocket::{ - models::{ - CursorPositionFromServer, WebSocketClientMessage, WebSocketServerMessage, - WebSocketVaultUpdate, - }, - utils::{ - get_authenticated_handshake, get_unseen_documents, send_update_over_websocket, - }, - }, - }, - errors::{SyncServerError, client_error, server_error}, - utils::normalize::normalize, -}; +use std::time::Duration; +use tokio::sync::broadcast::error::RecvError; #[derive(Deserialize)] pub struct WebSocketPathParams { @@ -34,160 +30,96 @@ pub struct WebSocketPathParams { vault_id: VaultId, } +#[axum::debug_handler] pub async fn websocket_handler( ws: WebSocketUpgrade, - Path(WebSocketPathParams { vault_id }): Path, + Path(path): Path, State(state): State, ) -> Result { - Ok(ws.on_upgrade(move |socket| websocket_wrapped(state, socket, vault_id))) -} - -async fn websocket_wrapped(state: AppState, stream: WebSocket, vault_id: VaultId) { - info!("WebSocket connection opened on vault `{vault_id}`"); - - let result = websocket(state, stream, vault_id.clone()).await; - - if let Err(err) = result { - debug!("WebSocket connection error on vault `{vault_id}`: {err}"); - } -} - -#[allow(clippy::too_many_lines)] -async fn websocket( - state: AppState, - stream: WebSocket, - vault_id: VaultId, -) -> Result<(), SyncServerError> { - let (mut sender, mut websocket_receiver) = stream.split(); - - let authed_handshake = get_authenticated_handshake( - &state, - &vault_id, - websocket_receiver - .next() - .await - .transpose() - .unwrap_or_default(), - )?; - - info!( - "WebSocket handshake successful for vault `{vault_id}` for `{}`", - authed_handshake.handshake.device_id + debug!( + "Upgrading WebSocket connection for vault `{}`", + path.vault_id ); - let mut broadcast_receiver = state.broadcasts.get_receiver(vault_id.clone()).await; + Ok(ws.on_upgrade(move |socket| async move { + if let Err(error) = websocket(state, socket, path.vault_id).await { + debug!("WebSocket disconnected: {error}"); + } + })) +} - send_update_over_websocket( - &WebSocketServerMessage::VaultUpdate(WebSocketVaultUpdate { - documents: get_unseen_documents( - &state, - &vault_id, - authed_handshake.handshake.last_seen_vault_update_id, - ) - .await?, - is_initial_sync: true, - }), - &mut sender, - ) - .await?; +async fn websocket( + state: AppState, + socket: WebSocket, + vault: VaultId, +) -> Result<(), SyncServerError> { + let (mut sender, mut receiver) = socket.split(); + let authenticated = get_authenticated_handshake( + &state, + &vault, + receiver.next().await.transpose().unwrap_or_default(), + )?; - send_update_over_websocket( - &WebSocketServerMessage::CursorPositions(CursorPositionFromServer { - clients: state.cursors.get_cursors(&vault_id).await, - }), - &mut sender, - ) - .await?; + let device = authenticated.handshake.device_id; + let mut after = authenticated + .handshake + .last_seen_vault_update_id + .unwrap_or(0); + let mut notifications = state.broadcasts.get_receiver(vault.clone()).await; + let mut timer = tokio::time::interval(Duration::from_secs(2)); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let device_id = authed_handshake.handshake.device_id.clone(); - let mut send_task = tokio::spawn(async move { - while let Ok(update) = broadcast_receiver.recv().await { - if Some(&device_id) == update.origin_device_id.as_ref() { - continue; + info!("WebSocket connected to vault {vault}"); + + // Exactly one sender drains durable events in commit order. In-memory + // notifications are only wakeups, so lag and commit/notify crashes are safe. + let result = async { + loop { + let batch = state.database.events_after(&vault, after).await.map_err(|error| server_error(error.into()))?; + + if !batch.events.is_empty() { + let head = batch.head_event_id; + send_update_over_websocket(&WebSocketServerMessage::VaultEvents(batch), &mut sender).await?; + after = head; } - let message = match update.message { - WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }) => { - WebSocketServerMessage::CursorPositions(CursorPositionFromServer { - clients: clients - .into_iter() - .filter(|client| client.device_id != device_id) - .collect(), - }) - } - WebSocketServerMessage::VaultUpdate(_) => update.message, - }; + tokio::select! { + _ = timer.tick() => {}, + notification = notifications.recv() => match notification { + Ok(update) => if let Notification::Cursors(mut positions) = update { + positions.clients.retain(|c| c.device_id != device); + send_update_over_websocket(&WebSocketServerMessage::CursorPositions(positions), &mut sender).await?; + }, - send_update_over_websocket(&message, &mut sender).await?; - } + Err(RecvError::Lagged(_)) => {}, // Drain the durable log on the next iteration. + Err(RecvError::Closed) => break, + }, - Ok::<(), SyncServerError>(()) - }); - - let device_id = authed_handshake.handshake.device_id.clone(); - let vault_id_clone = vault_id.clone(); - let cursor_manager = state.cursors.clone(); - let mut receive_task = tokio::spawn(async move { - while let Some(Ok(Message::Text(message))) = websocket_receiver.next().await { - let message: WebSocketClientMessage = serde_json::from_str(&message) - .context("Failed to parse WebSocket message from client") - .map_err(server_error)?; - - match message { - WebSocketClientMessage::Handshake(_) => { - return Err(client_error(anyhow::anyhow!( - "Unexpected handshake message" - ))); - } - WebSocketClientMessage::CursorPositions(cursors) => { - cursor_manager - .update_cursors( - vault_id_clone.clone(), - authed_handshake.user.name.clone(), - &device_id, - cursors.documents_with_cursors, - ) - .await; + incoming = receiver.next() => match incoming { + Some(Ok(Message::Text(text))) => { + let message: WebSocketClientMessage = serde_json::from_str(&text).map_err(|error| client_error(error.into()))?; + match message { + WebSocketClientMessage::Handshake(_) => return Err(client_error(anyhow::anyhow!("Unexpected handshake"))), + WebSocketClientMessage::CursorPositions(positions) => { + state.cursors.update_cursors(vault.clone(), authenticated.user.name.clone(), &device, positions.documents_with_cursors).await; + let clients = state.cursors.get_cursors(&vault).await.into_iter().filter(|c| c.device_id != device).collect(); + send_update_over_websocket(&WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }), &mut sender).await?; + } + } + }, + Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => {}, + _ => break, } } } - - Ok::<(), SyncServerError>(()) - }); - - tokio::select! { - _ = &mut send_task => receive_task.abort(), - _ = &mut receive_task => send_task.abort(), - }; - - let result: Result<(), SyncServerError> = (async { - send_task - .await - .context("WebSocket send task failed") - .map_err(client_error) - .and_then(|err| err)?; - - receive_task - .await - .context("WebSocket receive task failed") - .map_err(client_error) - .and_then(|err| err)?; - + Ok(()) - }) - .await; + }.await; state .cursors - .remove_cursors_of_device(&vault_id, &authed_handshake.handshake.device_id) + .remove_cursors_of_device(&vault, &device) .await; - if result.is_err() { - info!( - "WebSocket disconnected on vault `{vault_id}` for `{}`", - authed_handshake.handshake.device_id - ); - } - result } diff --git a/sync-server/src/server/requests.rs b/sync-server/src/server/requests.rs index 7b814559..a27e62a9 100644 --- a/sync-server/src/server/requests.rs +++ b/sync-server/src/server/requests.rs @@ -1,23 +1,149 @@ -use crate::app_state::database::models::VaultUpdateId; +use crate::{ + app_state::database::models::{DocumentId, FileManifestEntries, VaultUpdateId}, + utils::portable_path::validate_file_manifest, +}; use reconcile_text::NumberOrText; -use serde::{Deserialize, Serialize}; +use serde::{ + Deserialize, Deserializer, Serialize, + de::{Error, MapAccess, Visitor}, +}; +use std::{collections::BTreeMap, fmt}; use ts_rs::TS; +#[derive(TS, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +#[ts(export)] +pub struct PutFileContent { + pub request_id: uuid::Uuid, + #[ts(as = "Option")] + pub parent_version_id: Option, + pub content: PushContent, +} + #[derive(TS, Debug, Deserialize, Serialize)] #[serde(tag = "type", content = "value")] +#[serde(deny_unknown_fields)] pub enum PushContent { Snapshot(String), Diff(#[ts(type = "Array")] Vec), - Delete, } #[derive(TS, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] #[ts(export)] -pub struct PushDocument { +pub struct PushFileManifest { pub request_id: uuid::Uuid, - #[ts(as = "Option")] - pub parent_version_id: Option, - pub relative_path: String, - pub content: PushContent, + #[ts(as = "f64")] + pub parent_file_manifest_id: VaultUpdateId, + #[serde(deserialize_with = "deserialize_file_manifest_entries")] + #[ts(type = "Record")] + pub entries: FileManifestEntries, +} + +// A normal JSON map decoder silently overwrites duplicate keys. UUID parsing also +// catches differently spelled keys that denote the same identity. +fn deserialize_file_manifest_entries<'de, D: Deserializer<'de>>( + d: D, +) -> Result { + struct Entries; + + impl<'de> Visitor<'de> for Entries { + type Value = FileManifestEntries; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("unique document IDs mapped to paths") + } + + fn visit_map>(self, mut map: M) -> Result { + let mut entries = BTreeMap::new(); + + while let Some((id, path)) = map.next_entry::()? { + if entries.insert(id, path).is_some() { + return Err(M::Error::custom("Duplicate document ID")); + } + } + + Ok(entries) + } + } + + let entries = d.deserialize_map(Entries)?; + + validate_file_manifest(&entries).map_err(D::Error::custom)?; + + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::PushFileManifest; + + fn deserialize_manifest(entries: &str) -> serde_json::Result { + serde_json::from_str(&format!( + r#"{{ + "requestId": "00000000-0000-0000-0000-000000000001", + "parentFileManifestId": 0, + "entries": {entries} + }}"# + )) + } + + #[test] + fn deserializes_empty_and_valid_file_manifests() { + assert!(deserialize_manifest("{}").unwrap().entries.is_empty()); + + let push = deserialize_manifest( + r#"{ + "00000000-0000-0000-0000-000000000001": "Notes/Café.md", + "00000000-0000-0000-0000-000000000002": "Notes/日記.md" + }"#, + ) + .unwrap(); + + assert_eq!( + push.entries + .values() + .map(String::as_str) + .collect::>(), + ["Notes/Café.md", "Notes/日記.md"] + ); + } + + #[test] + fn deserialization_rejects_invalid_paths_and_manifest_conflicts() { + for (entries, expected_error) in [ + ( + r#"{"00000000-0000-0000-0000-000000000001": "../notes.md"}"#, + "Invalid path component", + ), + ( + r#"{ + "00000000-0000-0000-0000-000000000001": "Notes/a.md", + "00000000-0000-0000-0000-000000000002": "notes/b.md" + }"#, + "Conflicting path", + ), + ] { + let error = deserialize_manifest(entries).unwrap_err(); + + assert!(error.to_string().contains(expected_error), "{error}"); + } + } + + #[test] + fn deserialization_rejects_duplicate_document_ids_including_uuid_aliases() { + let id = "00000000-0000-0000-0000-000000000abc"; + + for duplicate in [id.to_owned(), id.to_uppercase()] { + let entries = format!(r#"{{"{id}": "a.md", "{duplicate}": "b.md"}}"#); + let error = deserialize_manifest(&entries).unwrap_err(); + + assert!( + error.to_string().contains("Duplicate document ID"), + "{error}" + ); + } + } } diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index ed469949..66c5b5cb 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -1,7 +1,9 @@ -use serde::{self, Serialize}; +use serde::{self, Deserialize, Serialize}; use ts_rs::TS; -use crate::app_state::database::models::{DocumentVersion, DocumentVersionWithoutContent}; +use crate::app_state::database::models::{ + DocumentVersion, DocumentVersionWithoutContent, FileManifest, VaultUpdateId, +}; /// Response to a ping request. #[derive(TS, Debug, Clone, Serialize)] @@ -24,7 +26,7 @@ pub struct PingResponse { } /// Response to an update document request. -#[derive(TS, Debug, Clone, Serialize)] +#[derive(TS, Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] #[ts(export)] pub enum DocumentUpdateResponse { @@ -34,3 +36,15 @@ pub enum DocumentUpdateResponse { /// Nothing was written. The client must incorporate this version before retrying. StaleBase(DocumentVersion), } + +#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +#[ts(export)] +pub enum FileManifestUpdateResponse { + Accepted { + #[serde(rename = "fileManifestId")] + #[ts(as = "f64")] + file_manifest_id: VaultUpdateId, + }, + StaleBase(FileManifest), +} -- 2.54.0 From 8a85a68dbb6759e27b629eeba2c95c1f17f99476 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 17:06:39 +0100 Subject: [PATCH 13/48] Wire it up --- sync-server/README.md | 136 ++++++++++----- sync-server/src/app_state.rs | 2 +- sync-server/src/errors.rs | 2 +- sync-server/src/server.rs | 16 +- sync-server/src/utils.rs | 4 +- sync-server/src/utils/portable_path.rs | 223 +++++++++++++++++++++++++ 6 files changed, 332 insertions(+), 51 deletions(-) create mode 100644 sync-server/src/utils/portable_path.rs diff --git a/sync-server/README.md b/sync-server/README.md index 9b11a061..90fc3e48 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -1,50 +1,106 @@ # Sync server -## Document pushes (API v3) +## API v4 -The server stores immutable document versions. One compare-and-swap endpoint, -`PUT /vaults/:vault_id/documents/:document_id`, handles every mutation. The client -generates the document UUID; the JSON body contains: +The server is a compare-and-swap store. Content versions do not contain paths or +deletions. A separately versioned file manifest owns the namespace and membership. +All routes below are under `/vaults/:vault_id` and require bearer authentication +(except the WebSocket, which authenticates its first message). -- `requestId`: a UUID identifying this exact push. -- `parentVersionId`: `null` to create an absent document, otherwise its base version. -- `relativePath`: the desired filename. -- `content`: `{ "type": "Snapshot", "value": "" }`, - `{ "type": "Diff", "value": [3, "inserted text", -2] }`, or `{ "type": "Delete" }`. +| Endpoint | Purpose | +| --- | --- | +| `PUT /documents/:document_id` | CAS an immutable content version. | +| `GET /documents/:document_id` | Current content version, including base64 bytes. | +| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. | +| `GET /file_manifest` | Current file manifest. | +| `PUT /file_manifest` | CAS the complete file manifest. | +| `GET /vault_snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. | +| `GET /events_since?after=N` | Every event after N, in order, with the current watermark. | +| `GET /ws` | Ordered event delivery and ephemeral cursor positions. | -All pushes return one of: +A content push includes a `Device-Id` header and this JSON body: -- `Accepted`: version metadata for the exact snapshot committed. A retry with the - same request ID returns the original acknowledgement, even after a restart or - subsequent updates. Clients must persist the ID with the unchanged request. - A payload fingerprint rejects reusing that ID for a different mutation. -- `StaleBase`: the current version, including `contentBase64`. Nothing was written; - the client merges locally and submits a new push with a new request ID. - -The base check, path allocation, new version, and durable acknowledgement share -one SQLite write transaction. A deleted document stays deleted; recreating a file -uses a new document ID. Occupied paths receive an available numbered filename. -Text updates accept reconcile-text transport diffs against the base version; -the server only reconstructs snapshots and never merges concurrent edits. - -Snapshot requests use base64, adding roughly one third to the raw file size. -Server and reverse-proxy request limits apply to the encoded JSON body. - -API v3 uses a fresh schema. Update clients together with the server and initialise -new vault databases; compatibility with previous releases is not supported. - -## Creating/resetting the Database for development - -```sh -rm -rf db.sqlite* -sqlx database create --database-url sqlite://db.sqlite3 -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 -cargo sqlx prepare --workspace +```json +{ + "requestId": "2e3ac3fe-a8d2-49ef-bdba-30ab5b7336f1", + "parentVersionId": null, + "content": { "type": "Snapshot", "value": "aGVsbG8=" } +} ``` -## Updating the DB schema through a migration +`parentVersionId: null` creates previously absent content using the UUID in the +URL. Subsequent pushes name its current version. `Diff` with a `value` array of +numbers/strings is also supported; the server only reconstructs the supplied +snapshot. File modification times are not part of the protocol. Paths and +`Delete` payloads are rejected. -```sh -sqlx migrate add --source src/app_state/database/migrations -sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 +A file manifest push is: + +```json +{ + "requestId": "69e6a1ba-1f20-4d81-9ae2-e7812c321e65", + "parentFileManifestId": 0, + "entries": { "7534676e-b51a-42d6-8a62-e70a477f731d": "notes/hello.md" } +} ``` + +The initial empty file manifest has version 0. Upload a new document's content +first, then publish its ID in the file manifest. Removal from the file manifest +deletes membership; content updates never restore membership. Unreferenced +content remains readable. + +File manifest versions are stored in `file_manifests`, with one row per +UUID-to-path mapping in `file_manifest_entries`. Each version has a header row +even when its entries are empty. Entries are unique by document ID and exact path +within a version; portable path validation also rejects aliases and directory +conflicts. Reads reconstruct the complete map, and vault snapshots join these +rows to the latest content versions. The version, entries, and complete file +manifest event in `events.event_json` commit atomically. Event replay and retry +acknowledgements continue to use `event_json`. + +Both pushes return a tagged `Accepted` or `StaleBase` response. An accepted file +manifest response returns its assigned file manifest ID. A stale content response +includes bytes; a stale file manifest response includes the complete map. +Accepted requests store their unique request ID and fingerprint in the event row, +atomically with the version. Retrying the exact request reconstructs the original +`Accepted` response from that event before checking the current head, even after +later changes or a restart. Changing a payload while reusing its request ID is +rejected, including reuse across content and file manifest pushes. Session/device +identity is not part of the fingerprint. + +File manifest validation rejects duplicate UUID keys (including aliases), missing +content, path aliases, inconsistent directory spelling, and file/ancestor +conflicts. Paths use `/`, NFC normalization, at most 240 UTF-8 bytes, and portable +Windows filename restrictions. Alias comparison is NFC → Unicode uppercase → +NFC. `.vault-link-sync` and its aliases are reserved. The server never sanitizes +or allocates a replacement name. + +## Ordering and recovery + +Every accepted mutation appends one event in the same SQLite write transaction +as its state. Events also provide durable request tracking; there is no separate +receipt table. Event IDs are contiguous safe JavaScript integers; +content/file-manifest version IDs are the event IDs that created them. Rejected +requests and retries of accepted requests append no events. Timestamps never +determine event ordering. + +The WebSocket handshake retains `lastSeenVaultUpdateId`. The server sends +`{type: "vaultEvents", headEventId, events}` in increasing order, including events +originated by that client. Each connection drains the database through one sender. +In-memory notifications only wake it; polling covers missed wakes and cancellation +after commit. Reconnect with the last durably processed event ID. The HTTP replay +endpoint uses the same history, so losing the final notification is recoverable. +A fresh client reads `/vault_snapshot`, applies it durably, then replays after its +watermark. +Cursor notifications are ephemeral and do not consume event IDs. + +SQLite uses WAL, `synchronous=FULL`, and `fullfsync` where supported. On Unix, startup also +flushes database directory entries; the filesystem must support those durability +operations. Content, file manifests, and events (including request fingerprints) +are retained indefinitely. + +API v4 requires fresh vault databases and upgraded clients. Existing database +migration checksums intentionally reject earlier schemas; there is no automatic +erasure or migration. Use a new database directory and preserve old data. +Base64 increases snapshot body size by about one third; request limits apply to +the encoded JSON body. diff --git a/sync-server/src/app_state.rs b/sync-server/src/app_state.rs index 2019e08e..910564f9 100644 --- a/sync-server/src/app_state.rs +++ b/sync-server/src/app_state.rs @@ -20,7 +20,7 @@ pub struct AppState { impl AppState { pub async fn try_new(config: Config) -> Result { let broadcasts = Broadcasts::new(&config.server); - let database = Database::try_new(&config.database, &broadcasts).await?; + let database = Database::try_new(&config.database).await?; let cursors: Cursors = Cursors::new(&config.database, &broadcasts); Cursors::start_background_task(cursors.clone()); diff --git a/sync-server/src/errors.rs b/sync-server/src/errors.rs index 831b0e86..c505b8ae 100644 --- a/sync-server/src/errors.rs +++ b/sync-server/src/errors.rs @@ -5,7 +5,7 @@ use axum::{ http::StatusCode, response::{IntoResponse, Response}, }; -use log::{debug, error}; +use log::debug; use serde::Serialize; use thiserror::Error; use ts_rs::TS; diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index ecc01de9..27ff1155 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -93,18 +93,22 @@ pub async fn create_server(config: Config) -> Result<()> { fn get_authed_routes(app_state: AppState) -> Router { Router::new() .route( - "/vaults/:vault_id/manifest", - get(endpoints::get_manifest::get_manifest).put(endpoints::put_manifest::put_manifest), + "/vaults/:vault_id/file_manifest", + get(endpoints::get_file_manifest::get_file_manifest) + .put(endpoints::put_file_manifest::put_file_manifest), ) .route( - "/vaults/:vault_id/state", - get(endpoints::snapshot::snapshot), + "/vaults/:vault_id/vault_snapshot", + get(endpoints::vault_snapshot::vault_snapshot), + ) + .route( + "/vaults/:vault_id/events_since", + get(endpoints::events::events), ) - .route("/vaults/:vault_id/events", get(endpoints::events::events)) .route( "/vaults/:vault_id/documents/:document_id", get(endpoints::fetch_latest_document_version::fetch_latest_document_version) - .put(endpoints::push_document::push_document), + .put(endpoints::put_file_content::put_file_content), ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id/content", diff --git a/sync-server/src/utils.rs b/sync-server/src/utils.rs index 35d6a340..4798e39c 100644 --- a/sync-server/src/utils.rs +++ b/sync-server/src/utils.rs @@ -1,6 +1,4 @@ -pub mod dedup_paths; -pub mod find_first_available_path; pub mod log_level; pub mod normalize; +pub mod portable_path; pub mod rotating_file_writer; -pub mod sanitize_path; diff --git a/sync-server/src/utils/portable_path.rs b/sync-server/src/utils/portable_path.rs new file mode 100644 index 00000000..34551abf --- /dev/null +++ b/sync-server/src/utils/portable_path.rs @@ -0,0 +1,223 @@ +use crate::app_state::database::models::FileManifestEntries; +use anyhow::{Result, bail, ensure}; +use std::collections::BTreeMap; +use unicode_normalization::UnicodeNormalization; + +pub const INTERNAL_DIRECTORY: &str = ".vault-link-sync"; + +// Use the same locale-independent Unicode upper-case mapping in the client. +// NFC -> uppercase -> NFC also catches ß/SS and final/ordinary sigma aliases. +pub fn path_key(path: &str) -> String { + path.nfc() + .collect::() + .to_uppercase() + .nfc() + .collect() +} + +pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { + let mut nodes: BTreeMap = BTreeMap::new(); + + for path in entries.values() { + ensure!( + !path.is_empty() && path.len() <= 240, + "Path must contain 1–240 UTF-8 bytes" + ); + + ensure!( + path.nfc().collect::() == *path, + "Paths must be NFC normalized" + ); + + let parts: Vec<_> = path.split('/').collect(); + + ensure!( + path_key(parts[0]) != path_key(INTERNAL_DIRECTORY), + "Reserved sync directory" + ); + + for (index, part) in parts.iter().enumerate() { + ensure!( + !part.is_empty() && *part != "." && *part != "..", + "Invalid path component" + ); + + ensure!(!part.ends_with(['.', ' ']), "Trailing dot or space in path"); + + ensure!( + !part + .chars() + .any(|c| c.is_control() || c == '\\' || "<>:\"|?*".contains(c)), + "Non-portable filename" + ); + + let stem = part + .split('.') + .next() + .unwrap_or_default() + .trim_end_matches(' ') + .to_uppercase(); + + ensure!( + !matches!( + stem.as_str(), + "CON" | "PRN" | "AUX" | "NUL" | "CONIN$" | "CONOUT$" + ), + "Reserved filename" + ); + + if stem.starts_with("COM") || stem.starts_with("LPT") { + ensure!( + !matches!( + &stem[3..], + "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "¹" | "²" | "³" + ), + "Reserved filename" + ); + } + + let prefix = parts[..=index].join("/"); + let is_file = index == parts.len() - 1; + + if let Some((spelling, previous_is_file)) = nodes.get(&path_key(&prefix)) { + if *spelling != prefix || *previous_is_file || is_file { + bail!("Conflicting path: {path}"); + } + } else { + nodes.insert(path_key(&prefix), (prefix, is_file)); + } + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{FileManifestEntries, path_key, validate_file_manifest}; + use uuid::Uuid; + + fn manifest(paths: &[&str]) -> FileManifestEntries { + paths + .iter() + .enumerate() + .map(|(index, path)| (Uuid::from_u128(index as u128), (*path).to_owned())) + .collect() + } + + #[test] + fn path_keys_normalize_unicode_and_case_aliases() { + for (path, expected) in [ + ("Notes/Café.md", "NOTES/CAFÉ.MD"), + ("notes/Cafe\u{301}.md", "NOTES/CAFÉ.MD"), + ("Straße.md", "STRASSE.MD"), + ("σς.md", "ΣΣ.MD"), + ] { + assert_eq!(path_key(path), expected, "path: {path:?}"); + } + } + + #[test] + fn accepts_empty_manifests_and_portable_paths_with_shared_directories() { + validate_file_manifest(&manifest(&[])).unwrap(); + + let entries = manifest(&[ + "Notes/Café.md", + "Notes/日記.md", + "Notes/Archive/old.md", + ".gitkeep", + "COM10.txt", + "LPT0.txt", + ".vault-link-sync-backup/state.json", + "Notes/.vault-link-sync/state.json", + ]); + + validate_file_manifest(&entries).unwrap(); + } + + #[test] + fn enforces_the_path_length_limit_in_utf8_bytes() { + for path in ["a".repeat(240), "é".repeat(120)] { + validate_file_manifest(&manifest(&[&path])).unwrap(); + + let too_long = format!("{path}a"); + + assert!( + validate_file_manifest(&manifest(&[&too_long])).is_err(), + "accepted a path over 240 bytes: {too_long:?}" + ); + } + } + + #[test] + fn rejects_invalid_components_characters_and_non_normalized_paths() { + for path in [ + "", + "/notes.md", + "notes.md/", + "notes//a.md", + "./notes.md", + "notes/../a.md", + "notes./a.md", + "notes /a.md", + "Cafe\u{301}.md", + ] { + assert!( + validate_file_manifest(&manifest(&[path])).is_err(), + "accepted invalid path: {path:?}" + ); + } + + for character in ['\\', '<', '>', ':', '"', '|', '?', '*', '\0', '\u{85}'] { + let path = format!("notes/a{character}b.md"); + + assert!( + validate_file_manifest(&manifest(&[&path])).is_err(), + "accepted non-portable character in path: {path:?}" + ); + } + } + + #[test] + fn rejects_reserved_sync_and_windows_names() { + for path in [ + ".vault-link-sync", + ".VAULT-LINK-SYNC/state.json", + "con", + "prn.txt", + "AUX.md", + "nul .txt", + "CONIN$", + "conout$.log", + "com1.txt", + "LPT9", + "COM¹.txt", + "lpt²/file.md", + "COM³.txt", + ] { + assert!( + validate_file_manifest(&manifest(&[path])).is_err(), + "accepted reserved path: {path:?}" + ); + } + } + + #[test] + fn rejects_path_conflicts_in_either_entry_order() { + for [first, second] in [ + ["notes.md", "notes.md"], + ["Notes.md", "notes.md"], + ["Straße.md", "STRASSE.md"], + ["ς.md", "σ.md"], + ["Notes/a.md", "notes/b.md"], + ["notes", "notes/a.md"], + ] { + for paths in [[first, second], [second, first]] { + assert!( + validate_file_manifest(&manifest(&paths)).is_err(), + "accepted conflicting paths: {paths:?}" + ); + } + } + } +} -- 2.54.0 From cc5ad0f27c50f5a41551f3637b5cf9f7e56f1e65 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 17:08:10 +0100 Subject: [PATCH 14/48] Fmt --- sync-server/src/app_state/database/mutations.rs | 5 ++++- sync-server/src/app_state/database/queries.rs | 8 +++++++- sync-server/src/server/endpoints/websocket.rs | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs index 59383789..09f7a73b 100644 --- a/sync-server/src/app_state/database/mutations.rs +++ b/sync-server/src/app_state/database/mutations.rs @@ -1,4 +1,7 @@ -use super::{Database, Transaction, models::*}; +use super::{ + Database, Transaction, + models::{EventRecord, FileManifest, StoredDocumentVersion, VaultUpdateId}, +}; use anyhow::Result; impl Database { diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs index edc77801..f9225ef4 100644 --- a/sync-server/src/app_state/database/queries.rs +++ b/sync-server/src/app_state/database/queries.rs @@ -1,4 +1,10 @@ -use super::{Database, Transaction, models::*}; +use super::{ + Database, Transaction, + models::{ + DocumentId, EventBatch, FileManifest, StoredDocumentVersion, VaultId, VaultSnapshot, + VaultUpdateId, + }, +}; use anyhow::{Result, ensure}; impl Database { diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index 97257215..2be4b8f2 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -75,7 +75,7 @@ async fn websocket( // notifications are only wakeups, so lag and commit/notify crashes are safe. let result = async { loop { - let batch = state.database.events_after(&vault, after).await.map_err(|error| server_error(error.into()))?; + let batch = state.database.events_after(&vault, after).await.map_err(server_error)?; if !batch.events.is_empty() { let head = batch.head_event_id; @@ -107,12 +107,12 @@ async fn websocket( } } }, - Some(Ok(Message::Ping(_))) | Some(Ok(Message::Pong(_))) => {}, + Some(Ok(Message::Ping(_) | Message::Pong(_))) => {}, _ => break, } } } - + Ok(()) }.await; -- 2.54.0 From dbe8e5c4ceb6334d743bce3a3b82650ffe997cff Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 17:08:27 +0100 Subject: [PATCH 15/48] Regenerate types --- frontend/sync-client/src/services/types/EventRecord.ts | 4 ++-- .../src/services/types/{Manifest.ts => FileManifest.ts} | 2 +- .../src/services/types/FileManifestUpdateResponse.ts | 4 ++++ .../sync-client/src/services/types/ManifestUpdateResponse.ts | 4 ---- frontend/sync-client/src/services/types/PushFileManifest.ts | 3 +++ frontend/sync-client/src/services/types/PushManifest.ts | 3 --- .../src/services/types/{PushDocument.ts => PutFileContent.ts} | 2 +- frontend/sync-client/src/services/types/VaultEvent.ts | 4 ++-- frontend/sync-client/src/services/types/VaultSnapshot.ts | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) rename frontend/sync-client/src/services/types/{Manifest.ts => FileManifest.ts} (55%) create mode 100644 frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts delete mode 100644 frontend/sync-client/src/services/types/ManifestUpdateResponse.ts create mode 100644 frontend/sync-client/src/services/types/PushFileManifest.ts delete mode 100644 frontend/sync-client/src/services/types/PushManifest.ts rename frontend/sync-client/src/services/types/{PushDocument.ts => PutFileContent.ts} (59%) diff --git a/frontend/sync-client/src/services/types/EventRecord.ts b/frontend/sync-client/src/services/types/EventRecord.ts index 6a1bac34..47895639 100644 --- a/frontend/sync-client/src/services/types/EventRecord.ts +++ b/frontend/sync-client/src/services/types/EventRecord.ts @@ -1,5 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; -import type { Manifest } from "./Manifest"; +import type { FileManifest } from "./FileManifest"; -export type EventRecord = { eventId: number, requestId: string, } & ({ "type": "content", document: DocumentVersionWithoutContent, } | { "type": "manifest", manifest: Manifest, }); +export type EventRecord = { eventId: number, requestId: string, } & ({ "type": "content", document: DocumentVersionWithoutContent, } | { "type": "fileManifest", fileManifest: FileManifest, }); diff --git a/frontend/sync-client/src/services/types/Manifest.ts b/frontend/sync-client/src/services/types/FileManifest.ts similarity index 55% rename from frontend/sync-client/src/services/types/Manifest.ts rename to frontend/sync-client/src/services/types/FileManifest.ts index 9572ad93..1d7ff7fc 100644 --- a/frontend/sync-client/src/services/types/Manifest.ts +++ b/frontend/sync-client/src/services/types/FileManifest.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type Manifest = { manifestId: number, entries: Record, }; +export type FileManifest = { fileManifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts b/frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts new file mode 100644 index 00000000..4ccb541f --- /dev/null +++ b/frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts @@ -0,0 +1,4 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { FileManifest } from "./FileManifest"; + +export type FileManifestUpdateResponse = { "type": "Accepted", fileManifestId: number, } | { "type": "StaleBase" } & FileManifest; diff --git a/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts b/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts deleted file mode 100644 index 76ac2f7f..00000000 --- a/frontend/sync-client/src/services/types/ManifestUpdateResponse.ts +++ /dev/null @@ -1,4 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { Manifest } from "./Manifest"; - -export type ManifestUpdateResponse = { "type": "Accepted" } & Manifest | { "type": "StaleBase" } & Manifest; diff --git a/frontend/sync-client/src/services/types/PushFileManifest.ts b/frontend/sync-client/src/services/types/PushFileManifest.ts new file mode 100644 index 00000000..f7db488a --- /dev/null +++ b/frontend/sync-client/src/services/types/PushFileManifest.ts @@ -0,0 +1,3 @@ +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type PushFileManifest = { requestId: string, parentFileManifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/PushManifest.ts b/frontend/sync-client/src/services/types/PushManifest.ts deleted file mode 100644 index 743ad219..00000000 --- a/frontend/sync-client/src/services/types/PushManifest.ts +++ /dev/null @@ -1,3 +0,0 @@ -// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. - -export type PushManifest = { requestId: string, parentManifestId: number, entries: Record, }; diff --git a/frontend/sync-client/src/services/types/PushDocument.ts b/frontend/sync-client/src/services/types/PutFileContent.ts similarity index 59% rename from frontend/sync-client/src/services/types/PushDocument.ts rename to frontend/sync-client/src/services/types/PutFileContent.ts index a23b3bc4..c5a86c9b 100644 --- a/frontend/sync-client/src/services/types/PushDocument.ts +++ b/frontend/sync-client/src/services/types/PutFileContent.ts @@ -1,4 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { PushContent } from "./PushContent"; -export type PushDocument = { requestId: string, parentVersionId: number | null, content: PushContent, }; +export type PutFileContent = { requestId: string, parentVersionId: number | null, content: PushContent, }; diff --git a/frontend/sync-client/src/services/types/VaultEvent.ts b/frontend/sync-client/src/services/types/VaultEvent.ts index a96d5a2d..885a608b 100644 --- a/frontend/sync-client/src/services/types/VaultEvent.ts +++ b/frontend/sync-client/src/services/types/VaultEvent.ts @@ -1,5 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; -import type { Manifest } from "./Manifest"; +import type { FileManifest } from "./FileManifest"; -export type VaultEvent = { "type": "content", document: DocumentVersionWithoutContent, } | { "type": "manifest", manifest: Manifest, }; +export type VaultEvent = { "type": "content", document: DocumentVersionWithoutContent, } | { "type": "fileManifest", fileManifest: FileManifest, }; diff --git a/frontend/sync-client/src/services/types/VaultSnapshot.ts b/frontend/sync-client/src/services/types/VaultSnapshot.ts index 95a192f9..10b2470d 100644 --- a/frontend/sync-client/src/services/types/VaultSnapshot.ts +++ b/frontend/sync-client/src/services/types/VaultSnapshot.ts @@ -1,5 +1,5 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; -import type { Manifest } from "./Manifest"; +import type { FileManifest } from "./FileManifest"; -export type VaultSnapshot = { headEventId: number, manifest: Manifest, documents: Array, }; +export type VaultSnapshot = { headEventId: number, fileManifest: FileManifest, documents: Array, }; -- 2.54.0 From 2a05930ae769e57b7247c8132e7fc24812fb98a2 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 17:08:54 +0100 Subject: [PATCH 16/48] Fix lint --- sync-server/src/server/endpoints/put_file_content.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index 9d757bb8..b09ff536 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -170,7 +170,10 @@ mod tests { Database, models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultUpdateId}, }, - config::{Config, database_config::DatabaseConfig, server_config::ServerConfig}, + config::{ + Config, database_config::DatabaseConfig, server_config::ServerConfig, + user_config::VaultAccess, + }, }; async fn send_document( @@ -189,7 +192,7 @@ mod tests { Extension(User { name: user_id, token: String::new(), - vault_access: Default::default(), + vault_access: VaultAccess::default(), }), TypedHeader(DeviceIdHeader(device_id)), State(state.clone()), -- 2.54.0 From 99fa298b9273b15564009e7d7b846ab62f093aaa Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 19:51:08 +0100 Subject: [PATCH 17/48] Review pass --- sync-server/Cargo.lock | 2 + sync-server/Cargo.toml | 4 + sync-server/README.md | 17 ++ sync-server/src/app_state/cursors.rs | 139 +++++++++++++-- sync-server/src/app_state/database.rs | 83 +++++---- sync-server/src/app_state/database/models.rs | 4 +- sync-server/src/app_state/database/queries.rs | 12 +- sync-server/src/app_state/database/tests.rs | 160 ++++++++++++++++++ sync-server/src/config/server_config.rs | 48 +++++- sync-server/src/config/user_config.rs | 37 +++- sync-server/src/server.rs | 3 + sync-server/src/server/auth.rs | 5 +- sync-server/src/server/endpoints/events.rs | 4 +- .../fetch_document_version_content.rs | 4 +- .../fetch_latest_document_version.rs | 4 +- .../src/server/endpoints/get_file_manifest.rs | 4 +- sync-server/src/server/endpoints/ping.rs | 4 +- .../src/server/endpoints/put_file_content.rs | 4 +- .../src/server/endpoints/put_file_manifest.rs | 4 +- .../src/server/endpoints/vault_snapshot.rs | 4 +- sync-server/src/server/endpoints/websocket.rs | 4 +- sync-server/src/server/tests.rs | 121 +++++++++++++ sync-server/src/utils.rs | 2 +- sync-server/src/utils/normalize.rs | 13 -- sync-server/src/utils/normalize_vault_id.rs | 29 ++++ 25 files changed, 621 insertions(+), 94 deletions(-) create mode 100644 sync-server/src/app_state/database/tests.rs create mode 100644 sync-server/src/server/tests.rs delete mode 100644 sync-server/src/utils/normalize.rs create mode 100644 sync-server/src/utils/normalize_vault_id.rs diff --git a/sync-server/Cargo.lock b/sync-server/Cargo.lock index 810b4b93..f143afcc 100644 --- a/sync-server/Cargo.lock +++ b/sync-server/Cargo.lock @@ -2135,8 +2135,10 @@ dependencies = [ "serde_yaml", "sha2", "sqlx", + "tempfile", "thiserror 2.0.17", "tokio", + "tower", "tower-http", "tracing", "tracing-subscriber", diff --git a/sync-server/Cargo.toml b/sync-server/Cargo.toml index a333545b..bc39c34b 100644 --- a/sync-server/Cargo.toml +++ b/sync-server/Cargo.toml @@ -36,6 +36,10 @@ ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl", "no-serde-w base64 = "0.22.1" reconcile-text = { version = "0.8.0", features = ["serde"] } +[dev-dependencies] +tempfile = "3" +tower = { version = "0.5", features = ["util"] } + [profile.release] codegen-units = 1 lto = true diff --git a/sync-server/README.md b/sync-server/README.md index 90fc3e48..fc54337c 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -93,6 +93,23 @@ endpoint uses the same history, so losing the final notification is recoverable. A fresh client reads `/vault_snapshot`, applies it durably, then replays after its watermark. Cursor notifications are ephemeral and do not consume event IDs. +Cursor expiry and disconnect broadcast the remaining clients, including an empty +list when the last client leaves. + +## Vault names and database files + +Vault names and configured allowlists are trimmed and lowercased consistently. +Empty names, `.` and `..`, path separators, and control characters are rejected. +Other Unicode spellings remain distinct identities, including composed and +decomposed characters. + +Databases use `vaults/.sqlite` underneath the +configured database directory. This prevents filesystem case and Unicode aliases +from sharing a database. This is the only supported storage layout; top-level +`.sqlite` files are not loaded or migrated. Back up the entire database +directory, including the `vaults` subdirectory. + +Vault snapshots query metadata and byte lengths without loading document contents. SQLite uses WAL, `synchronous=FULL`, and `fullfsync` where supported. On Unix, startup also flushes database directory entries; the filesystem must support those durability diff --git a/sync-server/src/app_state/cursors.rs b/sync-server/src/app_state/cursors.rs index 2c57a49d..65b941ed 100644 --- a/sync-server/src/app_state/cursors.rs +++ b/sync-server/src/app_state/cursors.rs @@ -39,7 +39,9 @@ impl Cursors { ) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; - let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new); + let all_device_cursors = vault_to_cursors + .entry(vault_id.clone()) + .or_insert_with(Vec::new); all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id); all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors { @@ -48,8 +50,7 @@ impl Cursors { documents_with_cursors: document_to_cursors, })); - drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock - self.broadcast_cursors().await; + self.broadcast_cursors(&vault_id, all_device_cursors).await; } pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec { @@ -78,31 +79,42 @@ impl Cursors { async fn remove_expired_cursors(&self) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; - for (_vault_id, cursors) in vault_to_cursors.iter_mut() { + for (vault_id, cursors) in vault_to_cursors.iter_mut() { + let previous_len = cursors.len(); cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout)); + if cursors.len() != previous_len { + self.broadcast_cursors(vault_id, cursors).await; + } } + vault_to_cursors.retain(|_, cursors| !cursors.is_empty()); } - async fn broadcast_cursors(&self) { - let vault_to_cursors = self.vault_to_cursors.lock().await; - - for (vault_id, cursors) in vault_to_cursors.iter() { - self.broadcasts - .send( - vault_id.clone(), - Notification::Cursors(CursorPositionFromServer { - clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), - }), - ) - .await; - } + async fn broadcast_cursors(&self, vault_id: &str, cursors: &[ClientCursorsWithTimeToLive]) { + self.broadcasts + .send( + vault_id.to_owned(), + Notification::Cursors(CursorPositionFromServer { + clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), + }), + ) + .await; } pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { + let previous_len = cursors.len(); + cursors.retain(|c| c.client_cursors.device_id != device_id); + + if cursors.len() != previous_len { + self.broadcast_cursors(vault_id, cursors).await; + } + + if cursors.is_empty() { + vault_to_cursors.remove(vault_id); + } } } } @@ -125,3 +137,96 @@ impl ClientCursorsWithTimeToLive { self.last_updated.elapsed() > ttl } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::server_config::ServerConfig; + + #[tokio::test] + async fn disconnect_broadcasts_remaining_clients_and_the_final_empty_state() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + for device in ["one", "two"] { + cursors + .update_cursors( + "vault".to_owned(), + "user".to_owned(), + &device.to_owned(), + vec![], + ) + .await; + } + let mut receiver = broadcasts.get_receiver("vault".to_owned()).await; + let mut other = broadcasts.get_receiver("other".to_owned()).await; + cursors.remove_cursors_of_device("vault", "one").await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert_eq!(update.clients.len(), 1); + assert_eq!(update.clients[0].device_id, "two"); + + cursors.remove_cursors_of_device("vault", "two").await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(other.try_recv().is_err()); + cursors.remove_cursors_of_device("vault", "two").await; + assert!(receiver.try_recv().is_err()); + } + + #[tokio::test] + async fn expiration_broadcasts_only_changed_vaults_including_empty_ones() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let config = DatabaseConfig::default(); + let cursors = Cursors::new(&config, &broadcasts); + for (vault, device) in [("vault", "expired"), ("vault", "live"), ("other", "live")] { + cursors + .update_cursors( + vault.to_owned(), + "user".to_owned(), + &device.to_owned(), + vec![], + ) + .await; + } + cursors + .vault_to_cursors + .lock() + .await + .get_mut("vault") + .unwrap()[0] + .last_updated = std::time::Instant::now() + .checked_sub(config.cursor_timeout + Duration::from_secs(1)) + .unwrap(); + let mut receiver = broadcasts.get_receiver("vault".to_owned()).await; + let mut other = broadcasts.get_receiver("other".to_owned()).await; + cursors.remove_expired_cursors().await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert_eq!(update.clients.len(), 1); + assert_eq!(update.clients[0].device_id, "live"); + assert!(other.try_recv().is_err()); + + cursors + .vault_to_cursors + .lock() + .await + .get_mut("vault") + .unwrap()[0] + .last_updated = std::time::Instant::now() + .checked_sub(config.cursor_timeout + Duration::from_secs(1)) + .unwrap(); + cursors.remove_expired_cursors().await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(cursors.get_cursors(&"vault".to_owned()).await.is_empty()); + cursors.remove_expired_cursors().await; + assert!(receiver.try_recv().is_err()); + assert!(other.try_recv().is_err()); + } +} diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 3bd5e9c0..c0199821 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -1,9 +1,14 @@ use core::time::Duration; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; use anyhow::{Context as _, Result}; use log::info; use models::VaultId; +use sha2::{Digest, Sha256}; use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions}; use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions}; use tokio::sync::Mutex; @@ -14,7 +19,15 @@ pub mod models; mod mutations; mod queries; -use crate::config::database_config::DatabaseConfig; +#[cfg(test)] +mod tests; + +use crate::{ + config::database_config::DatabaseConfig, + utils::normalize_vault_id::{normalize_string, validate_vault_id}, +}; + +const VAULT_DATABASE_DIRECTORY: &str = "vaults"; #[derive(Clone)] struct PoolWithTimestamp { @@ -41,7 +54,10 @@ pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; impl Database { pub async fn try_new(config: &DatabaseConfig) -> Result { - tokio::fs::create_dir_all(&config.databases_directory_path) + let vault_directory = config + .databases_directory_path + .join(VAULT_DATABASE_DIRECTORY); + tokio::fs::create_dir_all(&vault_directory) .await .with_context(|| { format!( @@ -50,34 +66,25 @@ impl Database { ) })?; - let mut connection_pools = std::collections::HashMap::new(); - info!("Applying pending database migrations"); - let mut entries = tokio::fs::read_dir(&config.databases_directory_path).await?; + + // Filenames encode only a digest, so their vault names cannot be + // recovered here. Check migrations now and open pools lazily by name. + let mut entries = tokio::fs::read_dir(&vault_directory).await?; while let Some(entry) = entries.next_entry().await? { - if !entry.file_name().to_string_lossy().ends_with(".sqlite") { - continue; + if entry.file_type().await?.is_file() + && entry.path().extension().is_some_and(|ext| ext == "sqlite") + { + Self::open_database(config, &entry.path()) + .await? + .close() + .await; } - - let vault: VaultId = entry - .file_name() - .to_string_lossy() - .trim_end_matches(".sqlite") - .to_owned(); - - let pool = Self::create_vault_database(config, &vault).await?; - connection_pools.insert( - vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, - ); } let database = Self { config: config.clone(), - connection_pools: Arc::new(Mutex::new(connection_pools)), + connection_pools: Arc::default(), }; // Start background task to cleanup idle connection pools @@ -86,16 +93,21 @@ impl Database { Ok(database) } - async fn create_vault_database( - config: &DatabaseConfig, - vault: &VaultId, - ) -> Result> { - let file_name = config + fn database_path(config: &DatabaseConfig, vault: &str) -> PathBuf { + // Hash the normalized vault name into a fixed-length ASCII filename so + // filesystem case/Unicode aliasing (e.g. composed vs. decomposed é on + // macOS) cannot make separately authorized vaults share a database. + // This is only a storage filename; the user-facing vault name is unchanged. + let digest = Sha256::digest(vault.as_bytes()); + config .databases_directory_path - .join(format!("{vault}.sqlite")); + .join(VAULT_DATABASE_DIRECTORY) + .join(format!("{digest:x}.sqlite")) + } + async fn open_database(config: &DatabaseConfig, file_name: &Path) -> Result> { let connection_options = SqliteConnectOptions::new() - .filename(file_name.clone()) + .filename(file_name) .create_if_missing(true) .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Full) .busy_timeout(Duration::from_secs(3600)) @@ -126,10 +138,13 @@ impl Database { } async fn get_connection_pool(&self, vault: &VaultId) -> Result> { + let vault = normalize_string(vault); + validate_vault_id(&vault)?; let mut pools = self.connection_pools.lock().await; - if !pools.contains_key(vault) { - let pool = Self::create_vault_database(&self.config, vault).await?; + if !pools.contains_key(&vault) { + let file_name = Self::database_path(&self.config, &vault); + let pool = Self::open_database(&self.config, &file_name).await?; pools.insert( vault.clone(), PoolWithTimestamp { @@ -140,7 +155,7 @@ impl Database { } let pool_with_timestamp = pools - .get_mut(vault) + .get_mut(&vault) .expect("Pool was just inserted or already exists"); // Update last accessed time diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index 81cb6343..57c4cd94 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -22,16 +22,18 @@ pub struct StoredDocumentVersion { pub device_id: DeviceId, } -#[derive(TS, Debug, Clone, Serialize, Deserialize)] +#[derive(TS, Debug, Clone, Serialize, Deserialize, sqlx::FromRow)] #[serde(rename_all = "camelCase")] #[ts(export)] pub struct DocumentVersionWithoutContent { #[ts(as = "f64")] pub vault_update_id: VaultUpdateId, + #[sqlx(try_from = "uuid::fmt::Hyphenated")] pub document_id: DocumentId, pub updated_date: DateTime, pub user_id: UserId, pub device_id: DeviceId, + #[sqlx(try_from = "i64")] pub content_size: usize, } diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs index f9225ef4..2134a58a 100644 --- a/sync-server/src/app_state/database/queries.rs +++ b/sync-server/src/app_state/database/queries.rs @@ -1,8 +1,8 @@ use super::{ Database, Transaction, models::{ - DocumentId, EventBatch, FileManifest, StoredDocumentVersion, VaultId, VaultSnapshot, - VaultUpdateId, + DocumentId, DocumentVersionWithoutContent, EventBatch, FileManifest, StoredDocumentVersion, + VaultId, VaultSnapshot, VaultUpdateId, }, }; use anyhow::{Result, ensure}; @@ -99,8 +99,10 @@ impl Database { let head_event_id = Self::latest_event_id(&mut tx).await?; let file_manifest = Self::current_file_manifest(&mut tx).await?; - let versions = sqlx::query_as::<_, StoredDocumentVersion>( - "SELECT d.* FROM latest_document_versions d + let documents = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT d.vault_update_id, d.document_id, d.updated_date, d.user_id, d.device_id, + length(d.content) AS content_size + FROM latest_document_versions d JOIN file_manifest_entries m ON m.document_id = d.document_id WHERE m.file_manifest_id = ? ORDER BY d.vault_update_id", ) @@ -108,8 +110,6 @@ impl Database { .fetch_all(&mut *tx) .await?; - let documents = versions.into_iter().map(Into::into).collect(); - Ok(VaultSnapshot { head_event_id, file_manifest, diff --git a/sync-server/src/app_state/database/tests.rs b/sync-server/src/app_state/database/tests.rs new file mode 100644 index 00000000..3181384a --- /dev/null +++ b/sync-server/src/app_state/database/tests.rs @@ -0,0 +1,160 @@ +use super::*; +use models::{DocumentId, EventRecord, FileManifest, StoredDocumentVersion, VaultEvent}; + +fn config(directory: &tempfile::TempDir) -> DatabaseConfig { + DatabaseConfig { + databases_directory_path: directory.path().to_owned(), + ..Default::default() + } +} + +async fn insert_content( + database: &Database, + vault: &str, + document_id: DocumentId, + content: &[u8], +) -> StoredDocumentVersion { + let mut tx = database + .create_write_transaction(&vault.to_owned()) + .await + .unwrap(); + let request_id = uuid::Uuid::new_v4(); + let version = StoredDocumentVersion { + vault_update_id: Database::allocate_event(&mut tx, request_id, b"test") + .await + .unwrap(), + document_id, + updated_date: chrono::Utc::now(), + content: content.to_vec(), + user_id: "user".to_owned(), + device_id: "device".to_owned(), + }; + Database::insert_document_version(&mut tx, &version) + .await + .unwrap(); + Database::write_event( + &mut tx, + &EventRecord { + event_id: version.vault_update_id, + request_id, + event: VaultEvent::Content { + document: (&version).into(), + }, + }, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + version +} + +#[tokio::test] +async fn unicode_vaults_remain_isolated_after_reopening() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + let document = uuid::Uuid::new_v4(); + let names = ["café", "cafe\u{301}", "σ", "ς"]; + + for name in names { + assert!( + database + .get_latest_document_version(&name.to_owned(), &document, None) + .await + .unwrap() + .is_none() + ); + insert_content(&database, name, document, name.as_bytes()).await; + } + + let reopened = Database::try_new(&config).await.unwrap(); + for name in names { + let version = reopened + .get_latest_document_version(&name.to_owned(), &document, None) + .await + .unwrap() + .unwrap(); + assert_eq!(version.content, name.as_bytes()); + } +} + +#[tokio::test] +async fn rejects_invalid_vault_names_at_the_database_boundary() { + let directory = tempfile::tempdir().unwrap(); + let config = DatabaseConfig { + databases_directory_path: directory.path().join("databases"), + ..Default::default() + }; + let database = Database::try_new(&config).await.unwrap(); + for name in [ + "", + " ", + ".", + "..", + "../escaped", + "/absolute", + "a/b", + "a\\b", + "a\0b", + ] { + assert!( + database.vault_snapshot(&name.to_owned()).await.is_err(), + "accepted {name:?}" + ); + } + assert!(!directory.path().join("escaped.sqlite").exists()); + assert!(database.connection_pools.lock().await.is_empty()); +} + +#[tokio::test] +async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "snapshot".to_owned(); + let text_id = uuid::Uuid::new_v4(); + let binary_id = uuid::Uuid::new_v4(); + insert_content(&database, &vault, text_id, b"old").await; + let binary = insert_content(&database, &vault, binary_id, &[0, 255, 0, 128]).await; + let text = insert_content(&database, &vault, text_id, "café 🌍".as_bytes()).await; + insert_content(&database, &vault, uuid::Uuid::new_v4(), b"unreferenced").await; + + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + let request_id = uuid::Uuid::new_v4(); + let manifest = FileManifest { + file_manifest_id: Database::allocate_event(&mut tx, request_id, b"manifest") + .await + .unwrap(), + entries: [ + (text_id, "text.md".to_owned()), + (binary_id, "binary.dat".to_owned()), + ] + .into(), + }; + Database::insert_file_manifest(&mut tx, &manifest) + .await + .unwrap(); + Database::write_event( + &mut tx, + &EventRecord { + event_id: manifest.file_manifest_id, + request_id, + event: VaultEvent::FileManifest { + file_manifest: manifest.clone(), + }, + }, + ) + .await + .unwrap(); + tx.commit().await.unwrap(); + + let snapshot = database.vault_snapshot(&vault).await.unwrap(); + assert_eq!(snapshot.head_event_id, manifest.file_manifest_id); + assert_eq!(snapshot.file_manifest.entries, manifest.entries); + assert_eq!(snapshot.documents.len(), 2); + for (actual, expected) in snapshot.documents.iter().zip([binary, text]) { + assert_eq!( + serde_json::to_value(actual).unwrap(), + serde_json::to_value(models::DocumentVersionWithoutContent::from(expected)).unwrap() + ); + } +} diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index 4a9da0f4..59d1480f 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -7,7 +7,7 @@ use crate::consts::{ DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS, }; -#[derive(Debug, Deserialize, Serialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct ServerConfig { #[serde(default = "default_host")] pub host: String, @@ -28,6 +28,19 @@ pub struct ServerConfig { pub mergeable_file_extensions: Vec, } +impl Default for ServerConfig { + fn default() -> Self { + Self { + host: default_host(), + port: default_port(), + max_body_size_mb: default_max_body_size_mb(), + max_clients_per_vault: default_max_clients_per_vault(), + response_timeout: default_response_timeout(), + mergeable_file_extensions: default_mergeable_file_extensions(), + } + } +} + fn default_host() -> String { debug!("Using default server host: {DEFAULT_HOST}"); DEFAULT_HOST.to_owned() @@ -60,3 +73,36 @@ fn default_mergeable_file_extensions() -> Vec { .map(|s| (*s).to_owned()) .collect() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + + #[tokio::test] + async fn fresh_and_omitted_server_configs_use_the_documented_defaults() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("config.yml"); + let fresh = Config::read_or_create(&path).await.unwrap(); + let reread = Config::load_from_file(&path).await.unwrap(); + let omitted: Config = serde_yaml::from_str("{}").unwrap(); + let explicit: Config = serde_yaml::from_str("server: {}").unwrap(); + for config in [fresh, reread, omitted, explicit] { + assert_eq!(config.server.host, DEFAULT_HOST); + assert_eq!(config.server.port, DEFAULT_PORT); + assert_eq!(config.server.max_body_size_mb, DEFAULT_MAX_BODY_SIZE_MB); + assert_eq!( + config.server.max_clients_per_vault, + DEFAULT_MAX_CLIENTS_PER_VAULT + ); + assert_eq!( + config.server.response_timeout, + DEFAULT_RESPONSE_TIMEOUT_SECONDS + ); + assert_eq!( + config.server.mergeable_file_extensions, + DEFAULT_MERGEABLE_FILE_EXTENSIONS + ); + } + } +} diff --git a/sync-server/src/config/user_config.rs b/sync-server/src/config/user_config.rs index 8b2537f0..0bb3379b 100644 --- a/sync-server/src/config/user_config.rs +++ b/sync-server/src/config/user_config.rs @@ -2,7 +2,10 @@ use bimap::BiHashMap; use rand::{Rng, distr::Alphanumeric, rng}; use serde::{Deserialize, Deserializer, Serialize, de::Error}; -use crate::app_state::database::models::VaultId; +use crate::{ + app_state::database::models::VaultId, + utils::normalize_vault_id::{normalize_string, validate_vault_id}, +}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct UserConfig { @@ -71,9 +74,23 @@ pub enum VaultAccess { #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct AllowListedVaults { + #[serde(deserialize_with = "deserialize_allowed_vaults")] pub allowed: Vec, } +fn deserialize_allowed_vaults<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Vec::::deserialize(deserializer)? + .into_iter() + .map(|name| { + let vault = normalize_string(&name); + validate_vault_id(&vault).map_err(D::Error::custom)?; + Ok(vault) + }) + .collect() +} + fn default_users() -> Vec { vec![User { name: "admin".to_owned(), @@ -95,6 +112,24 @@ mod tests { use super::*; + #[test] + fn allowlists_normalize_case_and_whitespace_but_preserve_unicode_identity() { + let list: AllowListedVaults = serde_json::from_value(json!({ + "allowed": [" MyVault ", "CAFÉ", "CAFE\u{301}"] + })) + .unwrap(); + assert_eq!(list.allowed, ["myvault", "café", "cafe\u{301}"]); + } + + #[test] + fn allowlists_reject_invalid_vault_names() { + for name in ["", " ", ".", "..", "../escaped", "a/b", "a\\b", "a\0b"] { + assert!( + serde_json::from_value::(json!({"allowed": [name]})).is_err() + ); + } + } + #[test] fn test_validate_users_unique_names_and_tokens() { let config_json = json!({ diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 27ff1155..28137746 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -4,6 +4,9 @@ mod endpoints; pub(crate) mod requests; pub(crate) mod responses; +#[cfg(test)] +mod tests; + use anyhow::{Context as _, Result}; use auth::auth_middleware; use axum::{ diff --git a/sync-server/src/server/auth.rs b/sync-server/src/server/auth.rs index e56f4acc..c601ef54 100644 --- a/sync-server/src/server/auth.rs +++ b/sync-server/src/server/auth.rs @@ -14,8 +14,8 @@ use log::info; use crate::{ app_state::{AppState, database::models::VaultId}, config::user_config::{AllowListedVaults, User, VaultAccess}, - errors::{SyncServerError, permission_denied_error, unauthenticated_error}, - utils::normalize::normalize_string, + errors::{SyncServerError, client_error, permission_denied_error, unauthenticated_error}, + utils::normalize_vault_id::{normalize_string, validate_vault_id}, }; pub async fn auth_middleware( @@ -40,6 +40,7 @@ pub async fn auth_middleware( } pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result { + validate_vault_id(vault_id).map_err(client_error)?; let user = state .config .users diff --git a/sync-server/src/server/endpoints/events.rs b/sync-server/src/server/endpoints/events.rs index ef113864..d73367df 100644 --- a/sync-server/src/server/endpoints/events.rs +++ b/sync-server/src/server/endpoints/events.rs @@ -11,12 +11,12 @@ use crate::{ database::models::{EventBatch, VaultId}, }, errors::{SyncServerError, client_error, server_error}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct EventsPath { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, } diff --git a/sync-server/src/server/endpoints/fetch_document_version_content.rs b/sync-server/src/server/endpoints/fetch_document_version_content.rs index 9fdd0ad8..ed9d3ae1 100644 --- a/sync-server/src/server/endpoints/fetch_document_version_content.rs +++ b/sync-server/src/server/endpoints/fetch_document_version_content.rs @@ -12,12 +12,12 @@ use crate::{ database::models::{DocumentId, VaultId, VaultUpdateId}, }, errors::{SyncServerError, not_found_error, server_error}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct FetchDocumentVersionContentPathParams { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, document_id: DocumentId, diff --git a/sync-server/src/server/endpoints/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs index a6b662e9..a05f632f 100644 --- a/sync-server/src/server/endpoints/fetch_latest_document_version.rs +++ b/sync-server/src/server/endpoints/fetch_latest_document_version.rs @@ -12,12 +12,12 @@ use crate::{ database::models::{DocumentId, DocumentVersion, VaultId}, }, errors::{SyncServerError, not_found_error, server_error}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct FetchLatestDocumentVersionPathParams { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, document_id: DocumentId, diff --git a/sync-server/src/server/endpoints/get_file_manifest.rs b/sync-server/src/server/endpoints/get_file_manifest.rs index cfafda02..d844ff41 100644 --- a/sync-server/src/server/endpoints/get_file_manifest.rs +++ b/sync-server/src/server/endpoints/get_file_manifest.rs @@ -14,12 +14,12 @@ use crate::{ }, }, errors::{SyncServerError, server_error}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct GetFileManifestPath { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, } diff --git a/sync-server/src/server/endpoints/ping.rs b/sync-server/src/server/endpoints/ping.rs index c42219be..8e910feb 100644 --- a/sync-server/src/server/endpoints/ping.rs +++ b/sync-server/src/server/endpoints/ping.rs @@ -14,12 +14,12 @@ use crate::{ app_state::{AppState, database::models::VaultId}, consts::SUPPORTED_API_VERSION, errors::SyncServerError, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct PingPathParams { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, } diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index b09ff536..26c9cfa3 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -14,7 +14,7 @@ use crate::{ requests::{PushContent, PutFileContent}, responses::DocumentUpdateResponse, }, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; use anyhow::anyhow; use axum::{ @@ -30,7 +30,7 @@ use sha2::{Digest, Sha256}; #[derive(Deserialize)] pub struct PutFileContentPath { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, document_id: DocumentId, } diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs index db99c374..667c7827 100644 --- a/sync-server/src/server/endpoints/put_file_manifest.rs +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -18,12 +18,12 @@ use crate::{ }, errors::{SyncServerError, client_error, server_error}, server::{requests::PushFileManifest, responses::FileManifestUpdateResponse}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct PutFileManifestPath { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, } diff --git a/sync-server/src/server/endpoints/vault_snapshot.rs b/sync-server/src/server/endpoints/vault_snapshot.rs index 552517c5..1d90f9e0 100644 --- a/sync-server/src/server/endpoints/vault_snapshot.rs +++ b/sync-server/src/server/endpoints/vault_snapshot.rs @@ -11,12 +11,12 @@ use crate::{ database::models::{VaultId, VaultSnapshot}, }, errors::{SyncServerError, server_error}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; #[derive(Deserialize)] pub struct VaultSnapshotPath { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, } diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index 2be4b8f2..6d7f4a10 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -9,7 +9,7 @@ use crate::{ }, }, errors::{SyncServerError, client_error, server_error}, - utils::normalize::normalize, + utils::normalize_vault_id::normalize_vault_id, }; use axum::{ extract::{ @@ -26,7 +26,7 @@ use tokio::sync::broadcast::error::RecvError; #[derive(Deserialize)] pub struct WebSocketPathParams { - #[serde(deserialize_with = "normalize")] + #[serde(deserialize_with = "normalize_vault_id")] vault_id: VaultId, } diff --git a/sync-server/src/server/tests.rs b/sync-server/src/server/tests.rs new file mode 100644 index 00000000..493e5283 --- /dev/null +++ b/sync-server/src/server/tests.rs @@ -0,0 +1,121 @@ +use super::*; +use axum::{ + body::{Body, to_bytes}, + http::StatusCode, +}; +use serde_json::{Value, json}; +use tower::ServiceExt; + +async fn app(directory: &tempfile::TempDir) -> Router { + let mut config: Config = serde_json::from_value(json!({ + "users": { "user_configs": [ + { "name": "admin", "token": "admin", "vault_access": { "type": "allow_access_to_all" } }, + { "name": "mixed", "token": "mixed", "vault_access": { "type": "allow_list", "allowed": [" MyVault "] } }, + { "name": "composed", "token": "composed", "vault_access": { "type": "allow_list", "allowed": ["café"] } }, + { "name": "decomposed", "token": "decomposed", "vault_access": { "type": "allow_list", "allowed": ["cafe\u{301}"] } } + ] } + })).unwrap(); + config.database.databases_directory_path = directory.path().join("databases"); + let state = AppState::try_new(config).await.unwrap(); + get_authed_routes(state.clone()).with_state(state) +} + +async fn request(app: Router, path: &str, token: &str, body: Option) -> (StatusCode, Value) { + let response = app + .oneshot( + Request::builder() + .uri(path) + .method(if body.is_some() { + Method::PUT + } else { + Method::GET + }) + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .header(http::header::CONTENT_TYPE, "application/json") + .header("Device-Id", "test") + .body(body.map_or_else(Body::empty, |value| Body::from(value.to_string()))) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) +} + +#[tokio::test] +async fn http_rejects_encoded_traversal_and_accepts_normalized_allowlists() { + let directory = tempfile::tempdir().unwrap(); + let app = app(&directory).await; + for vault in [ + "..%2Fescaped", + "%2Fabsolute", + "a%5Cb", + "%2E%2E", + "%20", + "a%00b", + ] { + let (status, _) = request( + app.clone(), + &format!("/vaults/{vault}/vault_snapshot"), + "admin", + None, + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST, "vault: {vault}"); + } + assert!(!directory.path().join("escaped.sqlite").exists()); + for vault in ["MyVault", "myvault", "%20MYVAULT%20"] { + let (status, _) = request( + app.clone(), + &format!("/vaults/{vault}/file_manifest"), + "mixed", + None, + ) + .await; + assert_eq!(status, StatusCode::OK, "vault: {vault}"); + } + assert_eq!( + request(app, "/vaults/other/file_manifest", "mixed", None) + .await + .0, + StatusCode::FORBIDDEN + ); +} + +#[tokio::test] +async fn http_keeps_unicode_vaults_separate_across_restart() { + let directory = tempfile::tempdir().unwrap(); + let router = app(&directory).await; + let document = uuid::Uuid::new_v4(); + let composed = format!("/vaults/caf%C3%A9/documents/{document}"); + let decomposed = format!("/vaults/cafe%CC%81/documents/{document}"); + let push = json!({ + "requestId": uuid::Uuid::new_v4(), "parentVersionId": null, + "content": { "type": "Snapshot", "value": "aGVsbG8=" } + }); + assert_eq!( + request(router, &composed, "composed", Some(push)).await.0, + StatusCode::OK + ); + + let reopened = app(&directory).await; + assert_eq!( + request(reopened.clone(), &composed, "decomposed", None) + .await + .0, + StatusCode::FORBIDDEN + ); + assert_eq!( + request(reopened.clone(), &decomposed, "decomposed", None) + .await + .0, + StatusCode::NOT_FOUND + ); + let (status, version) = request(reopened, &composed, "composed", None).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(version["contentBase64"], "aGVsbG8="); +} diff --git a/sync-server/src/utils.rs b/sync-server/src/utils.rs index 4798e39c..a652bec6 100644 --- a/sync-server/src/utils.rs +++ b/sync-server/src/utils.rs @@ -1,4 +1,4 @@ pub mod log_level; -pub mod normalize; +pub mod normalize_vault_id; pub mod portable_path; pub mod rotating_file_writer; diff --git a/sync-server/src/utils/normalize.rs b/sync-server/src/utils/normalize.rs deleted file mode 100644 index 6553dd25..00000000 --- a/sync-server/src/utils/normalize.rs +++ /dev/null @@ -1,13 +0,0 @@ -use serde::{Deserialize, Deserializer}; - -pub fn normalize<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - Ok(normalize_string(&s)) -} - -pub fn normalize_string(s: &str) -> String { - s.trim().to_lowercase() -} diff --git a/sync-server/src/utils/normalize_vault_id.rs b/sync-server/src/utils/normalize_vault_id.rs new file mode 100644 index 00000000..7cabf3e3 --- /dev/null +++ b/sync-server/src/utils/normalize_vault_id.rs @@ -0,0 +1,29 @@ +use anyhow::{Result, ensure}; +use serde::{Deserialize, Deserializer, de::Error}; + +pub fn normalize_vault_id<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + let vault = normalize_string(&s); + validate_vault_id(&vault).map_err(D::Error::custom)?; + Ok(vault) +} + +pub fn normalize_string(s: &str) -> String { + s.trim().to_lowercase() +} + +pub fn validate_vault_id(vault: &str) -> Result<()> { + ensure!( + !vault.is_empty() + && vault != "." + && vault != ".." + && !vault + .chars() + .any(|c| c.is_control() || matches!(c, '/' | '\\')), + "Vault names must be nonempty names without path separators, traversal components, or control characters" + ); + Ok(()) +} -- 2.54.0 From 2536f09cd70c9197d7ea10d78c1278d36bad4b19 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 19:51:24 +0100 Subject: [PATCH 18/48] Remove clutter --- .../sync-operations/unrestricted-syncer.ts | 596 ------------------ .../src/utils/find-matching-file.ts | 14 - 2 files changed, 610 deletions(-) delete mode 100644 frontend/sync-client/src/sync-operations/unrestricted-syncer.ts delete mode 100644 frontend/sync-client/src/utils/find-matching-file.ts diff --git a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts b/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts deleted file mode 100644 index e3964d30..00000000 --- a/frontend/sync-client/src/sync-operations/unrestricted-syncer.ts +++ /dev/null @@ -1,596 +0,0 @@ -import type { - Database, - DocumentRecord, - RelativePath -} from "../persistence/database"; - -import { diff } from "reconcile-text"; -import type { SyncService } from "../services/sync-service"; -import type { Logger } from "../tracing/logger"; -import type { - CommonHistoryEntry, - SyncCreateDetails, - SyncDeleteDetails, - SyncDetails, - SyncHistory, - SyncMovedDetails, - SyncUpdateDetails -} from "../tracing/sync-history"; -import { SyncStatus, SyncType } from "../tracing/sync-history"; -import { EMPTY_HASH, hash } from "../utils/hash"; - -import { base64ToBytes } from "byte-base64"; -import type { Settings } from "../persistence/settings"; -import type { FileOperations } from "../file-operations/file-operations"; -import { createPromise } from "../utils/create-promise"; -import { FileNotFoundError } from "../file-operations/file-not-found-error"; -import { SyncResetError } from "../services/sync-reset-error"; -import { globsToRegexes } from "../utils/globs-to-regexes"; -import type { DocumentVersion } from "../services/types/DocumentVersion"; -import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; -import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; -import { isFileTypeMergable } from "../utils/is-file-type-mergable"; -import { isBinary } from "../utils/is-binary"; -import type { ServerConfig } from "../services/server-config"; - -export class UnrestrictedSyncer { - private ignorePatterns: RegExp[]; - - public constructor( - private readonly logger: Logger, - private readonly database: Database, - private readonly settings: Settings, - private readonly syncService: SyncService, - private readonly operations: FileOperations, - private readonly history: SyncHistory, - private readonly contentCache: FixedSizeDocumentCache, - private readonly serverConfig: ServerConfig - ) { - this.ignorePatterns = globsToRegexes( - this.settings.getSettings().ignorePatterns, - this.logger - ); - - this.settings.onSettingsChanged.add((newSettings) => { - this.ignorePatterns = globsToRegexes( - newSettings.ignorePatterns, - this.logger - ); - }); - } - - public async unrestrictedSyncLocallyCreatedFile( - document: DocumentRecord - ): Promise { - const updateDetails: SyncCreateDetails = { - type: SyncType.CREATE, - relativePath: document.relativePath - }; - - return this.executeSync(updateDetails, async () => { - const originalRelativePath = document.relativePath; - if (document.isDeleted) { - this.logger.debug( - `Document ${originalRelativePath} has been already deleted, no need to create it` - ); - return; - } - - const contentBytes = - await this.operations.read(originalRelativePath); // this can throw FileNotFoundError - const contentHash = hash(contentBytes); - - const response = await this.syncService.create({ - documentId: document.documentId, - relativePath: originalRelativePath, - contentBytes - }); - - // In case a document with the same name (but different ID) had existed remotely that we haven't known about - if (response.relativePath != originalRelativePath) { - this.logger.debug( - `Document ${originalRelativePath} has been created remotely at a different path: ${response.relativePath}, moving it locally` - ); - await this.operations.move( - document.relativePath, - response.relativePath - ); // this can throw FileNotFoundError - } - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - - this.database.addSeenUpdateId(response.vaultUpdateId); - await this.updateCache( - response.vaultUpdateId, - contentBytes, - response.relativePath - ); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully uploaded locally created file` - }); - }); - } - - public async unrestrictedSyncLocallyDeletedFile( - document: DocumentRecord - ): Promise { - const updateDetails: SyncDeleteDetails = { - type: SyncType.DELETE, - relativePath: document.relativePath - }; - - await this.executeSync(updateDetails, async () => { - const response = await this.syncService.delete({ - documentId: document.documentId, - relativePath: document.relativePath - }); - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: document.relativePath - }, - document - ); - - this.database.addSeenUpdateId(response.vaultUpdateId); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully deleted locally deleted file on the server`, - author: response.userId - }); - }); - } - - public async unrestrictedSyncLocallyUpdatedFile({ - oldPath, - document, - // We use the same code path for both local and remote updates. We need to force the update - // if there are no local changes but we know that the remote version is newer. - force = false - }: { - oldPath?: RelativePath; - force?: boolean; - document: DocumentRecord; - }): Promise { - const updateDetails: SyncUpdateDetails | SyncMovedDetails = - oldPath !== undefined - ? { - type: SyncType.MOVE, - relativePath: document.relativePath, - movedFrom: oldPath - } - : { - type: SyncType.UPDATE, - relativePath: document.relativePath - }; - - await this.executeSync(updateDetails, async () => { - const originalRelativePath = document.relativePath; - - if (document.isDeleted || document.metadata === undefined) { - this.logger.debug( - `Document ${document.relativePath} has been already deleted, no need to update it` - ); - return; - } - - const contentBytes = await this.operations.read( - document.relativePath - ); // this can throw FileNotFoundError - let contentHash = hash(contentBytes); - - const areThereLocalChanges = !( - document.metadata.hash === contentHash && oldPath === undefined - ); - - let response: DocumentVersion | DocumentUpdateResponse | undefined = - undefined; - - if (areThereLocalChanges) { - const isText = - !isBinary(contentBytes) && - isFileTypeMergable( - document.relativePath, - (await this.serverConfig.getConfig()) - .mergeableFileExtensions - ); - const cachedVersion = this.contentCache.get( - document.metadata.parentVersionId - ); - - response = - isText && cachedVersion !== undefined - ? await this.syncService.putText({ - documentId: document.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - content: diff( - new TextDecoder().decode(cachedVersion), - new TextDecoder().decode(contentBytes) - ) - }) - : await this.syncService.putBinary({ - documentId: document.documentId, - parentVersionId: - document.metadata.parentVersionId, - relativePath: document.relativePath, - contentBytes - }); - } else { - if (!force) { - this.logger.debug( - `File hash of ${document.relativePath} matches with last synced version and the path hasn't changed; no need to sync` - ); - return; - } - - response = await this.syncService.get({ - documentId: document.documentId - }); - } - - // `document` is mutable and reflects the latest state in the local database - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (document.isDeleted) { - this.logger.info( - `Document ${document.relativePath} has been deleted before we could finish updating it` - ); - this.database.addSeenUpdateId(response.vaultUpdateId); - return; - } - - if ( - // `Syncer` creates fake local document metadata for all remote docs with invalid hashes. The parent IDs will likely match - // the latest versions so we still need to update the local versions to turn the fakes into real metadata. - document.metadata.parentVersionId > response.vaultUpdateId - ) { - this.logger.debug( - `Document ${document.relativePath} is already more up to date than the fetched version` - ); - this.database.addSeenUpdateId(response.vaultUpdateId); // in case the previous `vaultUpdateId` update hasn't made it through - return; - } - - if (response.isDeleted) { - return this.applyRemoteDeleteLocally(document, response); - } - - let actualPath = document.relativePath; - - if (response.relativePath != originalRelativePath) { - actualPath = response.relativePath; - // Make sure to update the remote relative path to avoid uploading - // the file as a result of this filesystem event. - document.metadata.remoteRelativePath = response.relativePath; - await this.operations.move( - document.relativePath, - response.relativePath - ); // this can throw FileNotFoundError - } - - if (!("type" in response) || response.type === "MergingUpdate") { - const responseBytes = base64ToBytes(response.contentBase64); - contentHash = hash(responseBytes); - - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - await this.operations.write( - actualPath, - contentBytes, - responseBytes - ); - await this.updateCache( - response.vaultUpdateId, - responseBytes, - actualPath - ); - - if (!force) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `The file we updated had been updated remotely, so we downloaded the merged version` - }); - } - } else { - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: contentHash, - remoteRelativePath: response.relativePath - }, - document - ); - await this.updateCache( - response.vaultUpdateId, - contentBytes, - actualPath - ); - } - - this.database.addSeenUpdateId(response.vaultUpdateId); - - const actualUpdateDetails: SyncUpdateDetails | SyncMovedDetails = - oldPath !== undefined || - response.relativePath != originalRelativePath - ? { - type: SyncType.MOVE, - relativePath: response.relativePath, - movedFrom: originalRelativePath - } - : { - type: SyncType.UPDATE, - relativePath: response.relativePath - }; - - if (areThereLocalChanges) { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: actualUpdateDetails, - message: `Successfully uploaded locally updated file to the server`, - author: response.userId - }); - } else { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: actualUpdateDetails, - message: `Successfully downloaded remotely updated file from the server`, - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - } - }); - } - - public async unrestrictedSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent, - document?: DocumentRecord - ): Promise { - const updateDetails: SyncCreateDetails = { - type: SyncType.CREATE, - relativePath: remoteVersion.relativePath - }; - - await this.executeSync(updateDetails, async () => { - if (document?.metadata !== undefined) { - // If the file exists locally, let's pretend the user has updated it - // and deal with remote update/deletion within `unrestrictedSyncLocallyUpdatedFile` - if ( - document.metadata.parentVersionId >= - remoteVersion.vaultUpdateId - ) { - this.logger.debug( - `Document ${remoteVersion.relativePath} is already at least as up to date as the fetched version` - ); - - return; - } - - return this.unrestrictedSyncLocallyUpdatedFile({ - document, - force: true - }); - } else if (remoteVersion.isDeleted) { - // Either the document hasn't made it to us before and therefore we don't need to delete it, - // or we already have it, in which case the preceeding if would've dealt with it - this.logger.debug( - `Document ${remoteVersion.relativePath} has been deleted remotely, no need to sync` - ); - return; - } - - // Don't download oversized files - const historyEntryForSkippedOversizedFile = - this.getHistoryEntryForSkippedOversizedFile( - remoteVersion.contentSize, - remoteVersion.relativePath - ); - if (historyEntryForSkippedOversizedFile !== undefined) { - this.history.addHistoryEntry( - historyEntryForSkippedOversizedFile - ); - return; - } - - const contentBytes = - await this.syncService.getDocumentVersionContent({ - documentId: remoteVersion.documentId, - vaultUpdateId: remoteVersion.vaultUpdateId - }); - - // We're trying to create an entirely new document that didn't exist locally - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - // It can happen that a concurrent sync operation has already created the document, so we can bail here - if (document !== undefined) { - this.logger.debug( - `Document ${remoteVersion.relativePath} has already been created locally, no need to create it again` - ); - return; - } - - await this.operations.ensureClearPath(remoteVersion.relativePath); - - const [promise, resolve] = createPromise(); - this.database.updateDocumentMetadata( - { - parentVersionId: remoteVersion.vaultUpdateId, - hash: hash(contentBytes), - remoteRelativePath: remoteVersion.relativePath - }, - this.database.createNewPendingDocument( - remoteVersion.documentId, - remoteVersion.relativePath, - promise - ) - ); - - await this.operations.create( - remoteVersion.relativePath, - contentBytes - ); - await this.updateCache( - remoteVersion.vaultUpdateId, - contentBytes, - remoteVersion.relativePath - ); - - resolve(); - this.database.removeDocumentPromise(promise); - - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: updateDetails, - message: `Successfully downloaded remote file which hadn't existed locally`, - author: remoteVersion.userId, - timestamp: new Date(remoteVersion.updatedDate) - }); - }); - } - - public async executeSync( - details: SyncDetails, - fn: () => Promise - ): Promise { - for (const pattern of this.ignorePatterns) { - if (pattern.test(details.relativePath)) { - this.logger.debug( - `File '${details.relativePath}' is ignored by the ignore pattern: ${pattern}` - ); - return; // bail without SKIPPED status because we were told to ignore this file and we shouldn't clutter up the history - } - } - - try { - // Only check the size of files which already exist locally. - if (await this.operations.exists(details.relativePath)) { - const sizeInBytes = await this.operations.getFileSize( - details.relativePath - ); - const historyEntryForSkippedOversizedFile = - this.getHistoryEntryForSkippedOversizedFile( - sizeInBytes, - details.relativePath - ); - if (historyEntryForSkippedOversizedFile !== undefined) { - this.history.addHistoryEntry( - historyEntryForSkippedOversizedFile - ); - return; - } - } - - return await fn(); - } catch (e) { - if (e instanceof FileNotFoundError) { - // A subsequent sync operation must have been creating to deal with this - this.logger.info( - `Skiping file '${details.relativePath}' because it no longer exists when trying to ${details.type.toLocaleLowerCase()} it` - ); - return; - } - if (e instanceof SyncResetError) { - this.logger.info( - `Interrupting sync operation because of a reset` - ); - return; - } else { - this.history.addHistoryEntry({ - status: SyncStatus.ERROR, - details, - message: `Failed to sync file '${details.relativePath}' because of ${e} when trying to ${details.type.toLocaleLowerCase()} it` - }); - throw e; - } - } - } - - private getHistoryEntryForSkippedOversizedFile( - sizeInBytes: number, - relativePath: RelativePath - ): CommonHistoryEntry | undefined { - const sizeInMB = Math.round(sizeInBytes / 1024 / 1024); - const { maxFileSizeMB } = this.settings.getSettings(); - if (sizeInMB > maxFileSizeMB) { - return { - status: SyncStatus.SKIPPED, - details: { - type: SyncType.SKIPPED, - relativePath - }, - message: `File size of ${sizeInMB} MB exceeds the maximum file size limit of ${ - maxFileSizeMB - } MB` - }; - } - } - - private async updateCache( - updateId: number, - contentBytes: Uint8Array, - filePath: RelativePath - ): Promise { - if ( - isFileTypeMergable( - filePath, - (await this.serverConfig.getConfig()).mergeableFileExtensions - ) && - !isBinary(contentBytes) - ) { - this.contentCache.put(updateId, contentBytes); - } - } - - private async applyRemoteDeleteLocally( - document: DocumentRecord, - response: DocumentVersion | DocumentUpdateResponse - ): Promise { - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - details: { - type: SyncType.DELETE, - relativePath: document.relativePath - }, - message: "File has been deleted remotely, so we deleted it locally", - author: response.userId, - timestamp: new Date(response.updatedDate) - }); - - this.database.delete(document.relativePath); - this.database.updateDocumentMetadata( - { - parentVersionId: response.vaultUpdateId, - hash: EMPTY_HASH, - remoteRelativePath: response.relativePath - }, - document - ); - - await this.operations.delete(document.relativePath); - - this.database.addSeenUpdateId(response.vaultUpdateId); - } -} diff --git a/frontend/sync-client/src/utils/find-matching-file.ts b/frontend/sync-client/src/utils/find-matching-file.ts deleted file mode 100644 index c3d323d3..00000000 --- a/frontend/sync-client/src/utils/find-matching-file.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { DocumentRecord } from "../persistence/database"; -import { EMPTY_HASH } from "./hash"; - -// TODO: make this smarter so that offline files can be renamed & edited at the same time -export function findMatchingFile( - contentHash: string, - candidates: DocumentRecord[] -): DocumentRecord | undefined { - if (contentHash === EMPTY_HASH) { - return undefined; - } - - return candidates.find(({ metadata }) => metadata?.hash === contentHash); -} -- 2.54.0 From 2a7a19fdf0494b32106770c79fdd2cbdc68b103a Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 19:58:57 +0100 Subject: [PATCH 19/48] Remove path repetition --- sync-server/src/server/endpoints.rs | 16 ++++++++ sync-server/src/server/endpoints/events.rs | 19 +++------- .../fetch_document_version_content.rs | 24 ++++-------- .../fetch_latest_document_version.rs | 18 ++------- .../src/server/endpoints/get_file_manifest.rs | 20 +++------- sync-server/src/server/endpoints/ping.rs | 17 ++------- .../src/server/endpoints/put_file_content.rs | 38 +++++-------------- .../src/server/endpoints/put_file_manifest.rs | 25 ++++-------- .../src/server/endpoints/vault_snapshot.rs | 20 +++------- sync-server/src/server/endpoints/websocket.rs | 17 ++------- 10 files changed, 65 insertions(+), 149 deletions(-) diff --git a/sync-server/src/server/endpoints.rs b/sync-server/src/server/endpoints.rs index ed873473..23323a69 100644 --- a/sync-server/src/server/endpoints.rs +++ b/sync-server/src/server/endpoints.rs @@ -11,3 +11,19 @@ pub mod put_file_manifest; mod utils; pub mod vault_snapshot; pub mod websocket; + +use serde::{Deserialize, Deserializer}; + +use crate::{app_state::database::models::VaultId, utils::normalize_vault_id::normalize_vault_id}; + +#[derive(Debug)] +pub struct VaultPath(pub VaultId); + +impl<'de> Deserialize<'de> for VaultPath { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + normalize_vault_id(deserializer).map(Self) + } +} diff --git a/sync-server/src/server/endpoints/events.rs b/sync-server/src/server/endpoints/events.rs index d73367df..2505afba 100644 --- a/sync-server/src/server/endpoints/events.rs +++ b/sync-server/src/server/endpoints/events.rs @@ -5,21 +5,12 @@ use axum::{ use log::debug; use serde::Deserialize; +use super::VaultPath; use crate::{ - app_state::{ - AppState, - database::models::{EventBatch, VaultId}, - }, + app_state::{AppState, database::models::EventBatch}, errors::{SyncServerError, client_error, server_error}, - utils::normalize_vault_id::normalize_vault_id, }; -#[derive(Deserialize)] -pub struct EventsPath { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, -} - #[derive(Deserialize)] pub struct EventsQuery { pub after: i64, @@ -27,13 +18,13 @@ pub struct EventsQuery { #[axum::debug_handler] pub async fn events( - Path(path): Path, + Path(VaultPath(vault_id)): Path, Query(query): Query, State(state): State, ) -> Result, SyncServerError> { debug!( "Fetching events after `{}` for vault `{}`", - query.after, path.vault_id + query.after, vault_id ); if query.after < 0 { @@ -42,7 +33,7 @@ pub async fn events( state .database - .events_after(&path.vault_id, query.after) + .events_after(&vault_id, query.after) .await .map(Json) .map_err(server_error) diff --git a/sync-server/src/server/endpoints/fetch_document_version_content.rs b/sync-server/src/server/endpoints/fetch_document_version_content.rs index ed9d3ae1..9e981c5f 100644 --- a/sync-server/src/server/endpoints/fetch_document_version_content.rs +++ b/sync-server/src/server/endpoints/fetch_document_version_content.rs @@ -4,33 +4,23 @@ use axum::{ extract::{Path, State}, }; use log::debug; -use serde::Deserialize; +use super::VaultPath; use crate::{ app_state::{ AppState, - database::models::{DocumentId, VaultId, VaultUpdateId}, + database::models::{DocumentId, VaultUpdateId}, }, errors::{SyncServerError, not_found_error, server_error}, - utils::normalize_vault_id::normalize_vault_id, }; -#[derive(Deserialize)] -pub struct FetchDocumentVersionContentPathParams { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, - - document_id: DocumentId, - vault_update_id: VaultUpdateId, -} - #[axum::debug_handler] pub async fn fetch_document_version_content( - Path(FetchDocumentVersionContentPathParams { - vault_id, - document_id, - vault_update_id, - }): Path, + Path((VaultPath(vault_id), document_id, vault_update_id)): Path<( + VaultPath, + DocumentId, + VaultUpdateId, + )>, State(state): State, ) -> Result { debug!( diff --git a/sync-server/src/server/endpoints/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs index a05f632f..870ac669 100644 --- a/sync-server/src/server/endpoints/fetch_latest_document_version.rs +++ b/sync-server/src/server/endpoints/fetch_latest_document_version.rs @@ -4,31 +4,19 @@ use axum::{ extract::{Path, State}, }; use log::debug; -use serde::Deserialize; +use super::VaultPath; use crate::{ app_state::{ AppState, - database::models::{DocumentId, DocumentVersion, VaultId}, + database::models::{DocumentId, DocumentVersion}, }, errors::{SyncServerError, not_found_error, server_error}, - utils::normalize_vault_id::normalize_vault_id, }; -#[derive(Deserialize)] -pub struct FetchLatestDocumentVersionPathParams { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, - - document_id: DocumentId, -} - #[axum::debug_handler] pub async fn fetch_latest_document_version( - Path(FetchLatestDocumentVersionPathParams { - vault_id, - document_id, - }): Path, + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, State(state): State, ) -> Result, SyncServerError> { debug!("Fetching latest document version for document `{document_id}` in vault `{vault_id}`"); diff --git a/sync-server/src/server/endpoints/get_file_manifest.rs b/sync-server/src/server/endpoints/get_file_manifest.rs index d844ff41..3f706696 100644 --- a/sync-server/src/server/endpoints/get_file_manifest.rs +++ b/sync-server/src/server/endpoints/get_file_manifest.rs @@ -3,36 +3,26 @@ use axum::{ extract::{Path, State}, }; use log::debug; -use serde::Deserialize; +use super::VaultPath; use crate::{ app_state::{ AppState, - database::{ - Database, - models::{FileManifest, VaultId}, - }, + database::{Database, models::FileManifest}, }, errors::{SyncServerError, server_error}, - utils::normalize_vault_id::normalize_vault_id, }; -#[derive(Deserialize)] -pub struct GetFileManifestPath { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, -} - #[axum::debug_handler] pub async fn get_file_manifest( - Path(path): Path, + Path(VaultPath(vault_id)): Path, State(state): State, ) -> Result, SyncServerError> { - debug!("Fetching file manifest for vault `{}`", path.vault_id); + debug!("Fetching file manifest for vault `{vault_id}`"); let mut transaction = state .database - .create_readonly_transaction(&path.vault_id) + .create_readonly_transaction(&vault_id) .await .map_err(server_error)?; diff --git a/sync-server/src/server/endpoints/ping.rs b/sync-server/src/server/endpoints/ping.rs index 8e910feb..412ed0d2 100644 --- a/sync-server/src/server/endpoints/ping.rs +++ b/sync-server/src/server/endpoints/ping.rs @@ -7,26 +7,15 @@ use axum_extra::{ headers::{Authorization, authorization::Bearer}, }; use log::debug; -use serde::Deserialize; +use super::VaultPath; use crate::server::{auth::auth, responses::PingResponse}; -use crate::{ - app_state::{AppState, database::models::VaultId}, - consts::SUPPORTED_API_VERSION, - errors::SyncServerError, - utils::normalize_vault_id::normalize_vault_id, -}; - -#[derive(Deserialize)] -pub struct PingPathParams { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, -} +use crate::{app_state::AppState, consts::SUPPORTED_API_VERSION, errors::SyncServerError}; #[axum::debug_handler] pub async fn ping( maybe_auth_header: Option>>, - Path(PingPathParams { vault_id }): Path, + Path(VaultPath(vault_id)): Path, State(state): State, ) -> Result, SyncServerError> { debug!("Pinging vault `{vault_id}`"); diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index 26c9cfa3..b7883299 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -1,4 +1,4 @@ -use super::utils::find_already_processed_event; +use super::{VaultPath, utils::find_already_processed_event}; use crate::{ app_state::{ AppState, @@ -14,7 +14,6 @@ use crate::{ requests::{PushContent, PutFileContent}, responses::DocumentUpdateResponse, }, - utils::normalize_vault_id::normalize_vault_id, }; use anyhow::anyhow; use axum::{ @@ -25,37 +24,26 @@ use axum_extra::TypedHeader; use base64::{Engine as _, engine::general_purpose::STANDARD}; use log::debug; use reconcile_text::{BuiltinTokenizer, EditedText, NumberOrText}; -use serde::Deserialize; use sha2::{Digest, Sha256}; -#[derive(Deserialize)] -pub struct PutFileContentPath { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, - document_id: DocumentId, -} - #[axum::debug_handler] pub async fn put_file_content( - Path(path): Path, + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, Extension(user): Extension, TypedHeader(device): TypedHeader, State(state): State, Json(push): Json, ) -> Result, SyncServerError> { - debug!( - "Pushing document `{}` in vault `{}`", - path.document_id, path.vault_id - ); + debug!("Pushing document `{}` in vault `{}`", document_id, vault_id); let fingerprint = Sha256::digest( - serde_json::to_vec(&("content", path.document_id, &push)) + serde_json::to_vec(&("content", document_id, &push)) .map_err(|error| server_error(error.into()))?, ); let mut tx = state .database - .create_write_transaction(&path.vault_id) + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -74,7 +62,7 @@ pub async fn put_file_content( let parent = state .database - .get_latest_document_version(&path.vault_id, &path.document_id, Some(&mut tx)) + .get_latest_document_version(&vault_id, &document_id, Some(&mut tx)) .await .map_err(server_error)?; @@ -87,7 +75,7 @@ pub async fn put_file_content( } else if push.parent_version_id.is_some() { return Err(not_found_error(anyhow!( "Document {} does not exist", - path.document_id + document_id ))); } @@ -123,7 +111,7 @@ pub async fn put_file_content( vault_update_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) .await .map_err(server_error)?, - document_id: path.document_id, + document_id, content, updated_date: chrono::Utc::now(), user_id: user.name, @@ -152,10 +140,7 @@ pub async fn put_file_content( .await .map_err(|error| server_error(error.into()))?; - state - .broadcasts - .notify_about_vault_update(path.vault_id) - .await; + state.broadcasts.notify_about_vault_update(vault_id).await; Ok(Json(response)) } @@ -185,10 +170,7 @@ mod tests { push: PutFileContent, ) -> Result { put_file_content( - Path(PutFileContentPath { - vault_id: vault.clone(), - document_id, - }), + Path((VaultPath(vault.clone()), document_id)), Extension(User { name: user_id, token: String::new(), diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs index 667c7827..945d4dcb 100644 --- a/sync-server/src/server/endpoints/put_file_manifest.rs +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -4,36 +4,28 @@ use axum::{ extract::{Path, State}, }; use log::debug; -use serde::Deserialize; use sha2::{Digest, Sha256}; -use super::utils::find_already_processed_event; +use super::{VaultPath, utils::find_already_processed_event}; use crate::{ app_state::{ AppState, database::{ Database, - models::{EventRecord, FileManifest, VaultEvent, VaultId}, + models::{EventRecord, FileManifest, VaultEvent}, }, }, errors::{SyncServerError, client_error, server_error}, server::{requests::PushFileManifest, responses::FileManifestUpdateResponse}, - utils::normalize_vault_id::normalize_vault_id, }; -#[derive(Deserialize)] -pub struct PutFileManifestPath { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, -} - #[axum::debug_handler] pub async fn put_file_manifest( - Path(path): Path, + Path(VaultPath(vault_id)): Path, State(state): State, Json(push): Json, ) -> Result, SyncServerError> { - debug!("Pushing file manifest for vault `{}`", path.vault_id); + debug!("Pushing file manifest for vault `{vault_id}`"); // BTreeMap canonicalizes file manifest entries for request fingerprinting. let fingerprint = Sha256::digest( @@ -43,7 +35,7 @@ pub async fn put_file_manifest( let mut tx = state .database - .create_write_transaction(&path.vault_id) + .create_write_transaction(&vault_id) .await .map_err(server_error)?; @@ -73,7 +65,7 @@ pub async fn put_file_manifest( for id in push.entries.keys() { if state .database - .get_latest_document_version(&path.vault_id, id, Some(&mut tx)) + .get_latest_document_version(&vault_id, id, Some(&mut tx)) .await .map_err(server_error)? .is_none() @@ -113,10 +105,7 @@ pub async fn put_file_manifest( .await .map_err(|error| server_error(error.into()))?; - state - .broadcasts - .notify_about_vault_update(path.vault_id) - .await; + state.broadcasts.notify_about_vault_update(vault_id).await; Ok(Json(response)) } diff --git a/sync-server/src/server/endpoints/vault_snapshot.rs b/sync-server/src/server/endpoints/vault_snapshot.rs index 1d90f9e0..93e5e4d1 100644 --- a/sync-server/src/server/endpoints/vault_snapshot.rs +++ b/sync-server/src/server/endpoints/vault_snapshot.rs @@ -3,33 +3,23 @@ use axum::{ extract::{Path, State}, }; use log::debug; -use serde::Deserialize; +use super::VaultPath; use crate::{ - app_state::{ - AppState, - database::models::{VaultId, VaultSnapshot}, - }, + app_state::{AppState, database::models::VaultSnapshot}, errors::{SyncServerError, server_error}, - utils::normalize_vault_id::normalize_vault_id, }; -#[derive(Deserialize)] -pub struct VaultSnapshotPath { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, -} - #[axum::debug_handler] pub async fn vault_snapshot( - Path(path): Path, + Path(VaultPath(vault_id)): Path, State(state): State, ) -> Result, SyncServerError> { - debug!("Fetching vault snapshot for vault `{}`", path.vault_id); + debug!("Fetching vault snapshot for vault `{vault_id}`"); state .database - .vault_snapshot(&path.vault_id) + .vault_snapshot(&vault_id) .await .map(Json) .map_err(server_error) diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index 6d7f4a10..ad1fe8c0 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -9,7 +9,6 @@ use crate::{ }, }, errors::{SyncServerError, client_error, server_error}, - utils::normalize_vault_id::normalize_vault_id, }; use axum::{ extract::{ @@ -20,29 +19,21 @@ use axum::{ }; use futures::StreamExt; use log::{debug, info}; -use serde::Deserialize; use std::time::Duration; use tokio::sync::broadcast::error::RecvError; -#[derive(Deserialize)] -pub struct WebSocketPathParams { - #[serde(deserialize_with = "normalize_vault_id")] - vault_id: VaultId, -} +use super::VaultPath; #[axum::debug_handler] pub async fn websocket_handler( ws: WebSocketUpgrade, - Path(path): Path, + Path(VaultPath(vault_id)): Path, State(state): State, ) -> Result { - debug!( - "Upgrading WebSocket connection for vault `{}`", - path.vault_id - ); + debug!("Upgrading WebSocket connection for vault `{vault_id}`"); Ok(ws.on_upgrade(move |socket| async move { - if let Err(error) = websocket(state, socket, path.vault_id).await { + if let Err(error) = websocket(state, socket, vault_id).await { debug!("WebSocket disconnected: {error}"); } })) -- 2.54.0 From c62ec9b98be90d6fbacd9e010704b2b3bec6a1ba Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:00:35 +0100 Subject: [PATCH 20/48] Fmt --- sync-server/src/server/endpoints/put_file_content.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index b7883299..ae04756f 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -4,7 +4,7 @@ use crate::{ AppState, database::{ Database, - models::{DocumentId, EventRecord, StoredDocumentVersion, VaultEvent, VaultId}, + models::{DocumentId, EventRecord, StoredDocumentVersion, VaultEvent}, }, }, config::user_config::User, @@ -34,7 +34,7 @@ pub async fn put_file_content( State(state): State, Json(push): Json, ) -> Result, SyncServerError> { - debug!("Pushing document `{}` in vault `{}`", document_id, vault_id); + debug!("Pushing document `{document_id}` in vault `{vault_id}`"); let fingerprint = Sha256::digest( serde_json::to_vec(&("content", document_id, &push)) @@ -74,8 +74,7 @@ pub async fn put_file_content( } } else if push.parent_version_id.is_some() { return Err(not_found_error(anyhow!( - "Document {} does not exist", - document_id + "Document {document_id} does not exist" ))); } @@ -153,7 +152,9 @@ mod tests { use crate::{ app_state::database::{ Database, - models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultUpdateId}, + models::{ + DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId, + }, }, config::{ Config, database_config::DatabaseConfig, server_config::ServerConfig, -- 2.54.0 From f2326b846ea4a3309cbb163c01b85beaadbab87b Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:03:45 +0100 Subject: [PATCH 21/48] Small FE improvements --- frontend/sync-client/src/index.ts | 21 ++++++++++++++++--- .../src/persistence/persistence.ts | 4 ++++ .../src/services/fetch-controller.test.ts | 2 +- .../src/services/fetch-controller.ts | 2 +- .../sync-client/src/services/server-config.ts | 13 +++++++++--- .../src/services/websocket-manager.ts | 6 +++--- .../sync-client/src/utils/set-up-telemetry.ts | 2 ++ 7 files changed, 39 insertions(+), 11 deletions(-) diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index cfcc5071..b639f635 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -22,13 +22,18 @@ export { Logger, LogLevel, LogLine } from "./tracing/logger"; export { type SyncSettings, DEFAULT_SETTINGS } from "./persistence/settings"; export { rateLimit } from "./utils/rate-limit"; export type { RelativePath, StoredDatabase } from "./persistence/database"; -export type { FileSystemOperations } from "./file-operations/filesystem-operations"; +export type { + FileSystemOperations, + FileSnapshot +} from "./file-operations/filesystem-operations"; export type { PersistenceProvider } from "./persistence/persistence"; export type { CursorSpan } from "./services/types/CursorSpan"; export type { ClientCursors } from "./services/types/ClientCursors"; export type { NetworkConnectionStatus } from "./types/network-connection-status"; -export type { ServerVersionMismatchError } from "./services/server-version-mismatch-error"; -export type { AuthenticationError } from "./services/authentication-error"; +export type { + ServerVersionMismatchError, + AuthenticationError +} from "./errors/errors"; export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; export { DocumentSyncStatus } from "./types/document-sync-status"; export { SyncClient } from "./sync-client"; @@ -47,3 +52,13 @@ export const utils = { awaitAll, removeFromArray }; + +export type { + FileManifest, + FileManifestEntries, + EventRecord, + EventBatch, + VaultSnapshot, + PutFileContent, + PushFileManifest +} from "./services/protocol"; diff --git a/frontend/sync-client/src/persistence/persistence.ts b/frontend/sync-client/src/persistence/persistence.ts index d226e611..f879d06b 100644 --- a/frontend/sync-client/src/persistence/persistence.ts +++ b/frontend/sync-client/src/persistence/persistence.ts @@ -1,3 +1,7 @@ +/** save() atomically replaces the entire value and resolves only after both the + * data and its directory entry are durable across power loss. Failed saves must + * leave either complete value readable; callers retain their recovery journal. + */ export interface PersistenceProvider { load: () => Promise; save: (data: T) => Promise; diff --git a/frontend/sync-client/src/services/fetch-controller.test.ts b/frontend/sync-client/src/services/fetch-controller.test.ts index 94fa8424..a0a70d80 100644 --- a/frontend/sync-client/src/services/fetch-controller.test.ts +++ b/frontend/sync-client/src/services/fetch-controller.test.ts @@ -3,7 +3,7 @@ import { describe, it, mock, beforeEach, afterEach } from "node:test"; import assert from "node:assert"; import { FetchController } from "./fetch-controller"; import { Logger } from "../tracing/logger"; -import { SyncResetError } from "./sync-reset-error"; +import { SyncResetError } from "../errors/errors"; import { sleep } from "../utils/sleep"; describe("FetchController", () => { diff --git a/frontend/sync-client/src/services/fetch-controller.ts b/frontend/sync-client/src/services/fetch-controller.ts index 77b87e3a..c1bd29c9 100644 --- a/frontend/sync-client/src/services/fetch-controller.ts +++ b/frontend/sync-client/src/services/fetch-controller.ts @@ -1,6 +1,6 @@ import type { Logger } from "../tracing/logger"; import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "./sync-reset-error"; +import { SyncResetError } from "../errors/errors"; /** * Offers a resettable fetch implementation that waits until syncing is enabled diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts index 309c637c..fcd8adb9 100644 --- a/frontend/sync-client/src/services/server-config.ts +++ b/frontend/sync-client/src/services/server-config.ts @@ -1,6 +1,8 @@ import { SUPPORTED_API_VERSION } from "../consts"; -import { AuthenticationError } from "./authentication-error"; -import { ServerVersionMismatchError } from "./server-version-mismatch-error"; +import { + AuthenticationError, + ServerVersionMismatchError +} from "../errors/errors"; import type { SyncService } from "./sync-service"; import type { PingResponse } from "./types/PingResponse"; @@ -74,7 +76,12 @@ export class ServerConfig { public async getConfig(): Promise { if (!this.config) { this.response ??= this.syncService.ping(); - this.config = await this.response; + try { + this.config = await this.response; + } catch (error) { + this.response = undefined; + throw error; + } } ServerConfig.validateConfig(this.config); diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 09787bce..8d62fb2b 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -5,7 +5,7 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { ClientCursors } from "./types/ClientCursors"; import { createPromise } from "../utils/create-promise"; -import type { WebSocketVaultUpdate } from "./types/WebSocketVaultUpdate"; +import type { EventBatch } from "./protocol"; import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; import { removeFromArray } from "../utils/remove-from-array"; import { EventListeners } from "../utils/data-structures/event-listeners"; @@ -17,7 +17,7 @@ export class WebSocketManager { >(); public readonly onRemoteVaultUpdateReceived = new EventListeners< - (update: WebSocketVaultUpdate) => Promise + (update: EventBatch) => Promise >(); public readonly onRemoteCursorsUpdateReceived = new EventListeners< @@ -252,7 +252,7 @@ export class WebSocketManager { private async handleWebSocketMessage( message: WebSocketServerMessage ): Promise { - if (message.type === "vaultUpdate") { + if (message.type === "vaultEvents") { await this.onRemoteVaultUpdateReceived.triggerAsync(message); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition diff --git a/frontend/sync-client/src/utils/set-up-telemetry.ts b/frontend/sync-client/src/utils/set-up-telemetry.ts index d9e73a79..d3704ee1 100644 --- a/frontend/sync-client/src/utils/set-up-telemetry.ts +++ b/frontend/sync-client/src/utils/set-up-telemetry.ts @@ -24,11 +24,13 @@ export const setUpTelemetry = (): (() => void) => { } }); }; + window.addEventListener("error", onError); const onUnhandledRejection = (event: PromiseRejectionEvent): void => { Sentry.captureException(event.reason); }; + window.addEventListener("unhandledrejection", onUnhandledRejection); return (): void => { -- 2.54.0 From 04db2b2abb331f2b31767f1535618fc83562ab7f Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:04:14 +0100 Subject: [PATCH 22/48] Clean up and fix reference error --- .../sync-client/src/persistence/settings.ts | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index d78170e6..944f6980 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -56,7 +56,10 @@ export class Settings { } public getSettings(): SyncSettings { - return this.settings; + return { + ...this.settings, + ignorePatterns: [...this.settings.ignorePatterns] + }; } public async setSetting( @@ -74,21 +77,14 @@ export class Settings { `Updating settings with: ${JSON.stringify(value)}` ); const oldSettings = this.settings; - this.settings = { - ...this.settings, - ...value - }; + const next = { ...this.settings, ...value }; + await this.saveData(next); + this.settings = next; await this.onSettingsChanged.triggerAsync( - this.settings, + this.getSettings(), oldSettings ); - - await this.save(); }); } - - private async save(): Promise { - await this.saveData(this.settings); - } } -- 2.54.0 From e8e986439bfd17ec0ca1c101d48a2706db46bca1 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:17:20 +0100 Subject: [PATCH 23/48] Simplify sync service --- .../src/views/settings/settings-tab.ts | 36 ++ .../sync-client/src/persistence/settings.ts | 2 + .../sync-client/src/services/sync-service.ts | 490 ++++-------------- 3 files changed, 141 insertions(+), 387 deletions(-) diff --git a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts index 5a5823c2..84da83d8 100644 --- a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts @@ -434,6 +434,42 @@ export class SyncSettingsTab extends PluginSettingTab { containerEl.createEl("h3", { text: "Advanced" }); + new Setting(containerEl) + .setName("Request timeout (ms)") + .setDesc( + "The maximum time to wait for a server request, in milliseconds." + ) + .addText((input) => + input + .setValue( + this.syncClient + .getSettings() + .requestTimeoutMs.toString() + ) + .onChange(async (value) => { + if (value === "") { + return; + } + let parsedValue = Number(value); + if ( + !Number.isSafeInteger(parsedValue) || + parsedValue <= 0 + ) { + parsedValue = + this.syncClient.getSettings().requestTimeoutMs; + } + + if (value !== parsedValue.toString()) { + input.setValue(parsedValue.toString()); + } + + return this.syncClient.setSetting( + "requestTimeoutMs", + parsedValue + ); + }) + ); + new Setting(containerEl) .setName("Network retry interval (ms)") .setDesc( diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 944f6980..2558139a 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -13,6 +13,7 @@ export interface SyncSettings { webSocketRetryIntervalMs: number; diffCacheSizeMB: number; enableTelemetry: boolean; + requestTimeoutMs: number; networkRetryIntervalMs: number; minimumSaveIntervalMs: number; } @@ -28,6 +29,7 @@ export const DEFAULT_SETTINGS: SyncSettings = { webSocketRetryIntervalMs: 3500, diffCacheSizeMB: 4, enableTelemetry: false, + requestTimeoutMs: 30_000, networkRetryIntervalMs: 1000, minimumSaveIntervalMs: 1000 }; diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 8190a638..3c5f604c 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -1,434 +1,150 @@ -import type { - DocumentId, - RelativePath, - VaultUpdateId -} from "../persistence/database"; - -import type { Logger } from "../tracing/logger"; import type { Settings } from "../persistence/settings"; import type { FetchController } from "./fetch-controller"; -import { sleep } from "../utils/sleep"; -import { SyncResetError } from "./sync-reset-error"; -import type { SerializedError } from "./types/SerializedError"; -import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; +import type { Logger } from "../tracing/logger"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; import type { DocumentVersion } from "./types/DocumentVersion"; -import type { FetchLatestDocumentsResponse } from "./types/FetchLatestDocumentsResponse"; +import type { EventBatch } from "./types/EventBatch"; +import type { FileManifest } from "./types/FileManifest"; +import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse"; +import type { PutFileContent } from "./types/PutFileContent"; +import type { PushFileManifest } from "./types/PushFileManifest"; +import type { VaultSnapshot } from "./types/VaultSnapshot"; import type { PingResponse } from "./types/PingResponse"; -import type { DeleteDocumentVersion } from "./types/DeleteDocumentVersion"; -import type { UpdateTextDocumentVersion } from "./types/UpdateTextDocumentVersion"; +import { AuthenticationError, PermanentSyncError } from "../errors/errors"; export class SyncService { - private readonly client: typeof globalThis.fetch; - private readonly pingClient: typeof globalThis.fetch; + private readonly client: typeof fetch; + private readonly rawFetch: typeof fetch; public constructor( private readonly deviceId: string, - private readonly fetchController: FetchController, + fetchController: FetchController, private readonly settings: Settings, - private readonly logger: Logger, - fetchImplementation: typeof globalThis.fetch = globalThis.fetch + logger: Logger, + fetchImplementation: typeof fetch = globalThis.fetch ) { - // ensure that if it's called a method, `this` won't be bound to the instance - const unboundFetch: typeof globalThis.fetch = async (...args) => - fetchImplementation(...args); - - this.client = this.fetchController.getControlledFetchImplementation( - this.logger, - unboundFetch + this.rawFetch = async (...args) => fetchImplementation(...args); + this.client = fetchController.getControlledFetchImplementation( + logger, + this.rawFetch ); - this.pingClient = unboundFetch; } - private static async errorFromResponse( - response: Response - ): Promise { + public getUrl(path: string): string { + const { remoteUri, vaultName } = this.settings.getSettings(); + return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`; + } + + private async request( + path: string, + body?: unknown, + raw = false + ): Promise { + const response = await (raw ? this.rawFetch : this.client)( + this.getUrl(path), + { + method: body === undefined ? "GET" : "PUT", + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout( + this.settings.getSettings().requestTimeoutMs + ), + headers: { + Authorization: `Bearer ${this.settings.getSettings().token}`, + "Device-Id": this.deviceId, + "Content-Type": "application/json" + } + } + ); + + if (!response.ok) { + this.throwForErrorResponse( + response, + `HTTP ${response.status}: ${await response.text()}` + ); + } + + return (await response.json()) as T; + } + + private throwForErrorResponse( + response: Response, + message: string + ): never { + if (response.status === 401 || response.status === 403) { + throw new AuthenticationError(message); + } if ( - response.headers - .get("Content-Type") - ?.includes("application/json") == true + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 ) { - const result: SerializedError = - (await response.json()) as SerializedError; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - return SyncService.formatError(result); + throw new PermanentSyncError(message); } - return `HTTP ${response.status}: ${response.statusText}`; + throw new Error(message); } - private static formatError(error: SerializedError): string { - let result = error.message; - if (error.causes.length > 0) { - const causes = error.causes.join(", "); - result += ` caused by: ${causes}`; - } - - return result; + public async ping(): Promise { + return this.request("/ping", undefined, true); } - public async create({ - documentId, - relativePath, - contentBytes - }: { - documentId?: DocumentId; - relativePath: RelativePath; - contentBytes: Uint8Array; - }): Promise { - return this.retryForever(async () => { - const formData = new FormData(); - if (documentId !== undefined) { - formData.append("document_id", documentId); - } - formData.append("relative_path", relativePath); - formData.append( - "content", - new Blob([new Uint8Array(contentBytes)]) - ); - - this.logger.debug( - `Creating document with id ${documentId} and relative path ${relativePath}` - ); - - const response = await this.client(this.getUrl("/documents"), { - method: "POST", - body: formData, - headers: this.getDefaultHeaders() - }); - - if (!response.ok) { - throw new Error( - `Failed to create document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentVersionWithoutContent = - (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug(`Created document ${JSON.stringify(result)}`); - - return result; - }); + public async vaultSnapshot(): Promise { + return this.request("/vault-snapshot"); } - public async putText({ - parentVersionId, - documentId, - relativePath, - content - }: { - parentVersionId: VaultUpdateId; - documentId: DocumentId; - relativePath: RelativePath; - content: (number | string)[]; - }): Promise { - return this.retryForever(async () => { - this.logger.debug( - `Updating text document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}, content [${content.join(", ")}]` - ); - - const request: UpdateTextDocumentVersion = { - parentVersionId, - relativePath, - content - }; - - const response = await this.client( - this.getUrl(`/documents/${documentId}/text`), - { - method: "PUT", - body: JSON.stringify(request), - headers: this.getDefaultHeaders({ type: "json" }) - } - ); - - if (!response.ok) { - throw new Error( - `Failed to update document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentUpdateResponse = - (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Updated document ${JSON.stringify(result)} with id ${ - result.documentId - }}` - ); - - return result; - }); + public async events(after: number): Promise { + return this.request(`/events-since?after=${after}`); } - public async putBinary({ - parentVersionId, - documentId, - relativePath, - contentBytes - }: { - parentVersionId: VaultUpdateId; - documentId: DocumentId; - relativePath: RelativePath; - contentBytes: Uint8Array; - }): Promise { - return this.retryForever(async () => { - this.logger.debug( - `Updating binary document ${documentId} with parent version ${parentVersionId} and relative path ${relativePath}` - ); - const formData = new FormData(); - formData.append("parent_version_id", parentVersionId.toString()); - formData.append("relative_path", relativePath); - formData.append( - "content", - new Blob([new Uint8Array(contentBytes)]) - ); - - const response = await this.client( - this.getUrl(`/documents/${documentId}/binary`), - { - method: "PUT", - body: formData, - headers: this.getDefaultHeaders() - } - ); - - if (!response.ok) { - throw new Error( - `Failed to update document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentUpdateResponse = - (await response.json()) as DocumentUpdateResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Updated document ${JSON.stringify(result)} with id ${ - result.documentId - }}` - ); - - return result; - }); + public async fileManifest(): Promise { + return this.request("/file-manifest"); } - public async delete({ - documentId, - relativePath - }: { - documentId: DocumentId; - relativePath: RelativePath; - }): Promise { - return this.retryForever(async () => { - const request: DeleteDocumentVersion = { - relativePath - }; - - this.logger.debug( - `Delete document with id ${documentId} and relative path ${relativePath}` - ); - - const response = await this.client( - this.getUrl(`/documents/${documentId}`), - { - method: "DELETE", - body: JSON.stringify(request), - headers: this.getDefaultHeaders({ type: "json" }) - } - ); - - if (!response.ok) { - throw new Error( - `Failed to delete document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentVersionWithoutContent = - (await response.json()) as DocumentVersionWithoutContent; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Deleted document ${relativePath} with id ${documentId}` - ); - - return result; - }); + public async pushFileManifest( + request: PushFileManifest + ): Promise { + return this.request("/file-manifest", request); } - public async get({ - documentId - }: { - documentId: DocumentId; - }): Promise { - return this.retryForever(async () => { - this.logger.debug(`Getting document with id ${documentId}`); + public async putFileContent( + id: string, + request: PutFileContent + ): Promise { + return this.request(`/documents/${id}`, request); + } - const response = await this.client( - this.getUrl(`/documents/${documentId}`), - { - headers: this.getDefaultHeaders() - } - ); - - if (!response.ok) { - throw new Error( - `Failed to get document: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: DocumentVersion = - (await response.json()) as DocumentVersion; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug(`Got document ${JSON.stringify(result)}`); - - return result; - }); + public async get(id: string): Promise { + return this.request(`/documents/${id}`); } public async getDocumentVersionContent({ documentId, vaultUpdateId }: { - documentId: DocumentId; - vaultUpdateId: VaultUpdateId; + documentId: string; + vaultUpdateId: number; }): Promise { - return this.retryForever(async () => { - this.logger.debug( - `Getting document with id ${documentId} and version ${vaultUpdateId}` - ); - - const response = await this.client( - this.getUrl( - `/documents/${documentId}/versions/${vaultUpdateId}/content` + const response = await this.client( + this.getUrl( + `/documents/${documentId}/versions/${vaultUpdateId}/content` + ), + { + signal: AbortSignal.timeout( + this.settings.getSettings().requestTimeoutMs ), - { - headers: this.getDefaultHeaders() + headers: { + Authorization: `Bearer ${this.settings.getSettings().token}` } - ); - - if (!response.ok) { - throw new Error( - `Failed to get document: ${await SyncService.errorFromResponse( - response - )}` - ); } - - const result = await response.bytes(); - this.logger.debug( - `Got document version content for document ${documentId} version ${vaultUpdateId}` - ); - return result; - }); - } - - public async getAll( - since?: VaultUpdateId - ): Promise { - return this.retryForever(async () => { - this.logger.debug( - "Getting all documents" + - (since != null ? ` since ${since}` : "") - ); - - const url = new URL(this.getUrl("/documents")); - if (since !== undefined) { - url.searchParams.append("since", since.toString()); - } - const response = await this.client(url.toString(), { - headers: this.getDefaultHeaders() - }); - - if (!response.ok) { - throw new Error( - `Failed to get documents: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: FetchLatestDocumentsResponse = - (await response.json()) as FetchLatestDocumentsResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Got ${result.latestDocuments.length} document metadata` - ); - - return result; - }); - } - - public async ping(): Promise { - this.logger.debug("Pinging server"); - const response = await this.pingClient(this.getUrl("/ping"), { - headers: this.getDefaultHeaders() - }); - - if (!response.ok) { - throw new Error( - `Failed to ping server: ${await SyncService.errorFromResponse( - response - )}` - ); - } - - const result: PingResponse = (await response.json()) as PingResponse; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - this.logger.debug( - `Pinged server, got response: ${JSON.stringify(result)}` ); - return result; - } - - private getUrl(path: string): string { - const { vaultName, remoteUri } = this.settings.getSettings(); - const remoteUriWithoutTrailingSlash = remoteUri.replace(/\/+$/, ""); - const encodedVaultName = encodeURIComponent(vaultName.trim()); - return `${remoteUriWithoutTrailingSlash}/vaults/${encodedVaultName}${path}`; - } - - private getDefaultHeaders( - { type }: { type?: "json" } = { type: undefined } - ): Record { - const headers: Record = { - "device-id": this.deviceId, - authorization: `Bearer ${this.settings.getSettings().token}` - }; - - if (type === "json") { - headers["Content-Type"] = "application/json"; + if (!response.ok) { + this.throwForErrorResponse( + response, + `Cannot fetch version ${vaultUpdateId}: HTTP ${response.status}` + ); } - return headers; - } - - private async retryForever(fn: () => Promise): Promise { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - try { - return await fn(); - } catch (e) { - // We must not retry errors coming from reset - if (e instanceof SyncResetError) { - throw e; - } - - const retryInterval = - this.settings.getSettings().networkRetryIntervalMs; - this.logger.error( - `Failed network call (${e}), retrying in ${retryInterval}ms` - ); - await sleep(retryInterval); - } - } + return new Uint8Array(await response.arrayBuffer()); } } -- 2.54.0 From 35783652703b83bc877a9b291e60c9386ddb9458 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:17:38 +0100 Subject: [PATCH 24/48] Use kebab endpoints --- sync-server/src/server.rs | 6 +++--- sync-server/src/server/tests.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 28137746..9eb66172 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -96,16 +96,16 @@ pub async fn create_server(config: Config) -> Result<()> { fn get_authed_routes(app_state: AppState) -> Router { Router::new() .route( - "/vaults/:vault_id/file_manifest", + "/vaults/:vault_id/file-manifest", get(endpoints::get_file_manifest::get_file_manifest) .put(endpoints::put_file_manifest::put_file_manifest), ) .route( - "/vaults/:vault_id/vault_snapshot", + "/vaults/:vault_id/vault-snapshot", get(endpoints::vault_snapshot::vault_snapshot), ) .route( - "/vaults/:vault_id/events_since", + "/vaults/:vault_id/events-since", get(endpoints::events::events), ) .route( diff --git a/sync-server/src/server/tests.rs b/sync-server/src/server/tests.rs index 493e5283..01d5f842 100644 --- a/sync-server/src/server/tests.rs +++ b/sync-server/src/server/tests.rs @@ -60,7 +60,7 @@ async fn http_rejects_encoded_traversal_and_accepts_normalized_allowlists() { ] { let (status, _) = request( app.clone(), - &format!("/vaults/{vault}/vault_snapshot"), + &format!("/vaults/{vault}/vault-snapshot"), "admin", None, ) @@ -71,7 +71,7 @@ async fn http_rejects_encoded_traversal_and_accepts_normalized_allowlists() { for vault in ["MyVault", "myvault", "%20MYVAULT%20"] { let (status, _) = request( app.clone(), - &format!("/vaults/{vault}/file_manifest"), + &format!("/vaults/{vault}/file-manifest"), "mixed", None, ) @@ -79,7 +79,7 @@ async fn http_rejects_encoded_traversal_and_accepts_normalized_allowlists() { assert_eq!(status, StatusCode::OK, "vault: {vault}"); } assert_eq!( - request(app, "/vaults/other/file_manifest", "mixed", None) + request(app, "/vaults/other/file-manifest", "mixed", None) .await .0, StatusCode::FORBIDDEN -- 2.54.0 From 0c277c37af8b64b03752bfbe1c6dce45e1c93580 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:17:54 +0100 Subject: [PATCH 25/48] Change imports --- frontend/sync-client/src/index.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index b639f635..e7476470 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -53,12 +53,10 @@ export const utils = { removeFromArray }; -export type { - FileManifest, - FileManifestEntries, - EventRecord, - EventBatch, - VaultSnapshot, - PutFileContent, - PushFileManifest -} from "./services/protocol"; +export type { EventBatch } from "./services/types/EventBatch"; +export type { EventRecord } from "./services/types/EventRecord"; +export type { FileManifest } from "./services/types/FileManifest"; +export type { PushFileManifest } from "./services/types/PushFileManifest"; +export type { PutFileContent } from "./services/types/PutFileContent"; +export type { VaultSnapshot } from "./services/types/VaultSnapshot"; +export type { FileManifestEntries } from "./types/file-manifest-entries"; -- 2.54.0 From 141db79f3a617ac178bcbcc4b867cde76ef6b043 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:20:50 +0100 Subject: [PATCH 26/48] Adjust cursor tracking to new api --- .../src/sync-operations/cursor-tracker.ts | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index bdd7d9b7..a3d37885 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -138,7 +138,7 @@ export class CursorTracker { const record = this.database.getLatestDocumentByRelativePath( doc.relative_path ); - if (record?.metadata?.hash !== hash(readContent)) { + if (record?.metadata?.hash !== (await hash(readContent))) { doc.vault_update_id = null; } } @@ -178,6 +178,15 @@ export class CursorTracker { result.push({ ...clientCursors, + documentsWithCursors: clientCursors.documentsWithCursors.map( + (document) => ({ + ...document, + relative_path: + this.database.getDocumentByDocumentId( + document.document_id + )?.relativePath ?? document.relative_path + }) + ), isOutdated: clientCursors.upToDateness === DocumentUpToDateness.Prior }); @@ -223,8 +232,8 @@ export class CursorTracker { private async getDocumentUpToDateness( document: DocumentWithCursors ): Promise { - const record = this.database.getLatestDocumentByRelativePath( - document.relative_path + const record = this.database.getDocumentByDocumentId( + document.document_id ); if (!record) { @@ -245,13 +254,16 @@ export class CursorTracker { return DocumentUpToDateness.Prior; } - const currentContent = await this.fileOperations.read( - document.relative_path - ); + let currentContent: Uint8Array; + try { + currentContent = await this.fileOperations.read( + record.relativePath + ); + } catch { + return DocumentUpToDateness.Prior; + } - return this.database.getLatestDocumentByRelativePath( - document.relative_path - )?.metadata?.hash === hash(currentContent) + return record.metadata?.hash === (await hash(currentContent)) ? DocumentUpToDateness.UpToDate : DocumentUpToDateness.Prior; } -- 2.54.0 From c9512ab8b1a24501865da5035f8ee0f656b3fdee Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:37:16 +0100 Subject: [PATCH 27/48] Add syncInterval --- .../src/views/settings/settings-tab.ts | 46 ++++++++++++++++++- .../sync-client/src/persistence/settings.ts | 2 + .../src/services/websocket-manager.ts | 2 +- .../src/types/file-manifest-entries.ts | 3 ++ 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 frontend/sync-client/src/types/file-manifest-entries.ts diff --git a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts index 84da83d8..3f7ed7ca 100644 --- a/frontend/obsidian-plugin/src/views/settings/settings-tab.ts +++ b/frontend/obsidian-plugin/src/views/settings/settings-tab.ts @@ -266,8 +266,9 @@ export class SyncSettingsTab extends PluginSettingTab { new Notice("Checking connection to the server..."); new Notice( - (await this.syncClient.checkConnection()) - .serverMessage + ( + await this.syncClient.checkConnection() + ).serverMessage ); await this.statusDescription.updateConnectionState(); } else { @@ -503,6 +504,47 @@ export class SyncSettingsTab extends PluginSettingTab { ); }) ); + + new Setting(containerEl) + .setName("Sync interval (ms)") + .setDesc( + "How often to reconcile when no changes are detected. Leave blank to rely on change notifications." + ) + .addText((input) => + input + .setPlaceholder("Disabled") + .setValue( + this.syncClient + .getSettings() + .syncIntervalMs?.toString() ?? "" + ) + .onChange(async (value) => { + if (value === "") { + return this.syncClient.setSetting( + "syncIntervalMs", + undefined + ); + } + + const parsedValue = Number(value); + if ( + !Number.isSafeInteger(parsedValue) || + parsedValue <= 0 + ) { + input.setValue( + this.syncClient + .getSettings() + .syncIntervalMs?.toString() ?? "" + ); + return; + } + + return this.syncClient.setSetting( + "syncIntervalMs", + parsedValue + ); + }) + ); } private setStatusDescriptionSubscription( diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 2558139a..f1f17f26 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -15,6 +15,7 @@ export interface SyncSettings { enableTelemetry: boolean; requestTimeoutMs: number; networkRetryIntervalMs: number; + syncIntervalMs?: number; minimumSaveIntervalMs: number; } @@ -31,6 +32,7 @@ export const DEFAULT_SETTINGS: SyncSettings = { enableTelemetry: false, requestTimeoutMs: 30_000, networkRetryIntervalMs: 1000, + syncIntervalMs: undefined, minimumSaveIntervalMs: 1000 }; diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 8d62fb2b..8b3b69dd 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -5,7 +5,7 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { ClientCursors } from "./types/ClientCursors"; import { createPromise } from "../utils/create-promise"; -import type { EventBatch } from "./protocol"; +import type { EventBatch } from "./types/EventBatch"; import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; import { removeFromArray } from "../utils/remove-from-array"; import { EventListeners } from "../utils/data-structures/event-listeners"; diff --git a/frontend/sync-client/src/types/file-manifest-entries.ts b/frontend/sync-client/src/types/file-manifest-entries.ts new file mode 100644 index 00000000..1e23dd06 --- /dev/null +++ b/frontend/sync-client/src/types/file-manifest-entries.ts @@ -0,0 +1,3 @@ +import type { FileManifest } from "../services/types/FileManifest"; + +export type FileManifestEntries = FileManifest["entries"]; -- 2.54.0 From f5fc71f388ebaa80a9c456985230d77e1f82cf70 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 20:37:25 +0100 Subject: [PATCH 28/48] Extract reconcile --- .../src/sync-operations/content.test.ts | 93 +++++++++++++++++++ .../src/sync-operations/content.ts | 69 ++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 frontend/sync-client/src/sync-operations/content.test.ts create mode 100644 frontend/sync-client/src/sync-operations/content.ts diff --git a/frontend/sync-client/src/sync-operations/content.test.ts b/frontend/sync-client/src/sync-operations/content.test.ts new file mode 100644 index 00000000..db446097 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/content.test.ts @@ -0,0 +1,93 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { mergeContent, fromStoredSnapshot, toStoredSnapshot } from "./content"; + +const extensions = ["md"]; +const snapshot = async (text: string) => + toStoredSnapshot({ content: new TextEncoder().encode(text) }); + +describe("content reconciliation without file timestamps", () => { + it("round-trips content and editor selections without timestamps", async () => { + const original = { + content: new Uint8Array([0, 255, 13, 10]), + cursors: [] + }; + assert.deepEqual( + fromStoredSnapshot(await toStoredSnapshot(original)), + original + ); + }); + + it("keeps local binary edits when the server content is unchanged", async () => { + const base = await snapshot("\0base"); + const local = await snapshot("\0local"); + const merged = await mergeContent( + "file.bin", + base, + local, + base, + extensions + ); + assert.equal(merged.contentBase64, local.contentBase64); + }); + + it("incorporates server binary edits when local content is unchanged", async () => { + const base = await snapshot("\0base"); + const remote = await snapshot("\0remote"); + const merged = await mergeContent( + "file.bin", + base, + base, + remote, + extensions + ); + assert.equal(merged.contentBase64, remote.contentBase64); + }); + + it("keeps the server on concurrent binary or unmergeable text edits", async () => { + for (const [path, prefix] of [ + ["file.bin", "\0"], + ["file.md", "\0"], + ["file.json", ""] + ]) { + const base = await snapshot(`${prefix}base`); + const local = await snapshot(`${prefix}local`); + const remote = await snapshot(`${prefix}remote`); + const merged = await mergeContent( + path, + base, + local, + remote, + extensions + ); + assert.equal(merged.contentBase64, remote.contentBase64, path); + } + }); + + it("keeps the server for differing binary content without a common base", async () => { + const local = await snapshot("\0local"); + const remote = await snapshot("\0remote"); + const merged = await mergeContent( + "file.bin", + undefined, + local, + remote, + extensions + ); + assert.equal(merged.contentBase64, remote.contentBase64); + }); + + it("still merges independent text edits from both sides", async () => { + const merged = await mergeContent( + "note.md", + await snapshot("First paragraph.\n\nLast paragraph.\n"), + await snapshot("Updated first paragraph.\n\nLast paragraph.\n"), + await snapshot("First paragraph.\n\nUpdated last paragraph.\n"), + extensions + ); + assert.equal( + new TextDecoder().decode(fromStoredSnapshot(merged).content), + "Updated first paragraph.\n\nUpdated last paragraph.\n" + ); + }); +}); diff --git a/frontend/sync-client/src/sync-operations/content.ts b/frontend/sync-client/src/sync-operations/content.ts new file mode 100644 index 00000000..a70896bb --- /dev/null +++ b/frontend/sync-client/src/sync-operations/content.ts @@ -0,0 +1,69 @@ +import { base64ToBytes, bytesToBase64 } from "byte-base64"; +import { reconcile } from "reconcile-text"; +import type { FileSnapshot } from "../file-operations/filesystem-operations"; +import type { StoredSnapshot } from "../persistence/database"; +import { hash } from "../utils/hash"; +import { isBinary } from "../utils/is-binary"; +import { isFileTypeMergable } from "../utils/is-file-type-mergable"; + +export async function toStoredSnapshot( + snapshot: FileSnapshot +): Promise { + return { + contentBase64: bytesToBase64(snapshot.content), + cursors: snapshot.cursors, + hash: await hash(snapshot.content) + }; +} + +export function fromStoredSnapshot(snapshot: StoredSnapshot): FileSnapshot { + return { + content: base64ToBytes(snapshot.contentBase64), + cursors: snapshot.cursors + }; +} + +export async function mergeContent( + path: string, + base: StoredSnapshot | undefined, + local: StoredSnapshot, + remote: StoredSnapshot, + extensions: string[] +): Promise { + if (local.contentBase64 === remote.contentBase64) + return { + ...remote, + cursors: local.cursors + }; + + if (!local.cursors && local.contentBase64 === base?.contentBase64) + return remote; + + if (remote.contentBase64 === base?.contentBase64) return local; + + const inputs = [ + base?.contentBase64 ?? "", + local.contentBase64, + remote.contentBase64 + ].map(base64ToBytes); + + // Concurrent unmergeable edits keep the server's content. The filesystem + // journal retains displaced local bytes for recovery. + if (!isFileTypeMergable(path, extensions) || inputs.some(isBinary)) + return remote; + + const [parentText, localText, remoteText] = inputs.map((bytes) => + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes) + ); + + const merged = reconcile( + parentText, + { text: localText, cursors: local.cursors }, + remoteText + ); + + return toStoredSnapshot({ + content: new TextEncoder().encode(merged.text), + cursors: local.cursors ? merged.cursors : undefined + }); +} -- 2.54.0 From 4d8fc7766591ec203452eda5b6bb1b51f7059a06 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:04:14 +0100 Subject: [PATCH 29/48] Nicer write transaction --- sync-server/src/app_state/database.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index c0199821..9aa99db5 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -178,14 +178,11 @@ impl Database { } pub async fn create_write_transaction(&self, vault: &VaultId) -> Result> { - let mut transaction = self.create_readonly_transaction(vault).await?; - - // sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 - sqlx::query("END; BEGIN IMMEDIATE;") - .execute(&mut *transaction) - .await?; - - Ok(transaction) + self.get_connection_pool(vault) + .await? + .begin_with("BEGIN IMMEDIATE") + .await + .context("Cannot create write transaction") } /// Cleanup idle connection pools that haven't been accessed in more than 5 minutes -- 2.54.0 From 45a6b1c115cc2efa25562d64c571200d8b66f077 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:10:38 +0100 Subject: [PATCH 30/48] Don't duplicate events --- .../src/app_state/database/mutations.rs | 11 +-- sync-server/src/app_state/database/queries.rs | 81 ++++++++++++++----- sync-server/src/app_state/database/tests.rs | 26 +----- .../src/server/endpoints/put_file_content.rs | 18 +---- .../src/server/endpoints/put_file_manifest.rs | 12 +-- sync-server/src/server/endpoints/utils.rs | 11 +-- sync-server/src/utils/portable_path.rs | 16 +--- 7 files changed, 75 insertions(+), 100 deletions(-) diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs index 09f7a73b..5d7b52b4 100644 --- a/sync-server/src/app_state/database/mutations.rs +++ b/sync-server/src/app_state/database/mutations.rs @@ -1,6 +1,6 @@ use super::{ Database, Transaction, - models::{EventRecord, FileManifest, StoredDocumentVersion, VaultUpdateId}, + models::{FileManifest, StoredDocumentVersion, VaultUpdateId}, }; use anyhow::Result; @@ -59,13 +59,4 @@ impl Database { Ok(()) } - - pub async fn write_event(tx: &mut Transaction<'_>, event: &EventRecord) -> Result<()> { - sqlx::query("UPDATE events SET event_json = ? WHERE event_id = ?") - .bind(serde_json::to_string(event)?) - .bind(event.event_id) - .execute(&mut **tx) - .await?; - Ok(()) - } } diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs index 2134a58a..5ba1a97d 100644 --- a/sync-server/src/app_state/database/queries.rs +++ b/sync-server/src/app_state/database/queries.rs @@ -1,23 +1,28 @@ use super::{ Database, Transaction, models::{ - DocumentId, DocumentVersionWithoutContent, EventBatch, FileManifest, StoredDocumentVersion, - VaultId, VaultSnapshot, VaultUpdateId, + DocumentId, DocumentVersionWithoutContent, EventBatch, EventRecord, FileManifest, + StoredDocumentVersion, VaultEvent, VaultId, VaultSnapshot, VaultUpdateId, }, }; -use anyhow::{Result, ensure}; +use anyhow::{Context as _, Result, ensure}; impl Database { pub async fn get_request_event( tx: &mut Transaction<'_>, request_id: uuid::Uuid, - ) -> Result, String)>> { - Ok(sqlx::query_as( - "SELECT request_fingerprint, event_json FROM events WHERE request_id = ?", - ) - .bind(request_id.to_string()) - .fetch_optional(&mut **tx) - .await?) + ) -> Result, VaultEvent)>> { + let row: Option<(Vec, VaultUpdateId)> = + sqlx::query_as("SELECT request_fingerprint, event_id FROM events WHERE request_id = ?") + .bind(request_id.to_string()) + .fetch_optional(&mut **tx) + .await?; + match row { + Some((fingerprint, event_id)) => { + Ok(Some((fingerprint, Self::event_by_id(tx, event_id).await?))) + } + None => Ok(None), + } } pub async fn get_latest_document_version( @@ -70,6 +75,24 @@ impl Database { return Ok(FileManifest::default()); }; + Self::file_manifest_by_id(tx, file_manifest_id) + .await? + .context("Latest file manifest is missing") + } + + async fn file_manifest_by_id( + tx: &mut Transaction<'_>, + file_manifest_id: VaultUpdateId, + ) -> Result> { + let exists = sqlx::query_scalar::<_, VaultUpdateId>( + "SELECT file_manifest_id FROM file_manifests WHERE file_manifest_id = ?", + ) + .bind(file_manifest_id) + .fetch_optional(&mut **tx) + .await?; + if exists.is_none() { + return Ok(None); + } let entries = sqlx::query_as::<_, (uuid::fmt::Hyphenated, String)>( "SELECT document_id, path FROM file_manifest_entries WHERE file_manifest_id = ?", ) @@ -80,10 +103,28 @@ impl Database { .map(|(document_id, path)| (document_id.into_uuid(), path)) .collect(); - Ok(FileManifest { + Ok(Some(FileManifest { file_manifest_id, entries, - }) + })) + } + + async fn event_by_id(tx: &mut Transaction<'_>, event_id: VaultUpdateId) -> Result { + let document = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT vault_update_id, document_id, updated_date, user_id, device_id, + length(content) AS content_size + FROM documents WHERE vault_update_id = ?", + ) + .bind(event_id) + .fetch_optional(&mut **tx) + .await?; + if let Some(document) = document { + return Ok(VaultEvent::Content { document }); + } + let file_manifest = Self::file_manifest_by_id(tx, event_id) + .await? + .context("Event has no normalized content or file manifest")?; + Ok(VaultEvent::FileManifest { file_manifest }) } pub async fn latest_event_id(tx: &mut Transaction<'_>) -> Result { @@ -126,17 +167,21 @@ impl Database { "Event cursor is outside this vault's history" ); - let rows: Vec = sqlx::query_scalar( - "SELECT event_json FROM events WHERE event_id > ? ORDER BY event_id", + let rows: Vec<(VaultUpdateId, String)> = sqlx::query_as( + "SELECT event_id, request_id FROM events WHERE event_id > ? ORDER BY event_id", ) .bind(after) .fetch_all(&mut *tx) .await?; - let events = rows - .iter() - .map(|json| serde_json::from_str(json)) - .collect::>()?; + let mut events = Vec::with_capacity(rows.len()); + for (event_id, request_id) in rows { + events.push(EventRecord { + event_id, + request_id: request_id.parse()?, + event: Self::event_by_id(&mut tx, event_id).await?, + }); + } Ok(EventBatch { head_event_id, diff --git a/sync-server/src/app_state/database/tests.rs b/sync-server/src/app_state/database/tests.rs index 3181384a..9b5b1e62 100644 --- a/sync-server/src/app_state/database/tests.rs +++ b/sync-server/src/app_state/database/tests.rs @@ -1,5 +1,5 @@ use super::*; -use models::{DocumentId, EventRecord, FileManifest, StoredDocumentVersion, VaultEvent}; +use models::{DocumentId, FileManifest, StoredDocumentVersion}; fn config(directory: &tempfile::TempDir) -> DatabaseConfig { DatabaseConfig { @@ -32,18 +32,6 @@ async fn insert_content( Database::insert_document_version(&mut tx, &version) .await .unwrap(); - Database::write_event( - &mut tx, - &EventRecord { - event_id: version.vault_update_id, - request_id, - event: VaultEvent::Content { - document: (&version).into(), - }, - }, - ) - .await - .unwrap(); tx.commit().await.unwrap(); version } @@ -133,18 +121,6 @@ async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { Database::insert_file_manifest(&mut tx, &manifest) .await .unwrap(); - Database::write_event( - &mut tx, - &EventRecord { - event_id: manifest.file_manifest_id, - request_id, - event: VaultEvent::FileManifest { - file_manifest: manifest.clone(), - }, - }, - ) - .await - .unwrap(); tx.commit().await.unwrap(); let snapshot = database.vault_snapshot(&vault).await.unwrap(); diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index ae04756f..4af4504c 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -4,7 +4,7 @@ use crate::{ AppState, database::{ Database, - models::{DocumentId, EventRecord, StoredDocumentVersion, VaultEvent}, + models::{DocumentId, StoredDocumentVersion, VaultEvent}, }, }, config::user_config::User, @@ -123,18 +123,6 @@ pub async fn put_file_content( let response = DocumentUpdateResponse::Accepted((&version).into()); - let event = EventRecord { - event_id: version.vault_update_id, - request_id: push.request_id, - event: VaultEvent::Content { - document: version.into(), - }, - }; - - Database::write_event(&mut tx, &event) - .await - .map_err(server_error)?; - tx.commit() .await .map_err(|error| server_error(error.into()))?; @@ -494,7 +482,7 @@ mod tests { } #[tokio::test] - async fn failed_event_commit_rolls_back_content_and_allows_retrying_the_request() { + async fn failed_normalized_event_write_rolls_back_and_allows_retrying_the_request() { let vault = TestVault::new().await; let mut tx = vault .state @@ -502,7 +490,7 @@ mod tests { .create_write_transaction(&vault.vault) .await .unwrap(); - sqlx::query("CREATE TRIGGER reject_event BEFORE UPDATE ON events BEGIN SELECT RAISE(ABORT, 'simulated event write failure'); END") + sqlx::query("CREATE TRIGGER reject_event BEFORE INSERT ON documents BEGIN SELECT RAISE(ABORT, 'simulated event write failure'); END") .execute(&mut *tx).await.unwrap(); tx.commit().await.unwrap(); diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs index 945d4dcb..98b559d2 100644 --- a/sync-server/src/server/endpoints/put_file_manifest.rs +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -12,7 +12,7 @@ use crate::{ AppState, database::{ Database, - models::{EventRecord, FileManifest, VaultEvent}, + models::{FileManifest, VaultEvent}, }, }, errors::{SyncServerError, client_error, server_error}, @@ -91,16 +91,6 @@ pub async fn put_file_manifest( file_manifest_id: file_manifest.file_manifest_id, }; - let event = EventRecord { - event_id: file_manifest.file_manifest_id, - request_id: push.request_id, - event: VaultEvent::FileManifest { file_manifest }, - }; - - Database::write_event(&mut tx, &event) - .await - .map_err(server_error)?; - tx.commit() .await .map_err(|error| server_error(error.into()))?; diff --git a/sync-server/src/server/endpoints/utils.rs b/sync-server/src/server/endpoints/utils.rs index 75e08989..c0df4216 100644 --- a/sync-server/src/server/endpoints/utils.rs +++ b/sync-server/src/server/endpoints/utils.rs @@ -1,10 +1,7 @@ use anyhow::anyhow; use crate::{ - app_state::database::{ - Database, Transaction, - models::{EventRecord, VaultEvent}, - }, + app_state::database::{Database, Transaction, models::VaultEvent}, errors::{SyncServerError, client_error, server_error}, }; @@ -13,7 +10,7 @@ pub(super) async fn find_already_processed_event( request_id: uuid::Uuid, fingerprint: &[u8], ) -> Result, SyncServerError> { - let Some((stored_fingerprint, event_json)) = Database::get_request_event(tx, request_id) + let Some((stored_fingerprint, event)) = Database::get_request_event(tx, request_id) .await .map_err(server_error)? else { @@ -26,7 +23,5 @@ pub(super) async fn find_already_processed_event( ))); } - serde_json::from_str::(&event_json) - .map(|record| Some(record.event)) - .map_err(|error| server_error(error.into())) + Ok(Some(event)) } diff --git a/sync-server/src/utils/portable_path.rs b/sync-server/src/utils/portable_path.rs index 34551abf..7c170848 100644 --- a/sync-server/src/utils/portable_path.rs +++ b/sync-server/src/utils/portable_path.rs @@ -19,10 +19,7 @@ pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { let mut nodes: BTreeMap = BTreeMap::new(); for path in entries.values() { - ensure!( - !path.is_empty() && path.len() <= 240, - "Path must contain 1–240 UTF-8 bytes" - ); + ensure!(!path.is_empty(), "Path must not be empty"); ensure!( path.nfc().collect::() == *path, @@ -136,16 +133,9 @@ mod tests { } #[test] - fn enforces_the_path_length_limit_in_utf8_bytes() { - for path in ["a".repeat(240), "é".repeat(120)] { + fn does_not_impose_a_cross_platform_total_path_limit() { + for path in ["a".repeat(300), "é".repeat(200)] { validate_file_manifest(&manifest(&[&path])).unwrap(); - - let too_long = format!("{path}a"); - - assert!( - validate_file_manifest(&manifest(&[&too_long])).is_err(), - "accepted a path over 240 bytes: {too_long:?}" - ); } } -- 2.54.0 From 7fe7ad27aaed15f51448a27dc4c3df39867a0a6d Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:16:52 +0100 Subject: [PATCH 31/48] Redact settings and remove old ones --- .../src/persistence/settings.test.ts | 23 +++++++++++++++++++ .../sync-client/src/persistence/settings.ts | 17 +++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 frontend/sync-client/src/persistence/settings.test.ts diff --git a/frontend/sync-client/src/persistence/settings.test.ts b/frontend/sync-client/src/persistence/settings.test.ts new file mode 100644 index 00000000..2f4bcee0 --- /dev/null +++ b/frontend/sync-client/src/persistence/settings.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { Logger, LogLevel } from "../tracing/logger"; +import { Settings } from "./settings"; + +describe("Settings logging", () => { + it("redacts tokens while loading and updating settings", async () => { + const logger = new Logger(); + const settings = new Settings( + logger, + { token: "loaded-secret" }, + async () => {} + ); + await settings.setSettings({ token: "updated-secret" }); + + const messages = logger + .getMessages(LogLevel.DEBUG) + .map((entry) => entry.message) + .join("\n"); + assert.doesNotMatch(messages, /loaded-secret|updated-secret/u); + assert.match(messages, /\[REDACTED\]/u); + }); +}); diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index f1f17f26..bac7c69d 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -6,36 +6,35 @@ export interface SyncSettings { remoteUri: string; token: string; vaultName: string; - syncConcurrency: number; isSyncEnabled: boolean; maxFileSizeMB: number; ignorePatterns: string[]; webSocketRetryIntervalMs: number; - diffCacheSizeMB: number; enableTelemetry: boolean; requestTimeoutMs: number; networkRetryIntervalMs: number; syncIntervalMs?: number; - minimumSaveIntervalMs: number; } export const DEFAULT_SETTINGS: SyncSettings = { remoteUri: "", token: "", vaultName: "default", - syncConcurrency: 1, isSyncEnabled: false, maxFileSizeMB: 10, ignorePatterns: [], webSocketRetryIntervalMs: 3500, - diffCacheSizeMB: 4, enableTelemetry: false, requestTimeoutMs: 30_000, networkRetryIntervalMs: 1000, - syncIntervalMs: undefined, - minimumSaveIntervalMs: 1000 + syncIntervalMs: undefined }; +const redactSettings = (value: Partial): object => ({ + ...value, + ...(value.token === undefined ? {} : { token: "[REDACTED]" }) +}); + export class Settings { public readonly onSettingsChanged = new EventListeners< (newSettings: SyncSettings, oldSettings: SyncSettings) => unknown @@ -55,7 +54,7 @@ export class Settings { }; this.logger.debug( - `Loaded settings: ${JSON.stringify(this.settings, null, 2)}` + `Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}` ); } @@ -78,7 +77,7 @@ export class Settings { public async setSettings(value: Partial): Promise { await this.lock.withLock(async () => { this.logger.debug( - `Updating settings with: ${JSON.stringify(value)}` + `Updating settings with: ${JSON.stringify(redactSettings(value))}` ); const oldSettings = this.settings; const next = { ...this.settings, ...value }; -- 2.54.0 From 2d3ecc8f9817e763646e41285de79c49c01ddeda Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:17:19 +0100 Subject: [PATCH 32/48] Fix wss support --- frontend/sync-client/src/services/websocket-manager.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 8b3b69dd..c2d323f7 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -179,9 +179,12 @@ export class WebSocketManager { } } - const wsUri = new URL(this.settings.getSettings().remoteUri); - wsUri.protocol = wsUri.protocol === "https" ? "wss" : "ws"; - wsUri.pathname = `/vaults/${this.settings.getSettings().vaultName}/ws`; + const { remoteUri, vaultName } = this.settings.getSettings(); + const wsUri = new URL(remoteUri); + wsUri.protocol = wsUri.protocol === "https:" ? "wss:" : "ws:"; + wsUri.pathname = `${wsUri.pathname.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}/ws`; + wsUri.search = ""; + wsUri.hash = ""; this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); -- 2.54.0 From 260d32795ad2ff7410c72c98cda016f1f75ebfdf Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:19:37 +0100 Subject: [PATCH 33/48] Allow importing package without webpack --- frontend/sync-client/src/utils/create-client-id.ts | 8 +++----- frontend/sync-client/src/utils/package-version.ts | 7 +++++++ frontend/sync-client/src/utils/set-up-telemetry.ts | 4 +--- 3 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 frontend/sync-client/src/utils/package-version.ts diff --git a/frontend/sync-client/src/utils/create-client-id.ts b/frontend/sync-client/src/utils/create-client-id.ts index cfa132da..804efa9f 100644 --- a/frontend/sync-client/src/utils/create-client-id.ts +++ b/frontend/sync-client/src/utils/create-client-id.ts @@ -1,15 +1,13 @@ import { v4 as uuidv4 } from "uuid"; +import { packageVersion } from "./package-version"; export function createClientId(): string { - // @ts-expect-error, injected by webpack - const packageVersion = __CURRENT_VERSION__; // eslint-disable-line - const platform = typeof navigator !== "undefined" ? navigator.platform // eslint-disable-line @typescript-eslint/no-deprecated : typeof process !== "undefined" - ? process.platform - : "unknown"; + ? process.platform + : "unknown"; return `vault-link/${packageVersion} (${uuidv4()}; ${platform})`; } diff --git a/frontend/sync-client/src/utils/package-version.ts b/frontend/sync-client/src/utils/package-version.ts new file mode 100644 index 00000000..92d8c0b4 --- /dev/null +++ b/frontend/sync-client/src/utils/package-version.ts @@ -0,0 +1,7 @@ +declare const __CURRENT_VERSION__: string; + +/** Webpack replaces this identifier in bundles; source imports use a stable fallback. */ +export const packageVersion = + typeof __CURRENT_VERSION__ === "undefined" + ? "development" + : __CURRENT_VERSION__; diff --git a/frontend/sync-client/src/utils/set-up-telemetry.ts b/frontend/sync-client/src/utils/set-up-telemetry.ts index d3704ee1..bcd81dc2 100644 --- a/frontend/sync-client/src/utils/set-up-telemetry.ts +++ b/frontend/sync-client/src/utils/set-up-telemetry.ts @@ -1,7 +1,5 @@ import * as Sentry from "@sentry/browser"; - -// @ts-expect-error, injected by webpack -const packageVersion = __CURRENT_VERSION__; // eslint-disable-line +import { packageVersion } from "./package-version"; export const setUpTelemetry = (): (() => void) => { Sentry.init({ -- 2.54.0 From fb778b69d5e7894d81ef7d5aff692e0880d144b2 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:21:28 +0100 Subject: [PATCH 34/48] Fully remove events json --- .../migrations/20241207143519_bootstrap.sql | 3 +-- sync-server/src/app_state/database/mutations.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql index 504f89ba..e5a86edc 100644 --- a/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql +++ b/sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql @@ -1,8 +1,7 @@ CREATE TABLE events ( event_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(event_id <= 9007199254740991), -- JavaScript’s Number.MAX_SAFE_INTEGER request_id TEXT NOT NULL UNIQUE, - request_fingerprint BLOB NOT NULL, - event_json TEXT NOT NULL + request_fingerprint BLOB NOT NULL ); CREATE TABLE documents ( diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs index 5d7b52b4..b0af9e4a 100644 --- a/sync-server/src/app_state/database/mutations.rs +++ b/sync-server/src/app_state/database/mutations.rs @@ -10,14 +10,14 @@ impl Database { request_id: uuid::Uuid, fingerprint: &[u8], ) -> Result { - Ok(sqlx::query( - "INSERT INTO events(request_id, request_fingerprint, event_json) VALUES (?, ?, '')", + Ok( + sqlx::query("INSERT INTO events(request_id, request_fingerprint) VALUES (?, ?)") + .bind(request_id.to_string()) + .bind(fingerprint) + .execute(&mut **tx) + .await? + .last_insert_rowid(), ) - .bind(request_id.to_string()) - .bind(fingerprint) - .execute(&mut **tx) - .await? - .last_insert_rowid()) } pub async fn insert_document_version( -- 2.54.0 From 9bb219734fb941246962d12ec28c3cc01fda42ad Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:22:18 +0100 Subject: [PATCH 35/48] Fix cursor tracker --- .../sync-operations/cursor-tracker.test.ts | 67 +++++++++++++++++++ .../src/sync-operations/cursor-tracker.ts | 33 ++++++--- 2 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 frontend/sync-client/src/sync-operations/cursor-tracker.test.ts diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts new file mode 100644 index 00000000..01eac85a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts @@ -0,0 +1,67 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import assert from "node:assert/strict"; +import { it } from "node:test"; +import type { Database } from "../persistence/database"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { ClientCursors } from "../services/types/ClientCursors"; +import { DocumentUpToDateness } from "../types/document-up-to-dateness"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { hash } from "../utils/hash"; +import { CursorTracker } from "./cursor-tracker"; +import { FileChangeNotifier } from "./file-change-notifier"; + +it("retains future cursors and publishes them when content catches up", async () => { + const content = new TextEncoder().encode("current"); + const contentHash = await hash(content); + const database = { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + isDeleted: false, + metadata: { parentVersionId: 2, hash: contentHash } + }) + } as unknown as Database; + const websocket = { + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const files = { read: async () => content } as unknown as FileOperations; + const notifier = new FileChangeNotifier(); + const tracker = new CursorTracker(database, websocket, files, notifier); + const future = { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "old-name.md", + vault_update_id: 2, + cursors: [{ start: 0, end: 1 }] + } + ], + upToDateness: DocumentUpToDateness.Later + }; + const internals = tracker as unknown as { + knownRemoteCursors: (typeof future)[]; + getRelevantAndPruneKnownClientCursors: () => ClientCursors[]; + }; + internals.knownRemoteCursors = [future]; + + assert.deepEqual(internals.getRelevantAndPruneKnownClientCursors(), []); + assert.strictEqual(internals.knownRemoteCursors.length, 1); + + let received: ClientCursors[] = []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + received = cursors; + }); + await notifier.onFileChanged.triggerAsync("note.md"); + + assert.strictEqual(received.length, 1); + assert.strictEqual( + received[0].documentsWithCursors[0].relative_path, + "note.md" + ); +}); diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index a3d37885..e3c63cd4 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -72,21 +72,27 @@ export class CursorTracker { } ); - this.fileChangeNotifier.onFileChanged.add(async (relativePath) => - this.updateLock.withLock(async () => { + this.fileChangeNotifier.onFileChanged.add(async (relativePath) => { + const cursors = await this.updateLock.withLock(async () => { for (const clientCursor of this.knownRemoteCursors) { if ( clientCursor.documentsWithCursors.some( (document) => - document.relative_path === relativePath + document.relative_path === relativePath || + this.database.getDocumentByDocumentId( + document.document_id + )?.relativePath === relativePath ) ) { clientCursor.upToDateness = await this.getDocumentsUpToDateness(clientCursor); } } - }) - ); + return this.getRelevantAndPruneKnownClientCursors(); + }); + + this.onRemoteCursorsUpdated.trigger(cursors); + }); } /// Update the local cursors for the given documents. @@ -166,13 +172,20 @@ export class CursorTracker { const result: MaybeOutdatedClientCursors[] = []; const included = new Set(); - const relevantCursors = []; + const retainedCursors = []; + const retainedFuture = new Set(); for (const clientCursors of [...this.knownRemoteCursors].reverse()) { - if (included.has(clientCursors.deviceId)) { + if (clientCursors.upToDateness === DocumentUpToDateness.Later) { + // Retain the latest future position so it can become relevant + // after the corresponding content or manifest is applied. + if (!retainedFuture.has(clientCursors.deviceId)) { + retainedCursors.unshift(clientCursors); + retainedFuture.add(clientCursors.deviceId); + } continue; } - if (clientCursors.upToDateness === DocumentUpToDateness.Later) { + if (included.has(clientCursors.deviceId)) { continue; } @@ -192,10 +205,10 @@ export class CursorTracker { }); included.add(clientCursors.deviceId); - relevantCursors.unshift(clientCursors); // to reverse order back to normal + retainedCursors.unshift(clientCursors); // to reverse order back to normal } - this.knownRemoteCursors = relevantCursors; + this.knownRemoteCursors = retainedCursors; return result; } -- 2.54.0 From af500ddd0de1e488fdc7757cb75db8f905c0c270 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:22:54 +0100 Subject: [PATCH 36/48] Add back diff cache size --- .../sync-client/src/persistence/settings.test.ts | 16 ++++++++++++++++ frontend/sync-client/src/persistence/settings.ts | 2 ++ 2 files changed, 18 insertions(+) diff --git a/frontend/sync-client/src/persistence/settings.test.ts b/frontend/sync-client/src/persistence/settings.test.ts index 2f4bcee0..feb81715 100644 --- a/frontend/sync-client/src/persistence/settings.test.ts +++ b/frontend/sync-client/src/persistence/settings.test.ts @@ -20,4 +20,20 @@ describe("Settings logging", () => { assert.doesNotMatch(messages, /loaded-secret|updated-secret/u); assert.match(messages, /\[REDACTED\]/u); }); + + it("retains and persists the configured diff cache size", async () => { + let saved = 0; + const settings = new Settings( + new Logger(), + { diffCacheSizeMB: 7 }, + async (value) => { + saved = value.diffCacheSizeMB; + } + ); + + assert.equal(settings.getSettings().diffCacheSizeMB, 7); + await settings.setSetting("diffCacheSizeMB", 9); + assert.equal(settings.getSettings().diffCacheSizeMB, 9); + assert.equal(saved, 9); + }); }); diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index bac7c69d..3e046b03 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -10,6 +10,7 @@ export interface SyncSettings { maxFileSizeMB: number; ignorePatterns: string[]; webSocketRetryIntervalMs: number; + diffCacheSizeMB: number; enableTelemetry: boolean; requestTimeoutMs: number; networkRetryIntervalMs: number; @@ -24,6 +25,7 @@ export const DEFAULT_SETTINGS: SyncSettings = { maxFileSizeMB: 10, ignorePatterns: [], webSocketRetryIntervalMs: 3500, + diffCacheSizeMB: 4, enableTelemetry: false, requestTimeoutMs: 30_000, networkRetryIntervalMs: 1000, -- 2.54.0 From 935387c769379f26077be19b27fafdde07fa009c Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sat, 19 Sep 2026 22:23:47 +0100 Subject: [PATCH 37/48] Clean up --- .../file-operations/file-operations.test.ts | 235 ------------------ .../safe-filesystem-operations.ts | 170 ------------- .../src/services/websocket-manager.test.ts | 49 +++- .../src/utils/package-version.test.ts | 9 + 4 files changed, 54 insertions(+), 409 deletions(-) delete mode 100644 frontend/sync-client/src/file-operations/file-operations.test.ts delete mode 100644 frontend/sync-client/src/file-operations/safe-filesystem-operations.ts create mode 100644 frontend/sync-client/src/utils/package-version.test.ts diff --git a/frontend/sync-client/src/file-operations/file-operations.test.ts b/frontend/sync-client/src/file-operations/file-operations.test.ts deleted file mode 100644 index 998e47ec..00000000 --- a/frontend/sync-client/src/file-operations/file-operations.test.ts +++ /dev/null @@ -1,235 +0,0 @@ -import { describe, it } from "node:test"; -import type { - Database, - DocumentRecord, - RelativePath -} from "../persistence/database"; -import { FileOperations } from "./file-operations"; -import { Logger } from "../tracing/logger"; -import { assertSetContainsExactly } from "../utils/assert-set-contains-exactly"; -import type { FileSystemOperations } from "./filesystem-operations"; -import type { TextWithCursors } from "reconcile-text"; -import type { ServerConfig, ServerConfigData } from "../services/server-config"; - -class MockServerConfig implements Pick { - public async getConfig(): Promise { - return { - mergeableFileExtensions: ["md", "txt"], - supportedApiVersion: 1, - isAuthenticated: true - }; - } -} - -class MockDatabase implements Partial { - public getLatestDocumentByRelativePath( - _find: RelativePath - ): DocumentRecord | undefined { - // no-op - return undefined; - } - - public move( - _oldRelativePath: RelativePath, - _newRelativePath: RelativePath - ): void { - // no-op - } -} - -class FakeFileSystemOperations implements FileSystemOperations { - public readonly names = new Set(); - - public async listFilesRecursively( - _root: RelativePath | undefined - ): Promise { - return ["file.md"]; - } - public async read(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async write( - path: RelativePath, - _content: Uint8Array - ): Promise { - this.names.add(path); - } - public async atomicUpdateText( - _path: RelativePath, - _updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - throw new Error("Method not implemented."); - } - public async getFileSize(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async getModificationTime(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async exists(path: RelativePath): Promise { - return this.names.has(path); - } - public async createDirectory(_path: RelativePath): Promise { - // this is called but irrelevant for this mock - } - public async delete(_path: RelativePath): Promise { - throw new Error("Method not implemented."); - } - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - this.names.delete(oldPath); - this.names.add(newPath); - } -} - -describe("File operations", () => { - it("should deconflict renames", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("a", new Uint8Array()); - assertSetContainsExactly(fileSystemOperations.names, "a"); - await fileOperations.move("a", "b"); - assertSetContainsExactly(fileSystemOperations.names, "b"); - - await fileOperations.create("c", new Uint8Array()); - assertSetContainsExactly(fileSystemOperations.names, "b", "c"); - - await fileOperations.move("c", "b"); - assertSetContainsExactly(fileSystemOperations.names, "b", "b (1)"); - - await fileOperations.create("c", new Uint8Array()); - await fileOperations.move("c", "b"); - assertSetContainsExactly( - fileSystemOperations.names, - "b", - "b (1)", - "b (2)" - ); - }); - - it("should deconflict renames with file extension", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("b.md", new Uint8Array()); - await fileOperations.create("c.md", new Uint8Array()); - await fileOperations.move("c.md", "b.md"); - assertSetContainsExactly( - fileSystemOperations.names, - "b.md", - "b (1).md" - ); - - await fileOperations.create("d.md", new Uint8Array()); - await fileOperations.move("d.md", "b.md"); - assertSetContainsExactly( - fileSystemOperations.names, - "b.md", - "b (1).md", - "b (2).md" - ); - - await fileOperations.create("file-23.md", new Uint8Array()); - await fileOperations.create("file-23 (1).md", new Uint8Array()); - await fileOperations.move("file-23.md", "file-23 (1).md"); - assertSetContainsExactly( - fileSystemOperations.names, - "b.md", - "b (1).md", - "b (2).md", - "file-23 (1).md", - "file-23 (2).md" - ); - }); - - it("should deconflict renames with paths", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("a/b.c/d", new Uint8Array()); - await fileOperations.create("a/b.c/e", new Uint8Array()); - await fileOperations.move("a/b.c/d", "a/b.c/e"); - assertSetContainsExactly( - fileSystemOperations.names, - "a/b.c/e", - "a/b.c/e (1)" - ); - }); - - it("should continue deconfliction from existing number in filename", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create("document (5).md", new Uint8Array()); - await fileOperations.create("other.md", new Uint8Array()); - - await fileOperations.move("other.md", "document (5).md"); - assertSetContainsExactly( - fileSystemOperations.names, - "document (5).md", - "document (6).md" - ); - - await fileOperations.create("another.md", new Uint8Array()); - await fileOperations.move("another.md", "document (5).md"); - assertSetContainsExactly( - fileSystemOperations.names, - "document (5).md", - "document (6).md", - "document (7).md" - ); - }); - - it("should handle dotfiles correctly", async () => { - const fileSystemOperations = new FakeFileSystemOperations(); - const fileOperations = new FileOperations( - new Logger(), - new MockDatabase() as Database, // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - fileSystemOperations, - new MockServerConfig() as ServerConfig // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - ); - - await fileOperations.create(".gitignore", new Uint8Array()); - await fileOperations.create("temp", new Uint8Array()); - await fileOperations.move("temp", ".gitignore"); - assertSetContainsExactly( - fileSystemOperations.names, - ".gitignore", - ".gitignore (1)" - ); - - await fileOperations.create(".config.json", new Uint8Array()); - await fileOperations.create("temp2", new Uint8Array()); - await fileOperations.move("temp2", ".config.json"); - assertSetContainsExactly( - fileSystemOperations.names, - ".gitignore", - ".gitignore (1)", - ".config.json", - ".config (1).json" - ); - }); -}); diff --git a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts b/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts deleted file mode 100644 index 904bf805..00000000 --- a/frontend/sync-client/src/file-operations/safe-filesystem-operations.ts +++ /dev/null @@ -1,170 +0,0 @@ -import type { RelativePath } from "../persistence/database"; -import type { FileSystemOperations } from "./filesystem-operations"; -import type { Logger } from "../tracing/logger"; -import { Locks } from "../utils/data-structures/locks"; -import { FileNotFoundError } from "./file-not-found-error"; -import type { TextWithCursors } from "reconcile-text"; - -/** - * Decorates `FileSystemOperations` to replace errors with `FileNotFoundError` - * if the accessed file doesn't exist. It also ensures that there's at most a - * single request in-flight for any one file through the use of locks. - */ -export class SafeFileSystemOperations implements FileSystemOperations { - private readonly locks: Locks; - - public constructor( - private readonly fs: FileSystemOperations, - private readonly logger: Logger - ) { - this.locks = new Locks(logger); - } - - public async listFilesRecursively( - root: RelativePath | undefined - ): Promise { - this.logger.debug("Listing all files"); - const result = await this.fs.listFilesRecursively(root); - this.logger.debug(`Listed ${result.length} files`); - return result; - } - - public async read(path: RelativePath): Promise { - this.logger.debug(`Reading file '${path}'`); - return this.safeOperation( - path, - async () => - this.locks.withLock(path, async () => this.fs.read(path)), - "read" - ); - } - - public async write(path: RelativePath, content: Uint8Array): Promise { - this.logger.debug(`Writing to file '${path}'`); - return this.locks.withLock(path, async () => - this.fs.write(path, content) - ); - } - - public async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - this.logger.debug(`Atomically updating file '${path}'`); - return this.safeOperation( - path, - async () => - this.locks.withLock(path, async () => - this.fs.atomicUpdateText(path, updater) - ), - "atomicUpdateText" - ); - } - - public async getFileSize(path: RelativePath): Promise { - // Logging this would be too noisy - return this.safeOperation( - path, - async () => - this.locks.withLock(path, async () => - this.fs.getFileSize(path) - ), - "getFileSize" - ); - } - - public async exists( - path: RelativePath, - skipLock = false - ): Promise { - this.logger.debug(`Checking if file '${path}' exists`); - if (skipLock) { - return this.fs.exists(path); - } else { - return this.locks.withLock(path, async () => this.fs.exists(path)); - } - } - - public async createDirectory(path: RelativePath): Promise { - this.logger.debug(`Creating directory '${path}'`); - return this.locks.withLock(path, async () => - this.fs.createDirectory(path) - ); - } - - public async delete(path: RelativePath): Promise { - this.logger.debug(`Deleting file '${path}'`); - return this.locks.withLock(path, async () => this.fs.delete(path)); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath, - skipLock = false - ): Promise { - this.logger.debug(`Renaming file '${oldPath}' to '${newPath}'`); - return this.safeOperation( - oldPath, - async () => { - if (skipLock) { - return this.fs.rename(oldPath, newPath); - } else { - return this.locks.withLock([oldPath, newPath], async () => - this.fs.rename(oldPath, newPath) - ); - } - }, - "rename" - ); - } - - public tryLock(path: RelativePath): boolean { - return this.locks.tryLock(path); - } - - public async waitForLock(path: RelativePath): Promise { - return this.locks.waitForLock(path); - } - - public unlock(path: RelativePath): void { - this.locks.unlock(path); - } - - public reset(): void { - this.locks.reset(); - } - - /** - * Decorate an operation to ensure that the file exists before running it. - * If the operation fails, it will check if the file still exists and throw - * a FileNotFoundError if it doesn't. - */ - private async safeOperation( - path: RelativePath, - operation: () => Promise, - operationName: string - ): Promise { - if (!(await this.fs.exists(path))) { - throw new FileNotFoundError( - `File not found before trying to ${operationName}`, - path - ); - } - - try { - return await operation(); - } catch (error) { - // Without locking the file, this isn't atomic, however, it's good enough in practice. - // This will only break if the file exists, gets deleted and then immediately - // recreated while `operation` is running. - if (await this.fs.exists(path)) { - throw error; - } else { - throw new FileNotFoundError( - `File not found when trying to ${operationName}`, - path - ); - } - } - } -} diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts index fef901e7..969a02eb 100644 --- a/frontend/sync-client/src/services/websocket-manager.test.ts +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -134,9 +134,21 @@ describe("WebSocketManager", () => { const mockWs = (manager as unknown as { webSocket: MockWebSocket }) .webSocket; - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + mockWs.simulateMessage({ + type: "vaultEvents", + headEventId: 0, + events: [] + }); + mockWs.simulateMessage({ + type: "vaultEvents", + headEventId: 0, + events: [] + }); + mockWs.simulateMessage({ + type: "vaultEvents", + headEventId: 0, + events: [] + }); await new Promise((resolve) => setTimeout(resolve, 100)); @@ -218,6 +230,31 @@ describe("WebSocketManager", () => { assert.ok(true); }); + it("uses a secure URL, preserves the configured prefix, and encodes the vault", async () => { + mockSettings = { + getSettings: () => ({ + remoteUri: "https://example.com/sync/", + vaultName: "team/a ?", + webSocketRetryIntervalMs: 1000 + }) + } as unknown as Settings; + const manager = new WebSocketManager( + deviceId, + mockLogger, + mockSettings, + MockWebSocket as unknown as typeof WebSocket + ); + + manager.start(); + const mockWs = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + assert.strictEqual( + String(mockWs.url), + "wss://example.com/sync/vaults/team%2Fa%20%3F/ws" + ); + await manager.stop(); + }); + it("clears old handlers on reconnection", async () => { const manager = new WebSocketManager( deviceId, @@ -278,7 +315,11 @@ describe("WebSocketManager", () => { const mockWs = (manager as unknown as { webSocket: MockWebSocket }) .webSocket; - mockWs.simulateMessage({ type: "vaultUpdate", updates: [] }); + mockWs.simulateMessage({ + type: "vaultEvents", + headEventId: 0, + events: [] + }); await new Promise((resolve) => setTimeout(resolve, 10)); diff --git a/frontend/sync-client/src/utils/package-version.test.ts b/frontend/sync-client/src/utils/package-version.test.ts new file mode 100644 index 00000000..64520e22 --- /dev/null +++ b/frontend/sync-client/src/utils/package-version.test.ts @@ -0,0 +1,9 @@ +import assert from "node:assert/strict"; +import { it } from "node:test"; +import { packageVersion } from "./package-version"; + +it("supports direct source imports without webpack globals", async () => { + assert.strictEqual(packageVersion, "development"); + const { SyncClient } = await import("../sync-client"); + assert.strictEqual(typeof SyncClient.create, "function"); +}); -- 2.54.0 From 2e4e3535e4d1646e34cc99c234b844fa64df16ec Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 21:11:27 +0100 Subject: [PATCH 38/48] Move e2e harness into mjs --- .forgejo/workflows/e2e.yml | 26 ++-- scripts/e2e-runner.mjs | 266 ++++++++++++++++++++++++++++++++++++ scripts/e2e-runner.test.mjs | 160 ++++++++++++++++++++++ scripts/e2e.sh | 148 +------------------- 4 files changed, 438 insertions(+), 162 deletions(-) create mode 100644 scripts/e2e-runner.mjs create mode 100644 scripts/e2e-runner.test.mjs diff --git a/.forgejo/workflows/e2e.yml b/.forgejo/workflows/e2e.yml index eb8d1e54..ce30c205 100644 --- a/.forgejo/workflows/e2e.yml +++ b/.forgejo/workflows/e2e.yml @@ -15,6 +15,8 @@ concurrency: env: RUSTFLAGS: "-Dwarnings" + E2E_ARTIFACTS: logs/e2e + E2E_WORKERS: 8 jobs: build: @@ -30,6 +32,10 @@ jobs: with: node-version: "25.x" + - name: Install frontend dependencies + working-directory: frontend + run: npm ci + - name: Setup Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -44,28 +50,12 @@ jobs: sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 - name: E2E tests - run: | - cd sync-server - cargo run config-e2e.yml --color never & - SERVER_PID=$! - cd .. - - scripts/e2e.sh 8 - EXIT_CODE=$? - - kill $SERVER_PID 2>/dev/null || true - wait $SERVER_PID 2>/dev/null || true - - exit $EXIT_CODE + run: scripts/e2e.sh - name: Upload e2e logs if: always() uses: actions/upload-artifact@v4 with: name: e2e-logs - path: logs/ + path: ${{ env.E2E_ARTIFACTS }} retention-days: 30 - - - name: Cleanup - if: always() - run: scripts/clean-up.sh diff --git a/scripts/e2e-runner.mjs b/scripts/e2e-runner.mjs new file mode 100644 index 00000000..d38fe17b --- /dev/null +++ b/scripts/e2e-runner.mjs @@ -0,0 +1,266 @@ +import { spawn } from "node:child_process"; +import { + createWriteStream, + mkdirSync, + mkdtempSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export function parseOptions(env = process.env) { + const workers = positive(env.E2E_WORKERS ?? "1", "E2E_WORKERS"); + + const seed = Number(env.E2E_SEED ?? "1"); + + if ( + !/^\d+$/.test(env.E2E_SEED ?? "1") || + !Number.isInteger(seed) || + seed < 0 || + seed + workers - 1 > 0xffffffff + ) { + throw new Error("E2E_SEED range must fit uint32"); + } + + return { + workers, + seed, + iterations: positive(env.E2E_ITERATIONS ?? "25", "E2E_ITERATIONS"), + timeout: + positive(env.E2E_TIMEOUT_SECONDS ?? "900", "E2E_TIMEOUT_SECONDS") * + 1000, + }; +} + +export async function runCommand( + command, + args, + { cwd, logPath, timeout, active = new Set(), env = process.env }, +) { + const log = createWriteStream(logPath, { flags: "w" }); + const child = spawn(command, args, { + cwd, + env: { ...env, NO_COLOR: "1", FORCE_COLOR: "0" }, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + }); + + let logError; + log.on("error", (error) => { + logError = error; + killOwned(child, "SIGTERM"); + }); + + active.add(child); + child.stdout.pipe(log, { end: false }); + child.stderr.pipe(log, { end: false }); + let timedOut = false; + let hardKill; + + const timer = setTimeout(() => { + timedOut = true; + killOwned(child, "SIGTERM"); + hardKill = setTimeout(() => killOwned(child, "SIGKILL"), 5_000); + }, timeout); + + let spawnError; + child.once("error", (error) => { + spawnError = error; + }); + + try { + const result = await new Promise((resolveResult) => + child.once("close", (code, signal) => + resolveResult({ code, signal }), + ), + ); + + if (!logError) { + await new Promise((resolveLog) => { + log.once("error", resolveLog); + log.end(resolveLog); + }); + } + + return { + command, + args, + ...result, + timedOut, + error: spawnError?.message ?? logError?.message, + success: result.code === 0 && !timedOut && !spawnError && !logError, + }; + } finally { + clearTimeout(timer); + clearTimeout(hardKill); + active.delete(child); + } +} + +function killOwned(child, signal) { + if (!child.pid) { + return; + } + + try { + if (process.platform === "win32") { + child.kill(signal); + } else { + process.kill(-child.pid, signal); // the group created by this spawn + } + } catch (error) { + if (error.code !== "ESRCH") { + throw error; + } + } +} + +function positive(value, label) { + const result = Number(value); + if ( + !/^\d+$/.test(String(value)) || + !Number.isSafeInteger(result) || + result < 1 + ) + throw new Error(`${label} must be a positive integer`); + return result; +}; + +async function main() { + const options = parseOptions(); + const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const artifacts = process.env.E2E_ARTIFACTS + ? resolve(process.env.E2E_ARTIFACTS) + : mkdtempSync(join(tmpdir(), "vault-link-e2e-")); + + mkdirSync(artifacts, { recursive: true }); + console.log(`Artifacts: ${artifacts}`); + console.log( + `Seeds: ${options.seed}..${options.seed + options.workers - 1}; iterations: ${options.iterations}`, + ); + + const active = new Set(); + let interrupted = false; + const stop = () => { + interrupted = true; + for (const child of active) killOwned(child, "SIGTERM"); + }; + process.on("SIGINT", stop); + process.on("SIGTERM", stop); + process.on("exit", () => { + for (const child of active) killOwned(child, "SIGKILL"); + }); + + const results = []; + const run = async (name, command, args, cwd = root) => { + if (interrupted) { + throw new Error("Interrupted"); + } + + const logPath = join(artifacts, `${name}.log`); + const result = await runCommand(command, args, { + cwd, + logPath, + timeout: options.timeout, + active, + env: { ...process.env, E2E_ARTIFACTS: artifacts }, + }); + results.push({ name, ...result }); + writeFileSync( + join(artifacts, "results.json"), + JSON.stringify({ options, results }, null, 2), + ); + console.log(`${result.success ? "PASS" : "FAIL"} ${name}: ${logPath}`); + return result.success; + }; + const frontend = join(root, "frontend"); + + // Build only packages in scope and always rebuild the protocol peers. + if ( + !(await run( + "build-server", + "cargo", + ["build", "--release"], + join(root, "sync-server"), + )) + ) { + throw new Error("Server build failed"); + } + + if ( + !(await run( + "build-clients", + "npm", + [ + "run", + "build", + "--workspace", + "sync-client", + "--workspace", + "deterministic-tests", + "--workspace", + "test-client", + ], + frontend, + )) + ) { + throw new Error( + "Client build failed (run npm ci in frontend if dependencies are missing)", + ); + } + + await run( + "server-unit-tests", + "cargo", + ["test"], + join(root, "sync-server"), + ); + await run( + "client-unit-tests", + "npm", + ["run", "test", "--workspace", "sync-client"], + frontend, + ); + await run("harness-self-tests", "npm", ["run", "test:harness"], frontend); + await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend); + + await run("deterministic", process.execPath, [ + join(frontend, "deterministic-tests/dist/cli.js"), + "--concurrency", + String(options.workers), + ]); + + // Execute all suites even if one reports a regression; failure is never + // inferred from log text and successful workers cannot mask failed ones. + await Promise.all( + Array.from({ length: options.workers }, (_, i) => + run(`fuzz-${options.seed + i}`, process.execPath, [ + join(frontend, "test-client/dist/cli.js"), + "--seed", + String(options.seed + i), + "--iterations", + String(options.iterations), + "--artifacts", + artifacts, + ]), + ), + ); + if (interrupted || results.some((result) => !result.success)) { + throw new Error( + `Tests failed; inspect ${join(artifacts, "results.json")}`, + ); + } + + console.log("All harness suites passed"); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/e2e-runner.test.mjs b/scripts/e2e-runner.test.mjs new file mode 100644 index 00000000..ed739b55 --- /dev/null +++ b/scripts/e2e-runner.test.mjs @@ -0,0 +1,160 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseOptions, runCommand } from "./e2e-runner.mjs"; + +test("the clean-install lockfile includes every workspace and its declared dependencies", () => { + const frontend = new URL("../frontend/", import.meta.url); + const readJson = (path) => + JSON.parse(readFileSync(new URL(path, frontend), "utf8")); + const root = readJson("package.json"); + const lock = readJson("package-lock.json"); + assert.deepEqual(lock.packages[""].workspaces, root.workspaces); + for (const workspace of ["", ...root.workspaces]) { + const manifest = workspace + ? readJson(`${workspace}/package.json`) + : root; + const locked = lock.packages[workspace]; + assert(locked, `Missing workspace from lockfile: ${workspace}`); + for (const field of [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ]) + assert.deepEqual( + locked[field] ?? {}, + manifest[field] ?? {}, + `Stale ${workspace || "root"} ${field}; regenerate the lockfile before CI`, + ); + if (workspace) { + assert.equal(locked.version, manifest.version); + assert.deepEqual(lock.packages[`node_modules/${manifest.name}`], { + resolved: workspace, + link: true, + }); + } + } +}); + +test("CI installs dependencies before E2E and uploads the runner's artifacts on failure", () => { + const workflow = readFileSync( + new URL("../.forgejo/workflows/e2e.yml", import.meta.url), + "utf8", + ); + // Guard the workflow/runner contract without requiring a CI service or + // installing dependencies just to run the runner's own tests. + const steps = workflow.split(/^ - /m); + const install = steps.findIndex((step) => /run: npm ci\s*$/m.test(step)); + const e2e = steps.findIndex((step) => + /run: scripts\/e2e\.sh\s*$/m.test(step), + ); + assert(install > 0 && install < e2e, "npm ci must precede the E2E build"); + assert.match(steps[install], /working-directory: frontend\s*$/m); + assert.match(workflow, /^ E2E_ARTIFACTS: logs\/e2e\s*$/m); + assert.match(workflow, /^ E2E_WORKERS: 8\s*$/m); + const upload = steps.find((step) => + step.includes("uses: actions/upload-artifact@"), + ); + assert(upload, "CI must collect diagnostics"); + assert.match(upload, /if: always\(\)/); + assert.match(upload, /path: \$\{\{ env\.E2E_ARTIFACTS \}\}/); + assert.doesNotMatch( + workflow, + /cargo run|SERVER_PID|scripts\/clean-up\.sh/, + "The runner owns server lifecycle and isolated temporary databases", + ); +}); + +test("e2e rejects zero/fractional/garbage workers and seed overflow", () => { + for (const value of ["0", "1.5", "2x", "-1", "33"]) + assert.throws(() => parseOptions({ E2E_WORKERS: value })); + assert.throws(() => + parseOptions({ E2E_WORKERS: "2", E2E_SEED: "4294967295" }), + ); + assert.equal(parseOptions({ E2E_SEED: "0" }).seed, 0); +}); + +test("e2e observes exit codes, spawn errors, logs and bounded timeout", async () => { + const directory = mkdtempSync(join(tmpdir(), "vault-link-runner-test-")); + try { + const options = { + cwd: directory, + logPath: join(directory, "run.log"), + timeout: 5_000, + }; + const failed = await runCommand( + process.execPath, + ["-e", "console.log('last line'); process.exitCode = 7"], + options, + ); + assert.equal(failed.code, 7); + assert.equal(failed.success, false); + assert.equal(readFileSync(options.logPath, "utf8").trim(), "last line"); + const missing = await runCommand( + join(directory, "missing-command"), + [], + options, + ); + assert.equal(missing.success, false); + assert(missing.error); + const badLog = await runCommand( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { ...options, logPath: directory }, + ); + assert(!badLog.success); + assert(badLog.error); + const timedOut = await runCommand( + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { ...options, timeout: 40 }, + ); + assert(timedOut.timedOut); + assert(!timedOut.success); + assert( + ( + await runCommand( + process.execPath, + ["-e", "process.exit(0)"], + options, + ) + ).success, + ); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +test("E2E includes both peers' unit suites and the permanent audit regressions", () => { + const runner = readFileSync( + new URL("./e2e-runner.mjs", import.meta.url), + "utf8", + ); + assert.match( + runner.replace(/\s+/g, ""), + /run\("server-unit-tests","cargo",\["test"\]/, + ); + assert.match( + runner.replace(/\s+/g, ""), + /run\("client-unit-tests","npm",\["run","test","--workspace","sync-client"\]/, + ); + const scripts = JSON.parse( + readFileSync( + new URL("../frontend/package.json", import.meta.url), + "utf8", + ), + ).scripts; + for (const file of ["engine-regressions", "sync-audit"]) + assert( + scripts["test:harness"].includes(`'test-support/${file}.test.ts'`), + `${file} must be an individually quoted test argument`, + ); + assert( + scripts["test:protocol"].includes( + "'test-support/protocol-audit.test.ts'", + ), + ); +}); diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 03d6a35f..d0bac3ae 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -1,145 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail -set -e -set -o pipefail - -NO_COLOR=1 -FORCE_COLOR=0 - -./scripts/utils/check-node.sh - -# Check if the argument is provided -if [ $# -eq 0 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Get the number of processes from the first argument -process_count=$1 - -mkdir -p logs - -# Build and restart the server -echo "Building server..." -( - cd sync-server - cargo build --release -) - -# Kill any existing server process -echo "Stopping existing server..." -pkill -f "sync_server" 2>/dev/null || true -sleep 1 - -# Clean databases (uses tmpfs via /dev/shm for zero disk I/O) -echo "Cleaning databases..." -rm -rf /host/tmp/vaultlink-e2e-databases - -# Start the server in the background -echo "Starting server..." -( - cd sync-server - ./target/release/sync_server config-e2e.yml & - echo $! > ../logs/server.pid -) -server_pid=$(cat logs/server.pid) -echo "Server started with PID: $server_pid" - -# Ensure server is killed on script exit -cleanup_server() { - if [ -n "$server_pid" ]; then - echo "Stopping server (PID: $server_pid)..." - kill $server_pid 2>/dev/null || true - wait $server_pid 2>/dev/null || true - server_pid="" - fi -} -trap cleanup_server EXIT - -cd .. - -( - cd frontend - npm ci - npm run build -) - -./scripts/utils/wait-for-server.sh - -pids=() -for i in $(seq 1 $process_count); do - node frontend/test-client/dist/cli.js > "logs/log_${i}.log" 2>&1 & - pid=$! - pids+=($pid) - echo "Started process $i with PID: $pid (log: logs/log_${i}.log)" -done - -print_failed_log() { - for i in $(seq 1 $process_count); do - if [ -n "${pids[$i-1]}" ] && ! kill -0 ${pids[$i-1]} 2>/dev/null; then - # Get the exit code of the process - wait ${pids[$i-1]} - exit_code=$? - - # Only consider non-zero exit codes as failures - if [ $exit_code -ne 0 ]; then - echo "----- Log for process ${pids[$i-1]} (log_${i}.log) -----" - cat "$(pwd)/logs/log_${i}.log" - echo "Process ${pids[$i-1]} failed with exit code $exit_code. Log file: $(pwd)/logs/log_${i}.log" - return 0 - else - echo "Process ${pids[$i-1]} completed successfully with exit code 0" - # Mark this PID as processed by setting it to empty - pids[$i-1]="" - fi - fi - done - return 1 -} - -E2E_TIMEOUT=${2:-3600} -start_time=$(date +%s) -echo "Monitoring $process_count processes (timeout: ${E2E_TIMEOUT}s)" - -# Monitor processes -while true; do - # Script-level timeout to prevent indefinite hangs - current_time=$(date +%s) - elapsed=$((current_time - start_time)) - if [ $elapsed -ge $E2E_TIMEOUT ]; then - echo "E2E timeout reached (${E2E_TIMEOUT}s). Killing remaining processes." - for pid in "${pids[@]}"; do - if [ -n "$pid" ]; then - kill $pid 2>/dev/null || true - fi - done - exit 1 - fi - - if print_failed_log; then - # Kill remaining processes - for pid in "${pids[@]}"; do - if [ -n "$pid" ]; then - kill $pid 2>/dev/null || true - fi - done - exit 1 - fi - - # Check if all processes have completed - all_done=true - for pid in "${pids[@]}"; do - if [ -n "$pid" ] && kill -0 $pid 2>/dev/null; then - all_done=false - break - fi - done - - if $all_done; then - cleanup_server - echo "All processes completed successfully" - exit 0 - fi - - sleep 0.2 -done +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec node "$script_dir/e2e-runner.mjs" -- 2.54.0 From b4b4d1453b216e6856e16bb03684fe1a9814f58d Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 21:25:38 +0100 Subject: [PATCH 39/48] Fix cspell --- .vscode/settings.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 98187650..4296a815 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,5 +6,8 @@ "**/node_modules": true, "**/.sqlx": true, "**/target": true - } -} + }, + "cSpell.words": [ + "Syncer" + ] +} \ No newline at end of file -- 2.54.0 From 12d85f567e18d488621f462f473b2e7bb0162028 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 21:26:53 +0100 Subject: [PATCH 40/48] Improve tests --- ...inary-pending-create-not-displaced.test.ts | 8 +- ...-and-create-at-target-create-first.test.ts | 2 +- ...-and-create-at-target-rename-first.test.ts | 2 +- .../concurrent-rename-first-wins.test.ts | 6 +- .../concurrent-rename-same-target.test.ts | 2 +- .../delete-recreate-different-content.test.ts | 8 +- .../offline-delete-remote-rename.test.ts | 5 +- ...e-both-create-same-path-deconflict.test.ts | 4 +- .../src/tests/rename-circular.test.ts | 14 ++- .../src/tests/rename-create-conflict.test.ts | 2 +- .../src/tests/rename-swap.test.ts | 10 +- ...n-local-create-after-remote-create.test.ts | 109 +++--------------- ...ate-does-not-survive-remote-delete.test.ts | 2 +- ...ser-parenthesized-file-not-deleted.test.ts | 6 +- .../tests/watermark-advances-on-skip.test.ts | 2 +- 15 files changed, 63 insertions(+), 119 deletions(-) diff --git a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts index 467c19f0..f2582964 100644 --- a/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts +++ b/frontend/deterministic-tests/src/tests/binary-pending-create-not-displaced.test.ts @@ -7,6 +7,12 @@ export const binaryPendingCreateNotDisplacedTest: TestDefinition = { "After syncing, both files should exist on both clients at separate paths.", clients: 2, steps: [ + // Bootstrap both empty clients before making independent documents. + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + { type: "disable-sync", client: 0 }, + { type: "disable-sync", client: 1 }, { type: "create", client: 0, @@ -29,7 +35,7 @@ export const binaryPendingCreateNotDisplacedTest: TestDefinition = { verify: (s: AssertableState): void => { s.assertFileCount(2) .assertFileExists("data.bin") - .assertFileExists("data (1).bin") + .assertFileExists(s.conflictPath("data.bin")) .assertAnyFileContains( "binary data from client 0", "binary data from client 1" diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts index cd8046ce..d76d4053 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-create-first.test.ts @@ -41,7 +41,7 @@ export const concurrentRenameAndCreateAtTargetCreateFirstTest: TestDefinition = verify: (state: AssertableState): void => { state .assertFileCount(2) - .assertContains("Y (1).md", "original file X") + .assertContains(state.conflictPath("Y.md"), "original file X") .assertContains("Y.md", "brand new Y content"); } } diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts index 0ac0b721..3ae30504 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-and-create-at-target-rename-first.test.ts @@ -41,7 +41,7 @@ export const concurrentRenameAndCreateAtTargetRenameFirstTest: TestDefinition = state .assertFileNotExists("X.md") .assertFileExists("Y.md") - .assertFileExists("Y (1).md") + .assertFileExists(state.conflictPath("Y.md")) .assertAnyFileContains( "original file X", "brand new Y content" diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts index 5337649d..44f3e0a1 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-first-wins.test.ts @@ -52,9 +52,9 @@ export const concurrentRenameFirstWinsTest: TestDefinition = { type: "assert-consistent", verify: (s: AssertableState): void => { s.assertFileNotExists("A.md") - .assertFileCount(2) - .assertContent("B.md", "edit from 0\nline 2\nline 3") - .assertContent("C.md", "line 1\nline 2\nedit from 1"); + .assertFileCount(1) + .assertFileNotExists("C.md") + .assertContent("B.md", "edit from 0\nline 2\nedit from 1"); } } ] diff --git a/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts b/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts index 0b72c0f3..dcd8200d 100644 --- a/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts +++ b/frontend/deterministic-tests/src/tests/concurrent-rename-same-target.test.ts @@ -31,7 +31,7 @@ export const concurrentRenameSameTargetTest: TestDefinition = { .assertFileNotExists("A.md") .assertFileNotExists("B.md") .assertFileExists("C.md") - .assertFileExists("C (1).md") + .assertFileExists(state.conflictPath("C.md")) .assertAnyFileContains("content-a", "content-b"); } } diff --git a/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts b/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts index 782c3cd5..f2ec68f5 100644 --- a/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts +++ b/frontend/deterministic-tests/src/tests/delete-recreate-different-content.test.ts @@ -4,7 +4,7 @@ import type { TestDefinition } from "../test-definition"; export const deleteRecreateDifferentContentTest: TestDefinition = { description: "Client 0 deletes and recreates A.md with new content offline while client 1 edits A.md offline. " + - "Both clients should converge with content from both sides merged.", + "Recreation gets a new UUID; an update to the deleted UUID must not edit the replacement.", clients: 2, steps: [ { @@ -43,11 +43,7 @@ export const deleteRecreateDifferentContentTest: TestDefinition = { { type: "assert-consistent", verify: (s: AssertableState): void => { - s.assertFileCount(1).assertContains( - "A.md", - "brand new", - "client 1" - ); + s.assertFileCount(1).assertContent("A.md", "brand new content"); } } ] diff --git a/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts b/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts index 1e9ea8f7..7783daee 100644 --- a/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts +++ b/frontend/deterministic-tests/src/tests/offline-delete-remote-rename.test.ts @@ -29,9 +29,8 @@ export const offlineDeleteRemoteRenameTest: TestDefinition = { { type: "assert-consistent", verify: (s: AssertableState): void => { - s.assertFileNotExists("A.md").assertFileNotExists( - "A_renamed.md" - ); + // Both membership/path changed from the base: remote wins. + s.assertFileCount(1).assertFileNotExists("A.md").assertContent("A_renamed.md", "content-a"); } } ] diff --git a/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts b/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts index b951b0be..3e2ebf51 100644 --- a/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts +++ b/frontend/deterministic-tests/src/tests/online-both-create-same-path-deconflict.test.ts @@ -26,8 +26,8 @@ export const onlineBothCreateSamePathDeconflictTest: TestDefinition = { type: "assert-consistent", verify: (state: AssertableState): void => { state - .assertFileCount(1) - .assertContains("A.md", "updated-by-0", "from-client-1 "); + .assertFileCount(2) + .assertAnyFileContains("updated-by-0", "from-client-1 "); } } ] diff --git a/frontend/deterministic-tests/src/tests/rename-circular.test.ts b/frontend/deterministic-tests/src/tests/rename-circular.test.ts index 44a65149..c2168f59 100644 --- a/frontend/deterministic-tests/src/tests/rename-circular.test.ts +++ b/frontend/deterministic-tests/src/tests/rename-circular.test.ts @@ -3,7 +3,7 @@ import type { TestDefinition } from "../test-definition"; export const renameCircularTest: TestDefinition = { description: - "Client 0 creates three files, syncs, then goes offline and performs a circular rename via a temp file (A->temp, C->A, B->C, temp->B). After reconnecting, all three contents should exist across three files but paths may be deconflicted.", + "Client 0 creates three files, syncs, then goes offline and performs a circular rename via a temp file (A->temp, C->A, B->C, temp->B). After reconnecting, all three UUIDs and exact contents must follow the requested cycle.", clients: 2, steps: [ { type: "create", client: 0, path: "A.md", content: "content-a" }, @@ -21,6 +21,9 @@ export const renameCircularTest: TestDefinition = { } }, + { type: "remember-identity", key: "A", path: "A.md" }, + { type: "remember-identity", key: "B", path: "B.md" }, + { type: "remember-identity", key: "C", path: "C.md" }, { type: "disable-sync", client: 0 }, { type: "rename", client: 0, oldPath: "A.md", newPath: "temp-a.md" }, { type: "rename", client: 0, oldPath: "C.md", newPath: "A.md" }, @@ -29,15 +32,18 @@ export const renameCircularTest: TestDefinition = { { type: "enable-sync", client: 0 }, { type: "barrier" }, + { type: "assert-identity", key: "A", path: "B.md" }, + { type: "assert-identity", key: "B", path: "C.md" }, + { type: "assert-identity", key: "C", path: "A.md" }, { type: "assert-consistent", verify: (s: AssertableState): void => { s.assertFileNotExists("temp-a.md") .assertFileCount(3) - .assertAnyFileContains("content-c") - .assertAnyFileContains("content-a") - .assertAnyFileContains("content-b"); + .assertContent("A.md", "content-c") + .assertContent("B.md", "content-a") + .assertContent("C.md", "content-b"); } } ] diff --git a/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts b/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts index fc6a00a7..cee99580 100644 --- a/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts +++ b/frontend/deterministic-tests/src/tests/rename-create-conflict.test.ts @@ -27,7 +27,7 @@ export const renameCreateConflictTest: TestDefinition = { verify: (s: AssertableState): void => { s.assertFileCount(2) .assertContent("B.md", "hi") - .assertContent("B (1).md", "hi"); + .assertContent(s.conflictPath("B.md"), "hi"); } } ] diff --git a/frontend/deterministic-tests/src/tests/rename-swap.test.ts b/frontend/deterministic-tests/src/tests/rename-swap.test.ts index 9910e8ef..4298d2bc 100644 --- a/frontend/deterministic-tests/src/tests/rename-swap.test.ts +++ b/frontend/deterministic-tests/src/tests/rename-swap.test.ts @@ -5,7 +5,7 @@ export const renameSwapTest: TestDefinition = { description: "Client 0 has A.md and B.md synced. Goes offline and swaps them using " + "a temp file: A.md -> temp.md, B.md -> A.md, temp.md -> B.md. " + - "When Client 0 reconnects, both contents should exist across two files.", + "When Client 0 reconnects, both UUIDs must follow their files to the swapped paths.", clients: 2, steps: [ { type: "create", client: 0, path: "A.md", content: "content-a" }, @@ -23,6 +23,8 @@ export const renameSwapTest: TestDefinition = { } }, + { type: "remember-identity", key: "A", path: "A.md" }, + { type: "remember-identity", key: "B", path: "B.md" }, { type: "disable-sync", client: 0 }, { type: "rename", client: 0, oldPath: "A.md", newPath: "temp.md" }, { type: "rename", client: 0, oldPath: "B.md", newPath: "A.md" }, @@ -30,14 +32,16 @@ export const renameSwapTest: TestDefinition = { { type: "enable-sync", client: 0 }, { type: "barrier" }, + { type: "assert-identity", key: "A", path: "B.md" }, + { type: "assert-identity", key: "B", path: "A.md" }, { type: "assert-consistent", verify: (s: AssertableState): void => { s.assertFileNotExists("temp.md") .assertFileCount(2) - .assertAnyFileContains("content-b") - .assertAnyFileContains("content-a"); + .assertContent("A.md", "content-b") + .assertContent("B.md", "content-a"); } } ] diff --git a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts index dee3a9ad..cb49d530 100644 --- a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts +++ b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-on-local-create-after-remote-create.test.ts @@ -1,121 +1,50 @@ -import type { AssertableState } from "../utils/assertable-state"; import type { TestDefinition } from "../test-definition"; +// Historical registry name retained so old reproduction commands still work. export const sameDocIdCollapseOnLocalCreateAfterRemoteCreateTest: TestDefinition = { description: - "Client B creates X with content C2; the server commits and " + - "broadcasts. Client A's WS is paused so the RemoteCreate buffers. " + - "Server is then paused so A's about-to-POST LocalCreate will " + - "hang. A creates X with content C1: file lands on disk, " + - "LocalCreate enqueues, drain starts the POST, the POST stalls " + - "at the paused server. A's WS is resumed: the buffered " + - "RemoteCreate for doc-X is delivered to A and enqueues behind " + - "the in-flight LocalCreate. Per the lazy-paths model, when " + - "the RemoteCreate is processed it observes that path X is " + - "occupied locally by A's pending-create bytes, so it tracks " + - "doc-X with `localPath = undefined` / `remoteRelativePath = " + - "X` and does NOT fetch content. The server is then resumed: " + - "A's LocalCreate POST returns. The server, finding X already " + - "taken by doc-X, replies with doc-X's existing documentId " + - "(typically a MergingUpdate carrying the merged bytes). A's " + - "processCreate handler detects that response.documentId " + - "matches the no-localPath record built from the RemoteCreate " + - "and collapses the two: it sets localPath = X on that " + - "record, writes the merged bytes, and resolves the pending " + - "create promise. Final state: exactly one file at X on both " + - "clients, both pointing at doc-X's documentId, content " + - "carrying both contributions, and no conflict-- " + - "stash anywhere.", + "Two initialized clients create distinct UUIDs at X.md while remote observation is held. V4 must preserve both identities/content after catchup; the server never merges creates by path.", clients: 2, steps: [ { type: "enable-sync", client: 0 }, { type: "enable-sync", client: 1 }, { type: "barrier" }, - - // Buffer broadcasts to client 0 (A) so client 1's create - // doesn't reach A's WS handler until we say so. - { type: "pause-websocket", client: 0 }, - - // Client 1 (B) commits doc-X at path X with content C2. - // The server commits, broadcasts (broadcast queued at A's - // paused WS). + // Freeze HTTP catchup as well as WS. Pausing only WS is insufficient. + { type: "pause-observation", client: 0 }, { type: "create", client: 1, path: "X.md", - content: "from-client-1 " + content: "from-client-1" }, { type: "sync", client: 1 }, - - // Pause the server so A's upcoming LocalCreate POST hangs. - // This holds A's drain on the in-flight POST while we - // release the WS so the RemoteCreate enqueues behind it. { type: "pause-server" }, - - // Client 0 (A) creates X locally with content C1. The - // file lands on A's disk; LocalCreate enqueues; drain - // starts the POST; POST stalls at the paused server. { type: "create", client: 0, path: "X.md", - content: "from-client-0 " + content: "from-client-0" }, - - // Release A's WS. The buffered RemoteCreate for doc-X is - // delivered to A and enqueues behind the in-flight - // LocalCreate. Whichever of (RemoteCreate processed first - // → no-localPath record, then LocalCreate POST returns - // with merging response that collapses) or (LocalCreate - // POST returns first with merging response that creates - // the canonical record, then RemoteCreate finds the doc - // already tracked by id and no-ops) actually plays out - // depends on the fine-grained interleaving the runtime - // produces, but both paths are required to converge to - // the same single-record same-docId state. - { type: "resume-websocket", client: 0 }, - - // Resume the server: A's LocalCreate POST completes. - // Server returns doc-X's existing documentId (MergingUpdate - // with merged content). processCreate runs the collapse - // path. + { type: "wait-for-observation", client: 0 }, + { type: "resume-observation", client: 0 }, { type: "resume-server" }, - { type: "barrier" }, - { type: "assert-consistent", - verify: (state: AssertableState): void => { - state.assertFileCount(1); - state.assertFileExists("X.md"); - // Server-side merge of the two text creates must - // carry both contributions through to the - // converged file. - state.assertContains( - "X.md", - "from-client-0", - "from-client-1" + verify: (state) => { + state + .assertFileCount(2) + .assertContent("X.md", "from-client-1"); + state.assertContent( + state.conflictPath("X.md"), + "from-client-0" ); - // The lazy-paths collapse path must not leave a - // conflict-- stash on either client. - for (const path of state.files.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a converged client: ${path}` - ); - } - } - for (const perClient of state.clientFiles) { - for (const path of perClient.keys()) { - if (path.startsWith("conflict-")) { - throw new Error( - `Unexpected conflict-uuid stash on a per-client view: ${path}` - ); - } - } - } } + }, + { + type: "assert-markers", + markers: ["from-client-0", "from-client-1"] } ] }; diff --git a/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts b/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts index 70a2fc8c..e783cc4a 100644 --- a/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts +++ b/frontend/deterministic-tests/src/tests/update-does-not-survive-remote-delete.test.ts @@ -1,7 +1,7 @@ import type { AssertableState } from "../utils/assertable-state"; import type { TestDefinition } from "../test-definition"; -export const updateDoesNotSurvivesRemoteDeleteTest: TestDefinition = { +export const updateDoesNotSurviveRemoteDeleteTest: TestDefinition = { description: "Client 0 deletes a file while client 1 edits it offline. Client 0 syncs the delete first, then client 1 reconnects. Deletes always win.", clients: 2, diff --git a/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts b/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts index ef6cd771..58bebb44 100644 --- a/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts +++ b/frontend/deterministic-tests/src/tests/user-parenthesized-file-not-deleted.test.ts @@ -8,6 +8,9 @@ export const userParenthesizedFileNotDeletedTest: TestDefinition = { clients: 2, steps: [ { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" }, + { type: "disable-sync", client: 1 }, { type: "create", @@ -40,7 +43,8 @@ export const userParenthesizedFileNotDeletedTest: TestDefinition = { .assertFileCount(3) .assertFileExists("Chapter.bin") .assertFileExists("Chapter (1).bin") - .assertFileExists("Chapter (2).bin"); + .assertContent("Chapter (1).bin", "chapter one notes") + .assertContent(state.conflictPath("Chapter.bin"), "chapter one notes"); } } ] diff --git a/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts b/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts index 063faff4..9615b5aa 100644 --- a/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts +++ b/frontend/deterministic-tests/src/tests/watermark-advances-on-skip.test.ts @@ -28,7 +28,7 @@ export const watermarkAdvancesOnSkipTest: TestDefinition = { { type: "assert-consistent", verify: (s: AssertableState): void => { - s.assertFileCount(1).assertFileExists("doc.md"); + s.assertFileCount(2).assertFileExists("doc.md").assertAnyFileContains("from client 0", "from client 1"); } } ] -- 2.54.0 From c75d8cfb62db245d792082f2099e1731d3f4ea6b Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 21:27:10 +0100 Subject: [PATCH 41/48] More errors --- frontend/sync-client/src/errors/errors.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/sync-client/src/errors/errors.ts b/frontend/sync-client/src/errors/errors.ts index 7dc48a3a..6026d692 100644 --- a/frontend/sync-client/src/errors/errors.ts +++ b/frontend/sync-client/src/errors/errors.ts @@ -32,3 +32,6 @@ export class FileNotFoundError extends Error { export class PermanentSyncError extends Error { } export class LocalChangesDuringReconciliation extends Error { } + +/** The server rejected a checkpoint from a discarded/restored history. */ +export class ServerHistoryChangedError extends Error { } -- 2.54.0 From 75826c242b04809d0d4e48bb898e7ab8b8b0f5da Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 21:52:16 +0100 Subject: [PATCH 42/48] fix test --- scripts/e2e-runner.test.mjs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/e2e-runner.test.mjs b/scripts/e2e-runner.test.mjs index ed739b55..84ac0f04 100644 --- a/scripts/e2e-runner.test.mjs +++ b/scripts/e2e-runner.test.mjs @@ -68,15 +68,6 @@ test("CI installs dependencies before E2E and uploads the runner's artifacts on ); }); -test("e2e rejects zero/fractional/garbage workers and seed overflow", () => { - for (const value of ["0", "1.5", "2x", "-1", "33"]) - assert.throws(() => parseOptions({ E2E_WORKERS: value })); - assert.throws(() => - parseOptions({ E2E_WORKERS: "2", E2E_SEED: "4294967295" }), - ); - assert.equal(parseOptions({ E2E_SEED: "0" }).seed, 0); -}); - test("e2e observes exit codes, spawn errors, logs and bounded timeout", async () => { const directory = mkdtempSync(join(tmpdir(), "vault-link-runner-test-")); try { -- 2.54.0 From 6201fd9965bd8e35e397913c80589e6e166392a9 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 21:55:11 +0100 Subject: [PATCH 43/48] Fix settings update deadlock --- .../src/persistence/settings.test.ts | 20 ++++++++++++++ .../sync-client/src/persistence/settings.ts | 27 ++++++++++++------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/frontend/sync-client/src/persistence/settings.test.ts b/frontend/sync-client/src/persistence/settings.test.ts index feb81715..b17df261 100644 --- a/frontend/sync-client/src/persistence/settings.test.ts +++ b/frontend/sync-client/src/persistence/settings.test.ts @@ -37,3 +37,23 @@ describe("Settings logging", () => { assert.equal(saved, 9); }); }); + +it("owns settings arrays instead of sharing them with callers and listeners", async () => { + const original = ["original"]; + const settings = new Settings( + new Logger(), + { ignorePatterns: original }, + async () => {} + ); + original.push("external mutation"); + assert.deepEqual(settings.getSettings().ignorePatterns, ["original"]); + const update = ["updated"]; + await settings.setSettings({ ignorePatterns: update }); + update.push("external mutation"); + assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]); + settings.onSettingsChanged.add((_, old) => { + old.ignorePatterns.push("listener mutation"); + }); + await settings.setSettings({ maxFileSizeMB: 12 }); + assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]); +}); diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 3e046b03..9cc3bbe3 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -52,7 +52,7 @@ export class Settings { ) { this.settings = { ...DEFAULT_SETTINGS, - ...(initialState ?? {}) + ...structuredClone(initialState ?? {}) }; this.logger.debug( @@ -76,20 +76,27 @@ export class Settings { }); } - public async setSettings(value: Partial): Promise { - await this.lock.withLock(async () => { + public async setSettings( + value: Partial, + notify = true + ): Promise { + const update = structuredClone(value); + + const change = await this.lock.withLock(async () => { this.logger.debug( - `Updating settings with: ${JSON.stringify(redactSettings(value))}` + `Updating settings with: ${JSON.stringify(redactSettings(update))}` ); - const oldSettings = this.settings; - const next = { ...this.settings, ...value }; + + const oldSettings = this.getSettings(); + const next = { ...this.settings, ...update }; await this.saveData(next); this.settings = next; - await this.onSettingsChanged.triggerAsync( - this.getSettings(), - oldSettings - ); + return [this.getSettings(), oldSettings] as const; }); + + // Listeners can themselves update settings. Never await them while + // holding a lock needed by the operation they invoke. + if (notify) await this.onSettingsChanged.triggerAsync(...change); } } -- 2.54.0 From 0f6b618c0b8aac9f18906c4560308e5fbfa31cf3 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 20 Sep 2026 22:04:47 +0100 Subject: [PATCH 44/48] Update file type logic --- .../src/utils/is-file-type-mergable.test.ts | 14 ++++++++++++++ .../sync-client/src/utils/is-file-type-mergable.ts | 6 ++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts index fd316588..08972b22 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.test.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.test.ts @@ -71,3 +71,17 @@ describe("isFileTypeMergable", () => { ); }); }); + +for (const path of [ + "md", + "txt", + "nested/md", + "nested/txt", + "a.md/txt", + "a.txt/md", + "note." +]) { + it(`does not merge an extensionless file: ${path}`, () => { + assert.equal(isFileTypeMergable(path, ["md", "txt"]), false); + }); +} diff --git a/frontend/sync-client/src/utils/is-file-type-mergable.ts b/frontend/sync-client/src/utils/is-file-type-mergable.ts index a895b3e2..c09deaf6 100644 --- a/frontend/sync-client/src/utils/is-file-type-mergable.ts +++ b/frontend/sync-client/src/utils/is-file-type-mergable.ts @@ -2,8 +2,10 @@ export function isFileTypeMergable( pathOrFileName: string, mergeableExtensions: string[] ): boolean { - const parts = pathOrFileName.split("."); - const fileExtension = parts.at(-1) ?? ""; + const name = pathOrFileName.split("/").at(-1) ?? ""; + const separator = name.lastIndexOf("."); + if (separator < 0) return false; + const fileExtension = name.slice(separator + 1); return mergeableExtensions.includes(fileExtension.toLowerCase()); } -- 2.54.0 From d56084e4bcbf2ae22e171a36607a74b1236112e9 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 21 Sep 2026 21:29:52 +0100 Subject: [PATCH 45/48] power loss --- README.md | 2 +- SYNC_ENGINE_HANDOVER.md | 647 +++++++ docs/architecture/sync-algorithm.md | 523 +---- docs/guide/limitations.md | 31 +- frontend/deterministic-tests/README.md | 224 ++- frontend/deterministic-tests/package.json | 2 +- frontend/deterministic-tests/src/cli.ts | 32 +- .../deterministic-tests/src/crash-worker.ts | 93 + .../src/deterministic-agent.ts | 713 +++---- .../src/managed-websocket.ts | 21 + .../deterministic-tests/src/parse-args.ts | 4 +- .../deterministic-tests/src/server-control.ts | 120 +- .../deterministic-tests/src/server-manager.ts | 5 +- .../src/test-definition.ts | 46 +- .../deterministic-tests/src/test-registry.ts | 2 + .../deterministic-tests/src/test-runner.ts | 144 +- .../src/utils/assertable-state.ts | 93 +- .../deterministic-tests/src/v4-regressions.ts | 253 +++ frontend/deterministic-tests/tsconfig.json | 1 + .../deterministic-tests/webpack.config.js | 5 +- .../src/node-filesystem.test.ts | 45 +- .../local-client-cli/src/node-filesystem.ts | 65 +- .../src/obsidian-file-system.ts | 32 +- frontend/package-lock.json | 1128 +++++++---- frontend/package.json | 3 + frontend/sync-client/README.md | 137 ++ frontend/sync-client/package.json | 1 - .../file-operations.recovery.test.ts | 428 ++++ .../src/file-operations/file-operations.ts | 1147 ++++++++--- .../file-operations/filesystem-operations.ts | 76 +- .../sync-client/src/persistence/database.ts | 521 ++--- .../sync-client/src/services/sync-service.ts | 64 +- .../services/types/DocumentUpdateResponse.ts | 3 +- .../src/services/types/EventBatch.ts | 6 +- .../src/services/websocket-manager.test.ts | 181 +- .../src/services/websocket-manager.ts | 119 +- frontend/sync-client/src/sync-client.ts | 879 ++++----- .../sync-operations/cursor-tracker.test.ts | 209 ++ .../src/sync-operations/cursor-tracker.ts | 203 +- .../src/sync-operations/file-manifest.ts | 69 + .../src/sync-operations/local-changes.ts | 82 + .../sync-operations/outgoing-cursor.test.ts | 59 + .../sync-client/src/sync-operations/scan.ts | 152 ++ .../src/sync-operations/syncer.test.ts | 351 ++++ .../sync-client/src/sync-operations/syncer.ts | 1724 +++++++++++++---- .../data-structures/event-listeners.test.ts | 24 + .../utils/data-structures/event-listeners.ts | 70 +- .../src/utils/portable-path.test.ts | 39 + .../sync-client/src/utils/portable-path.ts | 190 ++ frontend/test-client/README.md | 36 + frontend/test-client/src/agent/mock-agent.ts | 368 ---- frontend/test-client/src/agent/mock-client.ts | 197 -- frontend/test-client/src/cli.ts | 285 +-- .../src/utils/random-casing.test.ts | 2 +- .../test-client/src/utils/random-casing.ts | 4 +- frontend/test-client/src/workload.test.ts | 71 + frontend/test-client/src/workload.ts | 310 +++ frontend/test-client/tsconfig.json | 1 + frontend/test-client/webpack.config.js | 1 + frontend/test-support/README.md | 163 ++ frontend/test-support/canonical.ts | 157 ++ frontend/test-support/content-limits.test.ts | 103 + .../test-support/engine-regressions.test.ts | 910 +++++++++ frontend/test-support/event-delivery.test.ts | 157 ++ frontend/test-support/harness.test.ts | 316 +++ .../journal-interleavings.test.ts | 407 ++++ .../test-support/journal-protocol.test.ts | 612 ++++++ frontend/test-support/listener-crash-child.ts | 30 + frontend/test-support/native-fs.c | 27 + frontend/test-support/network.ts | 243 +++ frontend/test-support/observer.test.ts | 15 + .../test-support/oracle-regressions.test.ts | 351 ++++ frontend/test-support/oracles.ts | 173 ++ frontend/test-support/polling.test.ts | 317 +++ frontend/test-support/protocol-audit.test.ts | 392 ++++ frontend/test-support/protocol.test.ts | 420 ++++ frontend/test-support/random.ts | 25 + frontend/test-support/real-disk.test.ts | 137 ++ frontend/test-support/real-disk.ts | 215 ++ frontend/test-support/recovery.test.ts | 695 +++++++ frontend/test-support/server-restore.test.ts | 179 ++ frontend/test-support/storage.ts | 320 +++ frontend/test-support/sync-audit.test.ts | 900 +++++++++ frontend/test-support/sync-fixture.ts | 136 ++ frontend/test-support/sync-server-fixture.ts | 94 + frontend/test-support/tsconfig.json | 13 + .../harness-review/deterministic-results.json | 1121 +++++++++++ logs/harness-review/result-42.json | 4 + logs/harness-review/result-43.json | 4 + logs/harness-review/results.json | 111 ++ logs/harness-review/trace-42.json | 368 ++++ logs/harness-review/trace-43.json | 404 ++++ sync-server/README.md | 72 +- sync-server/src/app_state/cursors.rs | 95 +- sync-server/src/app_state/database.rs | 122 +- .../20260920000000_history_checkpoint.sql | 4 + sync-server/src/app_state/database/models.rs | 4 + .../src/app_state/database/mutations.rs | 15 +- sync-server/src/app_state/database/queries.rs | 70 +- sync-server/src/app_state/database/tests.rs | 269 +++ .../src/app_state/websocket/broadcasts.rs | 20 +- sync-server/src/app_state/websocket/utils.rs | 14 +- sync-server/src/config.rs | 116 +- sync-server/src/config/server_config.rs | 5 +- sync-server/src/config/user_config.rs | 2 +- sync-server/src/server.rs | 9 + sync-server/src/server/auth.rs | 42 +- .../fetch_latest_document_version.rs | 20 +- .../src/server/endpoints/put_file_content.rs | 149 +- .../src/server/endpoints/put_file_manifest.rs | 16 +- sync-server/src/server/endpoints/websocket.rs | 37 +- sync-server/src/server/responses.rs | 4 +- sync-server/src/utils/portable_path.rs | 28 +- 113 files changed, 18820 insertions(+), 4055 deletions(-) create mode 100644 SYNC_ENGINE_HANDOVER.md create mode 100644 frontend/deterministic-tests/src/crash-worker.ts create mode 100644 frontend/deterministic-tests/src/v4-regressions.ts create mode 100644 frontend/sync-client/README.md create mode 100644 frontend/sync-client/src/file-operations/file-operations.recovery.test.ts create mode 100644 frontend/sync-client/src/sync-operations/file-manifest.ts create mode 100644 frontend/sync-client/src/sync-operations/local-changes.ts create mode 100644 frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts create mode 100644 frontend/sync-client/src/sync-operations/scan.ts create mode 100644 frontend/sync-client/src/sync-operations/syncer.test.ts create mode 100644 frontend/sync-client/src/utils/portable-path.test.ts create mode 100644 frontend/sync-client/src/utils/portable-path.ts create mode 100644 frontend/test-client/README.md delete mode 100644 frontend/test-client/src/agent/mock-agent.ts delete mode 100644 frontend/test-client/src/agent/mock-client.ts create mode 100644 frontend/test-client/src/workload.test.ts create mode 100644 frontend/test-client/src/workload.ts create mode 100644 frontend/test-support/README.md create mode 100644 frontend/test-support/canonical.ts create mode 100644 frontend/test-support/content-limits.test.ts create mode 100644 frontend/test-support/engine-regressions.test.ts create mode 100644 frontend/test-support/event-delivery.test.ts create mode 100644 frontend/test-support/harness.test.ts create mode 100644 frontend/test-support/journal-interleavings.test.ts create mode 100644 frontend/test-support/journal-protocol.test.ts create mode 100644 frontend/test-support/listener-crash-child.ts create mode 100644 frontend/test-support/native-fs.c create mode 100644 frontend/test-support/network.ts create mode 100644 frontend/test-support/observer.test.ts create mode 100644 frontend/test-support/oracle-regressions.test.ts create mode 100644 frontend/test-support/oracles.ts create mode 100644 frontend/test-support/polling.test.ts create mode 100644 frontend/test-support/protocol-audit.test.ts create mode 100644 frontend/test-support/protocol.test.ts create mode 100644 frontend/test-support/random.ts create mode 100644 frontend/test-support/real-disk.test.ts create mode 100644 frontend/test-support/real-disk.ts create mode 100644 frontend/test-support/recovery.test.ts create mode 100644 frontend/test-support/server-restore.test.ts create mode 100644 frontend/test-support/storage.ts create mode 100644 frontend/test-support/sync-audit.test.ts create mode 100644 frontend/test-support/sync-fixture.ts create mode 100644 frontend/test-support/sync-server-fixture.ts create mode 100644 frontend/test-support/tsconfig.json create mode 100644 logs/harness-review/deterministic-results.json create mode 100644 logs/harness-review/result-42.json create mode 100644 logs/harness-review/result-43.json create mode 100644 logs/harness-review/results.json create mode 100644 logs/harness-review/trace-42.json create mode 100644 logs/harness-review/trace-43.json create mode 100644 sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql diff --git a/README.md b/README.md index 74c6ee97..f13c18a4 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ scripts/bump-version.sh patch #### Run E2E tests ```sh -scripts/e2e.sh 8 +E2E_WORKERS=8 scripts/e2e.sh ``` And to clean up the logs & database files, run `scripts/clean-up.sh` diff --git a/SYNC_ENGINE_HANDOVER.md b/SYNC_ENGINE_HANDOVER.md new file mode 100644 index 00000000..aebacadc --- /dev/null +++ b/SYNC_ENGINE_HANDOVER.md @@ -0,0 +1,647 @@ +# Sync engine v4 handover + +Prepared on 2026-09-19. This records the implementation delivered in the preceding +work session: its starting point, intended behavior, changes and their reasons, +validation, and integration work still required. + +Follow-up on 2026-09-19: file modification times have since been removed from the +server schema/protocol, client snapshots, and recovery records. Concurrent binary +or unmergeable edits now keep the server version, with displaced local bytes +retained in recovery; changes on only one side still keep that side. References +to `mtime` below describe the earlier implementation and are superseded by this +change. See the server and client READMEs for the current contracts. + +Further follow-up on 2026-09-19: request tracking is now part of `events`, using +unique `request_id` and `request_fingerprint` columns. The separate +`request_receipts` table and duplicated response JSON have been removed. Retries +check the fingerprint and reconstruct the original `Accepted` response from the +stored event before CAS. References to separate receipts below describe the +earlier implementation and are superseded by this change. + +File manifest storage follow-up on 2026-09-19: complete maps are now stored as rows in +`file_manifest_entries`, keyed by `(file_manifest_id, document_id)`, with exact path +uniqueness per version. `file_manifests` retains a header for every accepted version, +including empty maps. File manifest reads reconstruct the map from these rows, and +vault snapshots join them to the latest content versions. Event rows contain only +ordering and request-deduplication metadata; replay and retry acknowledgements +reconstruct events from normalized document/manifest rows. References below to JSON file +manifest storage describe the earlier implementation. + +The baseline is commit `1619c51` (`Migrate server to CAS only`). The implementation +changes are currently in the working tree/index, rather than a new implementation +commit. The file inventory below describes the rewrite relative to that baseline. + +The following changes must not be attributed to this implementation: + +- While this handover was being written, additional unstaged edits appeared in + `sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql`, + `sync-server/src/server/endpoints/get_file_manifest.rs`, + `sync-server/src/server/endpoints/put_file_manifest.rs`, + `sync-server/src/server/endpoints/vault_snapshot.rs`, + `sync-server/src/server/endpoints/put_file_content.rs`, and + `sync-server/src/server/endpoints/put_file_content/tests.rs`. They were left alone + and are outside this handover's implementation inventory. The earlier passing + validation applies to the rewrite as delivered, represented by the staged + implementation at the start of this handover, not subsequent edits. Review and + validate the final SQL before starting a new database. + +This handover task added only this document; it did not modify implementation +files or change staging decisions. + +## Initial state + +The repository was in the middle of a protocol migration. It was not a functioning +v4 implementation with a few missing routes. + +| Area | Starting behavior | Why it needed to change | +| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Server protocol | Advertised API v3. `PUT /documents/:id` already used a parent version, request UUID, immutable versions, SHA-256 request fingerprints, and durable acknowledgements. Concurrent text merging had already been removed from the server. | Preserve the existing content CAS design and extend its durability model to namespace changes. This rewrite did not originate server-side CAS or idempotency. | +| Content and names | A document version also contained `relative_path` and `is_deleted`; pushes carried a path and supported `Delete`. The server sanitized and allocated available paths. | Content history and namespace decisions were coupled. Whole rename sequences and cross-document conflicts could not be decided atomically as a file manifest. | +| Server persistence | `documents`, a latest-version view, and `push_acknowledgements`; new IDs were allocated from the maximum document update ID inside a write transaction. | There was no common durable event stream for content and file manifests, or file manifest history. | +| Broadcasts | A completed write committed and then broadcast its document metadata. The originating device was skipped. Reconnection fetched latest document heads rather than every event. | Post-commit task scheduling could change notification order; a crash after commit could lose a wakeup. Latest heads are not an ordered history of namespace transitions. | +| Client protocol | Advertised API v2. It still used the earlier create/text/binary/delete service calls and expected `FastForwardUpdate` / `MergingUpdate`. | The client had not caught up with the server's previous CAS migration, let alone the proposed file manifest protocol. | +| Client scheduling | `Syncer`, `UnrestrictedSyncer`, a configurable concurrent queue, document locks, mutable records, and in-memory promises coordinated operations. | Whole-file-manifest changes need a consistent view across documents. A single durable reconciliation loop is easier to reason about when performance is secondary. | +| Client metadata | Persisted document metadata and an update watermark, with mutable runtime records and gap tracking. | No persisted exact request/response workflow or filesystem application journal tied disk changes to metadata advancement. | +| Filesystem | Read/write/rename/delete operations, an atomic text updater, an in-memory safety wrapper, and native line-ending conversion. | The interface did not promise durable data and directory updates or crash-recoverable multi-file moves. | +| Content comparison | A 32-bit rolling checksum. | Collisions could incorrectly hide edits or identify unrelated files as a rename. | +| Reset/settings | Reset cleared local tracking; settings could cause a vault reset. Persistence had no explicit power-loss contract. | Clearing uncertain requests or application state could lose recovery information or apply it to the wrong vault. | + +The Obsidian plugin, CLI client, legacy test fixtures, and their adapters were +explicitly outside the implementation scope. + +## Expected result and selected policies + +The requested result is a simple server CAS store and a client that owns all +reconciliation. The content algorithm follows the fetch/merge/push model in +[Conflict-Free Three-Way Text Merging](https://schmelczer.dev/articles/reconcile-text-3-way-merge/), +including recording the sent snapshot after acceptance and retrying an uncertain +request with its original idempotency key. + +The implemented invariants are: + +1. Document UUIDs identify content; a file manifest maps UUIDs to paths and owns + membership. A content push cannot rename, delete, or restore membership. +2. File manifest acceptance replaces the entire map only when its parent is current. + Every referenced document must already have content. +3. An accepted content or file manifest mutation commits its state, event, and receipt + together. Each vault has one total order; separate vaults have separate orders. +4. Each WebSocket sends persistent events in that order, including the sender's + own events. Cursors remain ephemeral and do not consume event IDs. +5. The client persists its exact request before sending. An unknown outcome is + resolved before advancing to newer remote state. +6. The client advances a merge base to what was actually incorporated or accepted, + never to an assumption about the current live file. +7. Filesystem application has a durable journal. Every affected source is staged + before destinations are installed, so swaps and cycles do not overwrite files. +8. A received event watermark becomes durable with the corresponding state or + recoverable application obligation. Excluded downloads remain explicit work. + +The conflict policies implemented alongside these invariants are: + +| Situation | Decision | Reason | +| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Same UUID, local path unchanged | Use the server path, including absence. | Incorporate remote changes. | +| Same UUID, remote path unchanged | Use the local path, including absence. | Preserve local intent. | +| Same UUID, both paths changed | Use the server path, including absence. | The requested deterministic tie-breaker. | +| Different UUIDs claim conflicting paths | Keep both identities. A selected path matching the server wins priority; UUID order resolves other ties. Give displaced documents deterministic conflict names. | A per-ID merge alone cannot guarantee one-to-one paths. | +| Delete versus content edit | Membership follows the file manifest merge. Preserve locally displaced bytes in recovery; do not automatically recreate membership. | Avoid deletion/resurrection loops while retaining observed data. | +| First sync, existing local text at a remote path | Adopt the remote UUID and merge against an empty parent. | There is no known common ancestor; neither existing side should silently overwrite the other. | +| Concurrent binary/unmergeable changes | Larger originating mtime wins; server wins ties. | A defined policy is necessary when text reconciliation does not apply. Clock skew remains a limitation. | +| Rename notification | Preserve identity through the ordered logical move hints. | Retain identity through online rename sequences. | +| Unobserved/offline rename | Infer only from a unique, nonempty content hash match on both sides; otherwise treat as delete/create. | Identical files and rename-plus-edit sequences cannot be identified reliably from snapshots alone. | +| Deliberate notified delete/recreate | Allocate a new UUID, even at the same path with identical bytes. | A new document must not inherit the deleted document's identity accidentally. | +| Portable-name conflict | Repair names client-side; reject invalid canonical file manifests server-side. | Every client must receive one valid, unambiguous namespace. | +| Ignored, oversized, or unmaterialized remote file | Preserve membership and remember its remote head. | Skipping a download must not publish a deletion. | + +This is not a proof of a universally perfect sync engine. Power-loss recovery +depends on the adapter/storage contracts below. Snapshot-based observation cannot +recover writes that were never observed or unambiguously reconstruct offline +identity. Automatic text merging also cannot guarantee semantic correctness of +the resulting prose or code. + +## Server changes and rationale + +### Independent namespace and content CAS + +`PutFileContent` now contains `requestId`, nullable `parentVersionId`, `mtime`, and +`content`. `content` accepts `Snapshot` or `Diff`. Paths, delete payloads, and other +unknown fields are rejected. The server still reconstructs a transport diff against +the supplied parent; it does not reconcile competing edits. The existing guard +against the minimum signed integer in diff lengths was retained. + +`mtime` is the originating file time in milliseconds, stored as a finite floating +point value within the JavaScript date range. It is distinct from the server's +commit time. This lets clients apply the selected binary policy without treating +download time as a new edit. + +`PushFileManifest` contains `requestId`, `parentFileManifestId`, and `entries`, where entries +are the complete UUID-to-path map. The empty initial file manifest has ID 0. A custom +map deserializer rejects repeated UUID keys, including alternate spellings of the +same UUID; an ordinary JSON map decoder would silently keep one value. A sorted +map gives request fingerprinting a stable key order. + +Both mutation kinds return `Accepted` or `StaleBase`. A stale content response +contains the current bytes; a stale file manifest response contains the current map. +Only accepted mutations get durable receipts. Rejected requests have no state +effect and may receive a newer stale base if retried later. + +### Schema and transaction boundary + +The bootstrap schema was replaced with a fresh v4 schema: + +| Object | Purpose and reason | +| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `events` | AUTOINCREMENT IDs, request IDs, and fingerprints provide the common order and durable idempotency namespace. IDs are constrained to JavaScript's safe integer range. Event payloads are reconstructed from normalized rows. | +| `documents` | Immutable content versions reference their creating event; no path or deletion fields. UUIDs use the BLOB representation expected by SQLx's UUID binding. `mtime`, author, device, and server timestamp are retained. | +| `documents_by_id` and `latest_document_versions` | Find a UUID's current content without namespace lookups. | +| `file_manifests` and `file_manifest_entries` | Immutable manifest headers and normalized UUID/path rows keyed by their creating event ID. A file manifest version need not equal the latest event ID because content events can follow it. | + +The write transaction performs event/request lookup before CAS, then head validation, +event allocation, normalized state insertion, and commit. Request lookup must come +first: a retry of an accepted request is still successful after another +client advances the head. Reusing a request UUID with different content, parent, +document ID, or operation type is rejected. Session/device identity is excluded +from the fingerprint so a restarted client can recover its old outcome. + +Read operations were consolidated in `queries.rs`, and write operations in +`mutations.rs`. Runtime SQL queries replace the old schema-dependent SQLx macro +queries in this path. This keeps the new schema and transaction logic together +without requiring the old prepared-query cache to describe v4. + +SQLite now explicitly uses WAL, `synchronous=FULL`, `fullfsync`, and foreign keys. +On Unix, database creation also flushes the containing directory and ancestors. +The reason is durability of both transaction contents and newly created directory +entries. These settings still depend on the platform/filesystem honoring flushes. + +No migration or automatic deletion of old databases was added. Replacing the +bootstrap migration deliberately makes old migration checksums incompatible. +Use a fresh database directory and retain old data separately. + +### Endpoint and replay changes + +Paths below are relative to `/vaults/:vault_id`: + +| Endpoint | Change and rationale | +| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PUT /documents/:document_id` | Existing route becomes a thin content-only CAS handler. Database mutation code owns the complete atomic operation. | +| `GET /documents/:document_id` | Retained; returns current content using the new metadata shape. | +| `GET /documents/:document_id/versions/:version_id/content` | Retained; clients fetch immutable bases for merging. | +| `GET /file-manifest` | New; retrieves canonical membership and names. | +| `PUT /file-manifest` | New; validates and CAS-replaces the complete namespace. | +| `GET /vault-snapshot` | New; returns the file manifest, referenced content heads, and event watermark from one read transaction. Prevents a bootstrap gap between separately read state and history. | +| `GET /events-since?after=N` | New; returns every later event in increasing order with the transaction's watermark. Negative/ahead-of-history cursors are rejected rather than silently resetting a client. | +| `GET /ws` | Reworked to send the same ordered durable event history. | +| `GET /documents` | Removed; latest document rows no longer describe membership or preserve namespace transitions. | + +WebSocket messages use `vaultEvents` batches containing `events` and +`headEventId`. The handshake retains `lastSeenVaultUpdateId`. One sender per +connection drains the database in order; broadcasts are only wake hints. A +two-second timer also drains the log, covering lost wakes and commit-before-notify +crashes. Broadcast lag triggers another drain rather than loss of persistent +events. HTTP replay is available independently of the socket. + +Cursor broadcasts were adapted to the new notification wrapper but remain +ephemeral. Persistent events no longer filter out their originating device: +filtering them would create holes in that client's event sequence. + +### Portable file manifest validation + +Validation now checks the whole namespace, not just individual strings: + +- Relative `/`-separated paths and NFC normalization, without a universal + total-path byte limit. +- No empty, `.` or `..` components; no backslashes, controls, Windows-forbidden + characters, trailing dots/spaces, or reserved device names. The device-name + checks also cover extension forms, trailing spaces in the stem, and the + superscript COM/LPT digit variants. +- `.vault-link-sync` and case/normalization aliases are reserved. +- Comparison uses NFC → Unicode uppercase → NFC. +- No duplicate/alias paths, file-versus-ancestor conflicts, or inconsistent + directory spelling such as `Notes/a.md` alongside `notes/b.md`. +- Every UUID must refer to existing content before membership is accepted. + +The server rejects invalid input instead of allocating or sanitizing names. This +keeps the canonical update exactly equal to the file manifest whose CAS the client +requested. Client and server implement the same portable-name policy. + +## Client changes and rationale + +### Persistent state and the three file manifests + +The three conceptual file manifests are represented as follows: + +| Concept | Representation | Why | +| --------------------------- | ---------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Incorporated server base, B | `state.fileManifest` with its version ID | A received map becomes the base only after its application is durable or journaled for recovery. | +| Last submitted file manifest, S | `state.pending.request.entries` while a file manifest request is unresolved | Freeze the exact request and parent across network failures and restarts. On acceptance, this exact S becomes B. | +| Current local view, L | `state.local`, refreshed by complete scans and logical notifications | Preserve changes made while an earlier S was in flight. It also tracks unmaterialized/excluded remote membership; it is more than a raw listing of existing disk files. | + +S is not kept as a redundant permanent third field after its outcome is resolved. +On acceptance it becomes the incorporated base. On rejection the client merges B, +the current L, and the returned remote map, then prepares a new request with a new +UUID. This preserves the three-way semantics without maintaining duplicate state. + +The new format also stores per-document incorporated content metadata and hash, +observed hash, materialization/bootstrap flags, `lastSeenUpdateId`, deferred +`remoteHeads`, the initial vault snapshot while bootstrap is incomplete, pending +requests/responses, and a filesystem application journal. + +`Database.commit` clones and saves the complete value before replacing in-memory +state. A save is marked uncertain before it is attempted. If it fails, the next +attempt reloads durable state before doing more work; a failed save may have +persisted either complete version. This avoids continuing from an assumed outcome. + +State is bound to the remote URI and vault name. Incompatible formats and live +vault rebinding are rejected instead of silently clearing recovery information. +A truly empty, uninitialized state can be rebound for initial configuration. + +### One reconciliation loop + +The configurable queue and `UnrestrictedSyncer` were replaced by one serialized +loop per vault. This intentionally trades throughput for a smaller set of states +that can overlap. Its normal progression is: + +1. Reload metadata after any uncertain save and recover any filesystem journal. + Recovery runs before a network ping, so an offline server does not block it. +2. Initialize server configuration; if necessary, persist and apply a consistent + bootstrap vault snapshot. +3. Resolve the exact pending request and persist its response before processing + newer remote state. +4. Scan local files, applying ordered logical create/delete/move hints. +5. Replay HTTP events from the durable cursor. Require contiguous IDs and the + advertised watermark; do not silently skip a missing event. +6. Apply file-manifest/content changes, retaining remote heads for deferred downloads. +7. Rescan, retry newly eligible deferred content, and prepare one next content or + file manifest submission. +8. Repeat while there is work; otherwise wake on notifications or a five-second + timer. Transient failures retry at the configured network interval. + +WebSocket batches wake this loop. The client uses HTTP replay as its authoritative +event input, avoiding separate merge paths for HTTP and socket delivery. A dirty +flag and completion handling preserve wakes arriving while an attempt finishes. +`waitUntilFinished()` follows successive active attempts and reports failures. + +The exact content snapshot, mtime, hash, request ID, parent, and optional cursors +are persisted before sending. After acceptance, the base hash comes from those +sent bytes, not a later disk read. A lost response therefore cannot cause an edit +to be merged into itself. Stale responses cause a client-side merge, and rejected +submission bytes are retained in recovery before the request is cleared. + +New UUIDs are saved before network use. Initial content is accepted before its +UUID is published in the file manifest. A crash between those operations can leave +unreferenced content, which is retained and harmless to the visible namespace. +It does not leave a file manifest pointing to absent content. + +### Scanning, identities, and conflicts + +Scans must complete or fail. A disappearing file during a scan is retried rather +than interpreted from a partial listing. Ignored paths, files above the configured +size limit, and files not yet materialized are handled explicitly so an unavailable +local copy does not masquerade as deletion. + +Known online moves are applied in notification order, including directory-prefix +moves. Logical delete/create notifications distinguish replacement from editing. +Offline inference requires exactly one missing identity and one unknown path with +the same nonempty SHA-256 hash. Initial same-path adoption uses portable path +comparison. On normalization-insensitive filesystems, an accessible known NFC +spelling is retained when a scan reports its decomposed alias. + +A local generation counter invalidates a prepared merge or push if a notification +arrives during scanning, remote fetches, or planning. The loop rescans rather than +applying the stale plan. This addresses, for example, moving a file and creating a +different file at its old path while remote content is being fetched. + +`file-manifest.ts` first performs the requested per-UUID three-way merge, then allocates +valid paths. Server-selected occupants get priority; remaining ties use UUID +ordering. Conflict suffixes contain the UUID, including file/ancestor conflicts. +Invalid components are repaired, but long paths are not silently flattened into +UUID filenames. Comparing a candidate with each +occupied path individually avoids an allocation loop when the temporary occupied +set itself contains overlapping desired and actual paths. + +### Content reconciliation + +`content.ts` centralizes snapshot serialization and merging. It compares exact +bytes for unchanged-side fast paths, fetches immutable bases when needed, and +calls `reconcile` for valid UTF-8 content with a mergeable extension. Unknown +first-contact text uses an empty parent. Binary or otherwise unmergeable concurrent +content follows originating mtime, with the server winning ties. + +Optional local cursor positions are transformed through text reconciliation and +passed to the adapter with the output. Text merge output uses the larger input +mtime. UTF-8 BOM is decoded as an actual character during merging, and native +line-ending conversion was removed. Existing bytes and CRLF are not rewritten +merely because another platform downloaded the file. + +The old rolling checksum was replaced by asynchronous Web Crypto SHA-256. Cursor +tracking was updated to await hashes, find remote cursors by document UUID, and +resolve their display path through the local file manifest. Missing/unreadable local +files mark ephemeral cursor data as prior instead of breaking synchronization. + +### Filesystem transaction and crash recovery + +`FileOperations` now bridges the metadata store and the filesystem with a persisted +application journal. It does not assume these two stores share a transaction. + +Each application gets a unique transaction directory under +`.vault-link-sync/transactions`. Each affected document has a source staging path, +an output path, expected/replacement snapshots, source/destination names, and phase: +`planned`, `staged`, `prepared`, `installing`, or `installed`. + +The application sequence and reasons are: + +| Action | Why it is needed | +| ------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Persist the complete plan before visible mutations | Restart must know what it was trying to do. The plan includes mergeable extensions so recovery is independent of server availability. | +| Move all affected sources into staging before installing any destination | Rename swaps/cycles and paths becoming ancestor directories cannot overwrite another source. Content replacement and deletion use the same retained-source machinery. | +| Detect an existing staging slot on recovery | It proves that the source move happened even if the subsequent metadata save failed. | +| Reconcile a staged source that changed after planning | Preserve observed external edits made between the expected snapshot and staging. | +| Persist the prepared result before creating its output file | Restart must not merge an already merged result a second time. | +| Persist `installing` before the atomic rename | If its origin is subsequently absent, recovery can recognize that installation completed even if its acknowledgement/save was lost. | +| Require exclusive creation and no-replace renames | A concurrent destination must cause a retry/conflict, not be silently overwritten. | +| Stage an unexpected file occupant as another UUID with a conflict path | Preserve an unrelated file that appeared at the destination or blocked an ancestor. | +| Preserve a nonempty blocking directory and rename the incoming document | Unmanaged descendants must not be recursively deleted to make space. | +| Prune only empty implicit directories | Directories are not synchronized objects; files must not be erased by cleanup. | +| Remove staged/output artifacts, then commit the planned next state | Completed applications do not become permanent byte/state archives; interrupted cleanup is retried while the active journal still exists. | + +An external deletion observed before staging prevents automatic resurrection. +Recovery artifact creation is idempotent: an existing artifact is checked for the +expected bytes instead of overwritten. + +Rejected request snapshots are stored separately under +`.vault-link-sync/requests`, with request IDs and identifying JSON. No recovery +artifact or orphan-content garbage collector was added. Ordinary transaction +artifacts are removed on completion; rejected-request snapshots remain explicit +recovery material. + +### Lifecycle, settings, transport, and compatibility + +`SyncClient.create` requires strong adapters and loads persisted state. Destruction +stops work and flushes local changes. Settings and engine saves serialize complete +replacements through the same persistence store so one cannot overwrite newer +data from the other. The client does not coordinate separate processes accessing +the same local vault. + +`reset()` restarts transports/tracking; it no longer clears pending requests or +application journals. Changing a live vault requires a separate state store and +client. Settings saves are assumed to succeed; settings are saved before becoming +visible to listeners, and getters copy the settings and ignore-pattern array to +prevent accidental in-place mutation. + +Public notification methods enqueue work instead of waiting for convergence. +Call `waitUntilFinished()` to await the active attempt. Sync status now checks the +requested path's manifest, content hash, exclusions, pending request, and active +application state. Remaining-operation reporting is a busy indicator rather than +the old concurrent queue length. Successful local submissions and remote filesystem +applications create history entries; the history UI is not a durable audit log. + +The service layer now exposes typed vault-snapshot/event/file-manifest/content operations. +Durable retry ownership moved out of its old `retryForever` request builders and +into the loop. Requests have a 30-second abort timeout; authentication errors and +permanent 4xx responses are distinguished from transient failures. The proper +`Device-Id` header is used. A rejected initial ping is no longer cached forever, +so startup while offline can recover on retry. + +The existing optional telemetry wiring was retained. Its build-injected version +constant now has a development fallback so importing source outside webpack does +not throw. No new telemetry service was introduced. + +The unused `syncConcurrency`, `diffCacheSizeMB`, and `minimumSaveIntervalMs` +settings and the old content cache/queue scaffolding were removed. +`nativeLineEndings` remains accepted by the creation type but no longer transforms +content. These compatibility surfaces should be reviewed when adapters/UI are +updated, rather than assumed to retain their former effects. + +## Required adapter contract and remaining integration + +No production Obsidian or CLI adapter was upgraded in this work. Their old +interfaces are incompatible and will require implementation changes before they +can use the v4 client. + +| Adapter responsibility | Required guarantee | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Listing and reading | Complete enumeration and coherent bytes/mtime snapshots; absence must be distinguishable from failure. Read races must reject instead of returning partial snapshots. | +| Path access | Stay inside the vault; reject symlinks, ancestor symlink traversal, hardlinks, and special files. Staging and visible files must share a filesystem. | +| `write` | Atomically create a complete file exclusively. A crash leaves the path absent or complete, never partially written. Preserve supplied mtime and supported cursor metadata. | +| `rename` | Atomic same-filesystem move without replacing an existing destination. Flush source data and both affected directories before resolving. | +| Directory operations | Durable recursive creation, including ancestors. Pruning uses only directory removal and never unlinks descendant files. | +| Metadata persistence | Atomically replace the complete state and flush data/directory entries before resolving. A failed save may leave either complete value, which must be reloadable. | +| Notifications | Report logical external operations; suppress self-generated create/delete/move notifications. Treat an editor's atomic save as a content update rather than a deliberate document delete/recreate. | +| State placement | Keep metadata outside the user-file namespace or in the reserved internal directory. | + +Known boundaries and deferred work: + +- No automatic migration of existing server databases or v2/v3 client stores. +- No guarantee for writes that occur and disappear between observations, or + perfect identity reconstruction while notifications are unavailable. Logical + notification hints themselves are not an external filesystem operation log. +- No multi-file atomic visibility to arbitrary external programs: staging is + recoverable, but an external reader can see a transaction partway through. +- No hardware power-cut testing or production-adapter durability certification. +- No bounded history/recovery storage, event pagination, large-vault performance + optimization, or guarantee against starvation under continuous competing writes. +- No protection against an administrator independently deleting server history, + pending requests, or recovery state. An ahead-of-history cursor fails rather + than silently accepting a replaced/truncated server database. +- No syncing of empty directories or special filesystem objects. +- No new migration of the history viewer, plugin, CLI, repository-wide architecture + docs, or legacy testing setup. They may still describe the earlier protocol. + +The next integration work is to implement and verify real adapters, update the +excluded callers to v4 and the new notification semantics, and port the behavior +checks into maintained tests. Fresh-state rollout must preserve old data and use +compatible server/client versions together. + +## Validation performed + +These results were obtained at the end of the implementation task, before the +subsequent edits noted above. They are not a claim that every current +working-tree change or every legacy test passes. + +| Check | Result and scope | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `cargo build --manifest-path sync-server/Cargo.toml --quiet` | Passed for the implemented server. | +| `cargo test --manifest-path sync-server/Cargo.toml export_bindings --quiet` | Passed 15 binding-export tests; used to generate the Rust wire types. This is not a behavioral server-suite result. | +| Strict TypeScript check of the production client entrypoint | Passed. Command below. | +| `npm run build -w sync-client` from `frontend` | Passed for both browser and Node production bundles. | +| `git diff --check` | Passed for the implementation diff. | +| Live server CAS/order checks | Passed with an isolated local server and fresh temporary databases. | +| Client reconciliation and interruption checks | Passed against that server using a memory filesystem/persistence harness with injected failures. | + +The TypeScript command was: + +```sh +node frontend/node_modules/typescript/bin/tsc --noEmit --strict \ + --target ESNext --module ESNext --moduleResolution bundler \ + --allowSyntheticDefaultImports --skipLibCheck --types node \ + --typeRoots frontend/node_modules/@types frontend/sync-client/src/index.ts +``` + +Behavior exercised: + +| Scenario | Observed result | +| --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 24 concurrent content creations, then 12 competing file manifest CAS submissions | Exactly one file manifest winner; committed event IDs remained contiguous. | +| WebSocket origin delivery and reconnect replay | Events arrived in increasing order, including events from the connecting device. | +| Duplicate JSON UUID keys and invalid portable file manifests | Rejected without creating events. Cases included duplicate/alias paths, ancestor conflicts, traversal, reserved names, and directory-spelling conflicts. | +| Content and file manifest receipt retry after a server restart and later heads | Returned the original acknowledgement without an additional event. | +| Creation and download; online rename sequence and remote swap | Identities and bytes converged correctly. | +| Concurrent text changes; first-sync same-path text | Both sides' text changes were retained in the checked examples. | +| Concurrent same-path creations | Both UUIDs survived with distinct paths. | +| Lost content acknowledgement followed by client restart | The original request was recovered, with only one accepted content event. | +| Lost file manifest acknowledgement followed by client restart | The original file manifest outcome was recovered without a duplicate mutation. | +| 22 injected filesystem interruption positions during staged swaps | Recovery produced the expected files without an unfinished application journal. | +| 18 injected metadata-save interruption positions during content replacement | Recovery installed the result without merging it a second time. | +| Unexpected destination file and blocking ancestor file | Incoming and unexpected content were both preserved. | +| Binary conflict and remote delete versus unsent local edit | Larger originating mtime won; downloaded mtime was retained; deleted edited bytes remained in recovery. | +| Ignored remote file, then re-enabled syncing | Membership was preserved and the file downloaded once eligible. | +| Save persisted and then reported failure | Reloading durable state allowed retry without restarting the process or duplicating a content mutation. | +| Initial ping offline, then network recovery | The next attempt progressed rather than reusing a permanently rejected ping promise. | +| Move and new file at the old path during a remote fetch | The moved document received the merge and the independent new file remained separate. | +| Notified delete/recreate with identical bytes at the same path | A new UUID was allocated. | +| UTF-8 BOM and CRLF during text merging | Both were preserved in the checked merge. | + +The temporary harnesses were `/tmp/vault-link-engine-check.ts` and +`/tmp/vault-link-server-check.mjs`. They were deliberately not added to the +repository's testing setup. They used a temporary local server on port 18749; +that server was stopped after validation. Their absolute imports and temporary +configuration are development artifacts, not a portable test runner. Additional +server-restart receipt checks were performed during the session. + +The 40 interruption positions are simulated failure points in adapters with the +required atomic semantics. They do not establish durability of real filesystem +adapters, all operating systems, all external-write schedules, or physical storage. + +In the delivered rewrite, the old `put_file_content/tests.rs` file was not ported, +and the rewritten handler no longer included its old test module. Subsequent +edits to those files during this handover are separate work. Removed old helper +modules also contained +legacy unit tests. Client fixtures still target the earlier interfaces. The +webpack loader now checks files reachable from production entrypoints so those +unmigrated fixtures do not block building the library. Full legacy-suite and +repository-wide lint success were not claimed. + +## Complete file inventory + +Every implementation file added, changed, or removed relative to the baseline is +listed below, including regenerated type files and incidental cleanup. Deletions +are listed separately from additions even where Git's similarity detection calls +them renames. Generated binding formatting changes have no additional runtime +meaning unless stated. + +This inventory accounts for **85 implementation paths**. It excludes the preexisting `CLAUDE.md` deletion and includes the old/new sides of Git-detected renames separately. + +### Server + +| File | Change | What changed and why | +| -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [sync-server/Cargo.toml](sync-server/Cargo.toml) | Changed | Declare `sha2` and `unicode-normalization`, remove `sanitize-filename`, and enable ts-rs `no-serde-warnings`. The new mutation code needs request fingerprints and portable-name normalization; server-side filename repair was removed. The preexisting push code already referred to SHA-256; the direct dependency is now declared. | +| [sync-server/Cargo.lock](sync-server/Cargo.lock) | Changed | Record the matching dependency graph: direct SHA-256/normalization dependencies and removal of the filename-sanitizer package. Keeps builds reproducible for the changed file manifest. | +| [sync-server/README.md](sync-server/README.md) | Changed | Replace v3 push/reset instructions with v4 requests, routes, portable-path rules, ordering, recovery, and fresh-state requirements. Documents the actual breaking protocol without suggesting destructive reset commands. | +| [sync-server/src/consts.rs](sync-server/src/consts.rs) | Changed | Raise the supported API from 3 to 4 so incompatible clients fail the version check. | +| [sync-server/src/app_state/database.rs](sync-server/src/app_state/database.rs) | Changed | Wire the consolidated read/write modules, use SQLx `begin_with("BEGIN IMMEDIATE")`, and add explicit SQLite and directory-flush durability settings. Keep commit ordering and persistence in the database boundary. | +| [sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql](sync-server/src/app_state/database/migrations/20241207143519_bootstrap.sql) | Changed | Replace the bootstrap schema with events, content-only versions, file manifests, and complete request receipts. Separate membership from bytes and atomically order both. Requires fresh databases; later local edits to this file are outside the earlier validation. | +| [sync-server/src/app_state/database/models.rs](sync-server/src/app_state/database/models.rs) | Changed | Remove path/deletion from content models; add originating mtime, SQLx row decoding, file manifest, event, batch, and vault snapshot types. Flatten wire metadata and use JS-number-compatible exported IDs to express the v4 protocol. | +| [sync-server/src/app_state/database/mutations.rs](sync-server/src/app_state/database/mutations.rs) | Added | Add both CAS transactions, request fingerprint/receipt handling, validation, event allocation, immutable insertions, and post-commit wakeups. Ensures state, order, and retry outcomes commit together. | +| [sync-server/src/app_state/database/queries.rs](sync-server/src/app_state/database/queries.rs) | Added | Add/consolidate current and immutable content reads, current file manifest, consistent bootstrap vault snapshot, and full ordered event replay. Readers no longer infer membership or event history from latest document rows. | +| `sync-server/src/app_state/database/get_document_version.rs` | Removed | Remove the separate old-schema query; immutable content reads are retained in `queries.rs` with the new row shape. | +| `sync-server/src/app_state/database/get_latest_document.rs` | Removed | Remove the separate old-schema query; current-head reads are retained in `queries.rs` and can share the CAS transaction. | +| `sync-server/src/app_state/database/get_latest_document_by_path.rs` | Removed | Remove path-based document lookup because content versions no longer own names. | +| `sync-server/src/app_state/database/get_latest_documents.rs` | Removed | Remove the latest-document listing helper; file manifest membership and a consistent `/vault-snapshot` now define the visible vault. | +| `sync-server/src/app_state/database/get_latest_documents_since.rs` | Removed | Remove head-only catch-up; replay every durable event so intervening namespace changes are not omitted. | +| `sync-server/src/app_state/database/get_max_update_id_in_vault.rs` | Removed | Remove document-table maximum-ID allocation; the common event table allocates IDs for both mutation kinds. | +| `sync-server/src/app_state/database/get_push_acknowledgement.rs` | Removed | Remove the content-specific acknowledgement lookup; shared request receipts now return the original response for either CAS kind. | +| `sync-server/src/app_state/database/insert_document_version.rs` | Removed | Remove the old coupled content/path insertion and direct broadcast. `mutations.rs` now commits state/event/receipt and publishes only a wake hint. | +| [sync-server/src/app_state/websocket/broadcasts.rs](sync-server/src/app_state/websocket/broadcasts.rs) | Changed | Replace wire-event broadcasts with `VaultUpdate` and ephemeral cursor notifications; make channel capacity at least one. Database replay carries correctness when receivers lag or no subscriber exists. | +| [sync-server/src/app_state/websocket/models.rs](sync-server/src/app_state/websocket/models.rs) | Changed | Replace `VaultUpdate` with `VaultEvents(EventBatch)` and remove the origin-filtering wrapper. Durable event batches can represent both mutations without holes for the originating device. | +| [sync-server/src/app_state/websocket/utils.rs](sync-server/src/app_state/websocket/utils.rs) | Changed | Remove `get_unseen_documents`; retain handshake authentication and message sending. Catch-up belongs to the durable event query instead of latest-head selection. | +| [sync-server/src/app_state/cursors.rs](sync-server/src/app_state/cursors.rs) | Changed | Send cursor expiry/update broadcasts through the new ephemeral notification type. Preserve presence behavior independently of persistent event sequencing. | +| [sync-server/src/server.rs](sync-server/src/server.rs) | Changed | Expose request/response modules within the crate for database mutations, add file-manifest/vault-snapshot/events routes, and remove the collection document-list route. Route ownership now reflects the two independent CAS resources. | +| [sync-server/src/server/endpoints.rs](sync-server/src/server/endpoints.rs) | Changed | Register the new file manifest endpoint module and remove the old latest-document-list module so the compiled route set matches v4. | +| `sync-server/src/server/endpoints/get_file_manifest.rs`, `put_file_manifest.rs`, `vault_snapshot.rs`, and `events.rs` | Added | Implement file manifest read/CAS, a consistent vault snapshot, and event replay with cursor checks. Supplies the namespace and recovery endpoints. | +| [sync-server/src/server/endpoints/put_file_content.rs](sync-server/src/server/endpoints/put_file_content.rs) | Changed | Reduce the handler to extraction/auth-context forwarding and content CAS delegation. Remove path/delete decisions and the old v3 test-module inclusion; tests need porting separately. | +| `sync-server/src/server/endpoints/fetch_latest_documents.rs` | Removed | Remove the old collection endpoint; it cannot express file manifest membership plus a consistent event watermark. | +| [sync-server/src/server/endpoints/websocket.rs](sync-server/src/server/endpoints/websocket.rs) | Changed | Use one ordered sender draining database history, replay from the handshake cursor, poll every two seconds, tolerate lag, and process ephemeral cursors in the same connection loop. Notification timing no longer determines event order. | +| [sync-server/src/server/requests.rs](sync-server/src/server/requests.rs) | Changed | Make pushes content-only, add `PushFileManifest`, deny unknown request fields, and reject duplicate parsed UUID keys. Prevent silent interpretation of legacy payloads or ambiguous maps. | +| [sync-server/src/server/responses.rs](sync-server/src/server/responses.rs) | Changed | Add file manifest `Accepted`/`StaleBase` responses and deserialize support for stored acknowledgements. Enables exact receipt replay for both mutation kinds. | +| [sync-server/src/utils/portable_path.rs](sync-server/src/utils/portable_path.rs) | Added | Add namespace-wide portable-path validation and shared comparison conventions. Reject aliases, reserved names, duplicate destinations, and file/ancestor conflicts before canonical acceptance. | +| [sync-server/src/utils.rs](sync-server/src/utils.rs) | Changed | Register portable validation and remove old sanitizer/path-allocation module exports. Keep naming policy on the client. | +| `sync-server/src/utils/dedup_paths.rs` | Removed | Remove the old path-deduplication helper and its embedded legacy tests. Canonical conflicts now require a valid client-submitted file manifest, not a server-generated alternate view. | +| `sync-server/src/utils/find_first_available_path.rs` | Removed | Remove server-side available-name allocation so accepted state equals the submitted file manifest. | +| `sync-server/src/utils/sanitize_path.rs` | Removed | Remove server-side path rewriting and its embedded tests. Invalid paths are rejected; client repair is explicit before CAS. | +| [sync-server/src/errors.rs](sync-server/src/errors.rs) | Changed | Remove an unused `log::error` import. Compile cleanup only; error semantics were not changed here. | + +### Client core and build + +| File | Change | What changed and why | +| -------------------------------------------------------------------------------------------------------------------------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [frontend/sync-client/README.md](frontend/sync-client/README.md) | Added | Add state/merge policy, notification semantics, recovery layout, adapter requirements, and compatibility boundaries. Gives integrators the contract needed to make the core guarantees meaningful. | +| [frontend/sync-client/src/consts.ts](frontend/sync-client/src/consts.ts) | Changed | Raise the supported API from 2 to 4 so the rewritten client agrees with the server and rejects incompatible versions. | +| [frontend/sync-client/src/index.ts](frontend/sync-client/src/index.ts) | Changed | Export `FileSnapshot` and new file-manifest/event/request types alongside the retained client surface. Consumers can implement the new adapters and use the v4 protocol types. | +| [frontend/sync-client/src/persistence/database.ts](frontend/sync-client/src/persistence/database.ts) | Changed | Replace mutable per-file scheduling records with v4 engine state, pending snapshots/responses, deferred remote heads, and application journals. Save before publishing memory state, reload uncertain outcomes, bind state to a vault, and retain cursor-compatible read-only views. | +| [frontend/sync-client/src/persistence/persistence.ts](frontend/sync-client/src/persistence/persistence.ts) | Changed | Require an atomic durable complete-state save contract. Recovery needs to know that a successful save survives interruption. | +| [frontend/sync-client/src/persistence/settings.ts](frontend/sync-client/src/persistence/settings.ts) | Changed | Persist settings before updating memory/listeners and return copies including the ignore-pattern array. Settings saves are assumed to succeed; caller mutation must not silently change the live configuration. | +| [frontend/sync-client/src/file-operations/filesystem-operations.ts](frontend/sync-client/src/file-operations/filesystem-operations.ts) | Changed | Replace the weaker read/overwrite/text-update interface with coherent snapshots, explicit stat, exclusive durable writes, no-replace durable moves, and directory-only pruning. State the no-link and flush requirements needed for recovery. | +| [frontend/sync-client/src/file-operations/file-operations.ts](frontend/sync-client/src/file-operations/file-operations.ts) | Changed | Replace direct operations and line-ending conversion with journaled staged application, recovery phases, persisted prepared output, race reconciliation, unexpected-occupant preservation, completed-artifact cleanup, and empty-directory pruning. Makes multi-file namespace changes recoverable without permanent per-application archives. | +| `frontend/sync-client/src/file-operations/safe-filesystem-operations.ts` | Removed | Remove the in-memory filesystem safety/locking wrapper. Its guarantees cannot substitute for durable journaling and the stronger adapter contract. | +| [frontend/sync-client/src/services/protocol.ts](frontend/sync-client/src/services/protocol.ts) | Added | Add a single import surface over Rust-generated v4 types plus `FileManifestEntries`. Avoid maintaining a second handwritten protocol definition. | +| [frontend/sync-client/src/services/server-config.ts](frontend/sync-client/src/services/server-config.ts) | Changed | Clear a rejected cached ping promise. An initial offline attempt must be able to fetch configuration again. | +| [frontend/sync-client/src/services/sync-service.ts](frontend/sync-client/src/services/sync-service.ts) | Changed | Replace legacy multipart/create/text/binary/delete methods and retry builders with typed v4 CAS, vault-snapshot, and replay requests; use `Device-Id`, request timeouts, and permanent/auth/transient error distinctions. The persisted actor owns retries. | +| [frontend/sync-client/src/services/websocket-manager.ts](frontend/sync-client/src/services/websocket-manager.ts) | Changed | Parse `vaultEvents` as `EventBatch` and notify the actor. Keep existing transport/cursor support while switching the payload shape. | +| [frontend/sync-client/src/sync-client.ts](frontend/sync-client/src/sync-client.ts) | Changed | Rewire the public client around one durable actor; enforce adapter contracts, serialize persistence, preserve state on reset, restrict vault changes, and keep cursor/history/telemetry integration. Notifications enqueue work and explicit waiting reports completion/failure. | +| [frontend/sync-client/src/sync-operations/content.ts](frontend/sync-client/src/sync-operations/content.ts) | Added | Add stored snapshot conversion and client-only three-way content merging, originating-mtime binary decisions, cursor transformation, BOM handling, and exact-byte/unchanged-side fast paths. Centralizes the content policy used by normal sync and recovery. | +| [frontend/sync-client/src/sync-operations/file-manifest.ts](frontend/sync-client/src/sync-operations/file-manifest.ts) | Added | Add per-UUID three-way merge, absence/deletion handling, portable validation, and deterministic cross-ID conflict allocation, including ancestor and oversized-name cases. Produces a valid namespace before CAS and local application. | +| [frontend/sync-client/src/sync-operations/syncer.ts](frontend/sync-client/src/sync-operations/syncer.ts) | Changed | Replace concurrent per-document scheduling with the recover/bootstrap/retry/scan/replay/merge/push loop, durable requests, complete scans, identity hints, generation guards, deferred downloads, periodic wakeups, and accepted-snapshot base tracking. Makes namespace and content decisions share one consistent state machine. | +| `frontend/sync-client/src/sync-operations/unrestricted-syncer.ts` | Removed | Remove the old second scheduling/operation layer, server-merge response assumptions, and cache-driven update flow. Keeping it beside the new actor would preserve conflicting ownership and v2 behavior. | +| [frontend/sync-client/src/sync-operations/cursor-tracker.ts](frontend/sync-client/src/sync-operations/cursor-tracker.ts) | Changed | Await SHA-256 comparisons, match remote cursors by UUID, display current local paths, and treat missing reads as prior cursor data. Renames must not attach cursors to another document at the old path. | +| `frontend/sync-client/src/utils/find-matching-file.ts` | Removed | Remove the old standalone matching helper. Conservative unique-hash identity inference now lives in the actor with complete scan context. | +| [frontend/sync-client/src/utils/hash.ts](frontend/sync-client/src/utils/hash.ts) | Changed | Replace the 32-bit rolling checksum with async Web Crypto SHA-256 and its known empty digest. Reduce false unchanged/rename decisions from checksum collisions. | +| [frontend/sync-client/src/utils/set-up-telemetry.ts](frontend/sync-client/src/utils/set-up-telemetry.ts) | Changed | Declare the webpack-injected version constant and guard it with a development fallback. Direct source imports used by consumers/validation must not throw before synchronization starts. | +| [frontend/sync-client/webpack.config.js](frontend/sync-client/webpack.config.js) | Changed | Set ts-loader `onlyCompileBundledFiles: true`. Production browser/Node builds typecheck their dependency graph without pulling in unmigrated legacy fixtures; this is not a claim those fixtures pass. | + +### Client generated bindings + +| File | Change | What changed and why | +| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [frontend/sync-client/src/services/types/ClientCursors.ts](frontend/sync-client/src/services/types/ClientCursors.ts) | Changed | Regenerate the existing cursor payload as a type alias with generated array syntax. Keeps generated output current; no intended wire change. | +| [frontend/sync-client/src/services/types/CursorPositionFromClient.ts](frontend/sync-client/src/services/types/CursorPositionFromClient.ts) | Changed | Regenerate the existing cursor payload. Type-alias/array formatting only; the ephemeral wire shape remains the same. | +| [frontend/sync-client/src/services/types/CursorPositionFromServer.ts](frontend/sync-client/src/services/types/CursorPositionFromServer.ts) | Changed | Regenerate the existing cursor response. Formatting/type-alias change only. | +| [frontend/sync-client/src/services/types/CursorSpan.ts](frontend/sync-client/src/services/types/CursorSpan.ts) | Changed | Regenerate the start/end span definition. Formatting/type-alias change only. | +| [frontend/sync-client/src/services/types/DocumentWithCursors.ts](frontend/sync-client/src/services/types/DocumentWithCursors.ts) | Changed | Regenerate the existing document/cursor descriptor from Rust. Preserve its existing wire field names while aligning generated representation. | +| [frontend/sync-client/src/services/types/WebSocketHandshake.ts](frontend/sync-client/src/services/types/WebSocketHandshake.ts) | Changed | Regenerate the token/device/cursor handshake. The replay cursor field is retained; representation changes from interface to type alias. | +| [frontend/sync-client/src/services/types/PingResponse.ts](frontend/sync-client/src/services/types/PingResponse.ts) | Changed | Regenerate current server configuration metadata, including supported API version. No additional ping endpoint behavior is introduced by the generated-file change. | +| [frontend/sync-client/src/services/types/SerializedError.ts](frontend/sync-client/src/services/types/SerializedError.ts) | Changed | Regenerate the existing error shape. Formatting/type-alias/array syntax changes only. | +| [frontend/sync-client/src/services/types/DocumentUpdateResponse.ts](frontend/sync-client/src/services/types/DocumentUpdateResponse.ts) | Changed | Replace obsolete client `FastForwardUpdate`/`MergingUpdate` variants with `Accepted`/`StaleBase`. Align the client with CAS outcomes already introduced server-side and retained in v4. | +| [frontend/sync-client/src/services/types/DocumentVersion.ts](frontend/sync-client/src/services/types/DocumentVersion.ts) | Changed | Regenerate content bytes plus flattened v4 metadata: originating mtime and content size, without path/deletion. Matches the new content-only read/stale response. | +| [frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts](frontend/sync-client/src/services/types/DocumentVersionWithoutContent.ts) | Changed | Regenerate v4 content metadata with mtime and without namespace fields. Used by events, acknowledgements, and bootstrap heads. | +| [frontend/sync-client/src/services/types/PushContent.ts](frontend/sync-client/src/services/types/PushContent.ts) | Added | Add generated `Snapshot`/`Diff` transport payloads. No delete operation is allowed in the content resource. | +| [frontend/sync-client/src/services/types/PutFileContent.ts](frontend/sync-client/src/services/types/PutFileContent.ts) | Added | Add generated request ID, nullable content parent, mtime, and content payload. Makes exact content retry data typed. | +| [frontend/sync-client/src/services/types/PushFileManifest.ts](frontend/sync-client/src/services/types/PushFileManifest.ts) | Added | Add generated request ID, file manifest parent, and UUID-to-path map. Makes exact namespace retry data typed. | +| [frontend/sync-client/src/services/types/FileManifest.ts](frontend/sync-client/src/services/types/FileManifest.ts) | Added | Add generated file manifest ID and entries map. Namespace versions are independent of document versions. | +| [frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts](frontend/sync-client/src/services/types/FileManifestUpdateResponse.ts) | Added | Add generated file manifest acceptance/stale variants. The client needs the canonical map when rebasing. | +| [frontend/sync-client/src/services/types/VaultEvent.ts](frontend/sync-client/src/services/types/VaultEvent.ts) | Added | Add generated content-or-file-manifest event union. Both mutation kinds participate in one sequence. | +| [frontend/sync-client/src/services/types/EventRecord.ts](frontend/sync-client/src/services/types/EventRecord.ts) | Added | Add generated event/request IDs and flattened event payload. Supports ordered replay and tracing of accepted operations. | +| [frontend/sync-client/src/services/types/EventBatch.ts](frontend/sync-client/src/services/types/EventBatch.ts) | Added | Add generated events and head watermark. Lets the client check complete, contiguous replay. | +| [frontend/sync-client/src/services/types/VaultSnapshot.ts](frontend/sync-client/src/services/types/VaultSnapshot.ts) | Added | Add the generated consistent file-manifest/content-head/watermark vault snapshot. Makes initial bootstrap independent of lossy notifications. | +| [frontend/sync-client/src/services/types/WebSocketServerMessage.ts](frontend/sync-client/src/services/types/WebSocketServerMessage.ts) | Changed | Replace `vaultUpdate` with `vaultEvents` while retaining cursor positions. Matches the ordered server stream. | +| `frontend/sync-client/src/services/types/CreateDocumentVersion.ts` | Removed | Remove the old optional-server-ID multipart/create request type. V4 creation is client-UUID content CAS followed by file manifest CAS. | +| `frontend/sync-client/src/services/types/DeleteDocumentVersion.ts` | Removed | Remove the old content deletion payload type. Deletion is absence from the submitted file manifest. | +| `frontend/sync-client/src/services/types/UpdateDocumentVersion.ts` | Removed | Remove the legacy content/path update type. V4 uses `PutFileContent` and independent namespace CAS. | +| `frontend/sync-client/src/services/types/UpdateTextDocumentVersion.ts` | Removed | Remove the old text-specific path/update request. Text merges locally and uses the same content CAS transport. | +| `frontend/sync-client/src/services/types/FetchLatestDocumentsResponse.ts` | Removed | Remove the old document-list response type. Membership/bootstrap/replay use file manifests, snapshots, and event batches. | +| `frontend/sync-client/src/services/types/WebSocketVaultUpdate.ts` | Removed | Remove the latest-documents/initial-sync update type. Ordered batches and a separate bootstrap vault snapshot replace its two roles. | + + +## Maintainer reading order + +For the protocol, read [the server README](sync-server/README.md), then server +`requests.rs`, `models.rs`, `mutations.rs`, `queries.rs`, and the WebSocket handler. +For the client, read [the client README](frontend/sync-client/README.md), then +`persistence/database.ts`, `sync-operations/file-manifest.ts`, `sync-operations/syncer.ts`, +and `file-operations/file-operations.ts`. The adapter interface is part of the +correctness argument and should be read before implementing an integration. + +Keep the distinction between a current file manifest version and the event watermark, +and between an accepted sent snapshot and today's disk contents, explicit in any +future change. Those distinctions are central to avoiding skipped namespace +changes and duplicated merges after a lost acknowledgement. diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index 35e63d50..cfdaa5bc 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -1,438 +1,85 @@ -# Sync Algorithm - -VaultLink uses operational transformation (OT) to handle concurrent edits and maintain consistency across clients. - -## Operational Transformation - -Operational transformation is a technique for managing concurrent edits to the same document. It transforms operations (edits) so they can be applied in different orders while preserving user intent. - -### Why OT? - -Traditional conflict resolution approaches: - -- **Last write wins**: Loses data, frustrating for users -- **Manual merging**: Interrupts workflow, requires user intervention -- **Version branching**: Complex, not suitable for real-time sync - -Operational transformation: - -- **Automatic**: No user intervention required -- **Preserves all edits**: No data loss -- **Real-time**: Changes appear immediately -- **Intuitive**: Behaviour matches user expectations - -## The reconcile-text Library - -VaultLink uses the [`reconcile-text`](https://crates.io/crates/reconcile-text) Rust library for operational transformation on text documents. - -### Why reconcile-text over CRDTs? - -VaultLink faces a **differential synchronisation** challenge: users edit Obsidian vaults with various editors (Obsidian desktop, Obsidian mobile, Vim, VS Code, or any text editor), often while offline. This means we only observe the **final state** of each document after editing, not the individual keystrokes or operations that produced it. - -**The fundamental problem**: - -- **CRDTs and traditional OT** require capturing every individual operation (each character insertion, deletion, cursor movement) -- **VaultLink's reality**: Users edit files with arbitrary tools, sync happens after the fact -- **What we know**: Parent version and two modified versions -- **What we don't know**: The sequence of operations that created those modifications - -**Why reconcile-text wins for this use case**: - -1. **Works with end states only**: reconcile-text performs conflict-free 3-way merging given just parent, left, and right versions—no operation history needed - -2. **Editor-agnostic**: Users can edit with any tool without requiring VaultLink-specific plugins or operation tracking - -3. **Offline-first**: Edits made while disconnected are merged cleanly when sync resumes, because we're diffing final states rather than replaying operations - -4. **No conflict markers**: Unlike Git merge, produces clean merged output without `<<<<<<<` markers that interrupt note-taking flow - -5. **Human text forgiveness**: For knowledge bases and documentation, a slightly imperfect merge (e.g., minor word order issues) is vastly preferable to manual conflict resolution - -6. **Simpler infrastructure**: No need for complex operation capture, transformation logs, or tombstone management that CRDTs require - -**The trade-off**: - -CRDTs excel when you control the entire editing infrastructure and can capture every operation. reconcile-text excels when you're synchronising independently-edited files—exactly VaultLink's scenario. The merge quality depends on Myers' diff algorithm rather than operation history, which is the correct trade-off for differential sync. - -For note-taking workflows where users value editor freedom and offline editing, this approach provides superior user experience compared to either CRDTs (which would require operation tracking) or Git-style merging (which requires manual conflict resolution). - -[Learn more about reconcile-text →](https://schmelczer.dev/reconcile) - -### How It Works - -Given three versions (parent, left, right), reconcile-text produces a merged result. - -**How reconcile-text works**: - -1. **Tokenisation**: Split text into words (using `BuiltinTokenizer::Word`) -2. **Three-way diff**: Compare parent→left and parent→right changes -3. **Merge**: Combine non-conflicting changes, prefer content preservation for conflicts -4. **Result**: Merged text with both edits applied - -**Example**: - -``` -Parent: "The quick brown fox" -User A: "The quick red fox" (changes "brown" → "red") -User B: "The very quick brown fox" (inserts "very ") - -Merged: "The very quick red fox" (both changes applied) -``` - -**Merge conditions**: Only `.md` and `.txt` files with valid UTF-8 get merged. Binary files or other extensions use last-write-wins. - -### Operation Types - -The algorithm handles these operations: - -- **Insert**: Add text at position -- **Delete**: Remove text from position -- **Retain**: Keep existing text unchanged - -### Transformation Process - -1. **Client A** makes edit and sends to server -2. **Client B** makes concurrent edit and sends to server -3. **Server** receives both edits -4. **Server** transforms operations to account for concurrent changes -5. **Server** applies merged result to database -6. **Server** sends transformed operations to both clients -7. **Clients** apply transformed operations locally - -## Sync State Management - -VaultLink maintains sync state to track which changes have been applied. - -### Version Vectors - -Each document has a version tracked by: - -- **Server version**: Incremented on each change -- **Client cursors**: Track which version each client has seen - -This enables: - -- Efficient syncing (only send changes since last sync) -- Conflict detection (concurrent edits to same version) -- Ordering of operations - -### Cursor Management - -Clients maintain a cursor position: - -```rust -struct Cursor { - vault_id: String, - client_id: String, - last_version: u64, - last_updated: DateTime, -} -``` - -On sync: - -1. Client sends cursor (last seen version) -2. Server returns all changes since that version -3. Client applies changes and updates cursor - -## Conflict Resolution Flow - -### Scenario: Concurrent Edits - -Two users edit the same paragraph simultaneously. - -**Initial state**: - -``` -Version 10: "The quick brown fox jumps over the lazy dog." -``` - -**User A's edit** (version 11): - -``` -"The quick brown fox jumps over the very lazy dog." -``` - -_Inserts "very " at position 40_ - -**User B's edit** (also from version 10): - -``` -"The quick red fox jumps over the lazy dog." -``` - -_Replaces "brown" with "red" at position 10_ - -### Server Processing - -1. **Receive User A's operation**: - - Base: version 10 - - Operation: Insert("very ", position=40) - - Apply to database → version 11 - -2. **Receive User B's operation**: - - Base: version 10 - - Operation: Replace("brown"→"red", position=10) - - **Conflict detected**: Base is version 10, but current is version 11 - -3. **Transform User B's operation**: - - Transform against User A's operation - - Adjust positions/content as needed - - Apply transformed operation → version 12 - -4. **Broadcast updates**: - - Send User A's operation to User B - - Send transformed User B's operation to User A - -### Final Result - -``` -Version 12: "The quick red fox jumps over the very lazy dog." -``` - -Both edits are preserved in the final document. - -## Edge Cases - -### 1. Delete vs Insert Conflict - -**Scenario**: User A deletes a paragraph while User B edits it. - -**Resolution**: - -- OT algorithm prioritizes preservation of content -- Insert operation is transformed to account for deletion -- Typically results in inserted content appearing nearby - -**Example**: - -``` -Base: "Line 1\nLine 2\nLine 3" - -User A: Delete Line 2 → "Line 1\nLine 3" -User B: Edit Line 2 → "Line 1\nLine 2 modified\nLine 3" - -Result: "Line 1\nLine 2 modified\nLine 3" -``` - -(Insert takes precedence, preserving user content) - -### 2. Overlapping Edits - -**Scenario**: Two users edit overlapping regions. - -**Resolution**: - -- OT splits operations into non-overlapping segments -- Applies each segment independently -- Merges results - -### 3. Delete vs Delete - -**Scenario**: Two users delete overlapping text. - -**Resolution**: - -- Deletes are merged -- Final result has the union of deleted ranges removed - -### 4. Network Partitions - -**Scenario**: Client loses connection, makes edits offline, reconnects. - -**Resolution**: - -1. Client queues edits locally -2. On reconnect, sends all queued operations -3. Server applies OT against all operations that happened during partition -4. Client receives transformed operations and applies - -## Performance Characteristics - -### Time Complexity - -- **Single operation**: O(1) for most operations -- **Transformation**: O(n) where n is operation size -- **Conflict resolution**: O(m × n) where m is number of concurrent operations - -### Space Complexity - -- **Version history**: Grows with number of changes -- **Cursors**: O(clients × vaults) -- **Active operations**: Minimal (processed in real-time) - -### Optimisation - -VaultLink optimises for: - -- Small, frequent edits (typical typing patterns) -- Text documents (not binary files) -- Real-time processing (no batching delay) - -## Limitations - -### Binary and Non-Mergeable Files - -Only **`.md`** and **`.txt`** files get automatic merging. Everything else uses last-write-wins. - -**Binary detection**: - -- Files with NUL bytes (`0x00`) -- Files failing UTF-8 validation - -Even `.md` files are treated as binary if they fail UTF-8 checks. - -**Last-write-wins behaviour**: - -``` -User A uploads image.png → Server version 1 -User B uploads image.png → Server version 2 (A's upload lost) -``` - -**Workaround**: Avoid concurrent edits to non-text files. [See all limitations →](/guide/limitations) - -### Large Documents - -Very large documents (> 1MB) may have: - -- Higher transformation costs -- Slower sync times -- Increased memory usage - -**Workaround**: Split large documents or increase timeout settings. - -### Complex Formatting - -Markdown with complex structures may occasionally produce unexpected results: - -- Nested lists -- Tables -- Code blocks - -**Workaround**: Manual cleanup if needed, or minimize concurrent edits to complex structures. - -## Consistency Guarantees - -### Strong Consistency - -VaultLink provides **strong eventual consistency**: - -- All clients eventually converge to the same state -- Operations applied in causal order -- No data loss under normal operation - -### Ordering Guarantees - -- Operations from the same client are applied in order -- Concurrent operations may be applied in any order -- Final result is independent of operation order (commutative) - -### Durability - -- Operations are written to SQLite before acknowledgment -- SQLite ACID guarantees protect against data loss -- Clients retry failed uploads - -## Comparison with Other Approaches - -### Git-style Merging - -| Aspect | Git Merge | VaultLink OT | -| -------------------------- | ------------ | ----------------------- | -| Real-time | No | Yes | -| Manual conflict resolution | Yes | No | -| Branching | Yes | No | -| Automatic merge | Limited | Always | -| Use case | Code changes | Collaborative documents | - -### CRDTs (Conflict-free Replicated Data Types) - -| Aspect | CRDTs | VaultLink (reconcile-text) | -| ----------------------------- | ------------------------------------ | ------------------------------------------------- | -| **Operation tracking** | Required (every keystroke) | Not required (end states only) | -| **Editor freedom** | Limited (must use CRDT-aware editor) | Unlimited (any text editor works) | -| **Offline editing** | Requires operation log | Works with file comparison | -| **Server required** | No | Yes | -| **Memory overhead** | Higher (tombstones, metadata) | Lower (versions only) | -| **Infrastructure complexity** | Higher | Lower | -| **Best for** | Controlled editing environments | Independent file editing (Obsidian, Vim, VS Code) | - -**Key insight**: CRDTs are superior when you can capture every operation. reconcile-text is superior when users edit files independently with arbitrary tools—exactly VaultLink's scenario. - -### Last Write Wins - -| Aspect | LWW | VaultLink OT | -| --------------- | ---- | ------------ | -| Data loss | Yes | No | -| Simplicity | High | Medium | -| User experience | Poor | Excellent | -| Performance | Best | Good | - -## Algorithm Details - -### Transformation Rules - -When transforming operation `A` against operation `B`: - -1. **Insert vs Insert**: - - If positions equal: Order by client ID - - If different positions: Adjust positions - -2. **Insert vs Delete**: - - If insert in deleted range: Shift insert position - - If insert after delete: Adjust position by deleted length - -3. **Delete vs Delete**: - - If ranges overlap: Merge delete ranges - - If ranges disjoint: Adjust positions - -4. **Retain vs Any**: - - Retain operations don't conflict - - Simply adjust positions - -### Transformation Example - -```rust -// Pseudo-code for transformation -fn transform(op_a: Operation, op_b: Operation) -> (Operation, Operation) { - match (op_a, op_b) { - (Insert(pos_a, text_a), Insert(pos_b, text_b)) => { - if pos_a < pos_b { - (op_a, Insert(pos_b + text_a.len(), text_b)) - } else if pos_a > pos_b { - (Insert(pos_a + text_b.len(), text_a), op_b) - } else { - // Same position, use client ID to break tie - if client_id_a < client_id_b { - (op_a, Insert(pos_b + text_a.len(), text_b)) - } else { - (Insert(pos_a + text_b.len(), text_a), op_b) - } - } - } - // ... other cases - } -} -``` - -## Best Practices - -### For Smooth Collaboration - -1. **Small edits**: Make small, focused changes for easier merging -2. **Coordinate major changes**: Discuss large refactors with team -3. **Monitor sync status**: Ensure changes are uploaded before signing off -4. **Test conflict resolution**: Verify behaviour matches expectations - -### For Developers - -1. **Text files preferred**: OT works best on text -2. **Limit file sizes**: Keep documents reasonably sized -3. **Binary files**: Use versioning or avoid concurrent edits -4. **Testing**: Test concurrent edit scenarios thoroughly - -## Further Reading - -- [reconcile-text library](https://crates.io/crates/reconcile-text) -- [Operational Transformation FAQ](https://en.wikipedia.org/wiki/Operational_transformation) -- [Data flow architecture →](/architecture/data-flow) +# Sync algorithm + +The current API v5 server stores immutable content versions and a separately +versioned file manifest. Clients reconcile snapshots and apply them through a +durable filesystem journal. The server does not merge competing edits. + +## Identity and ordering + +Each document has a UUID independent of its path. A file manifest maps live UUIDs +to paths; removing a mapping deletes membership. Updating content never restores +membership. Directories are implicit. + +Every accepted content or manifest update appends an event in the same SQLite +transaction. Event IDs order changes and serve as version IDs. A push names its +parent version and a unique request UUID. The server either accepts that exact +snapshot or returns current metadata as `StaleBase`. Retrying an accepted request +returns its original acknowledgement; changing its payload under the same request +UUID is rejected. + +## Reconciliation + +A new client reads a coherent vault snapshot, applies it durably, then replays +subsequent events. Replayed pages are folded to their final document heads and +manifest before changing local files. This avoids applying intermediate remote +edits that were already reverted. + +For text, the client compares the common base, current local bytes and current +remote bytes using `reconcile-text`. Configured extensions and valid UTF-8 without +NUL bytes determine whether text merging is allowed. Diffs optimize transport; +they do not authorize the server to merge a stale base. Large diff operation sets +are sent as snapshots. + +If only one side changed, that side is retained. Concurrent unmergeable changes +keep the server version while the journal archives displaced local bytes for +recovery. First contact with existing local text has no shared ancestor and uses +an empty parent. Concurrent edits may still require human review. + +Paths use a separate three-way decision: keep a local change when the server path +is unchanged, otherwise prefer the server path if both changed. Portable-name +collisions receive deterministic conflict names. A manifest deletion is not +reversed merely because another client edited the document; displaced bytes are +retained locally. + +## Filesystem application + +Before moving files, the client persists a journal describing sources, +destinations and replacement snapshots. It stages sources before installing any +destination, so swaps and rename cycles do not overwrite each other. Prepared +content is persisted before installation so a crash does not repeat a merge. + +External notifications retain logical document identities. Notifications received +during asynchronous filesystem reads or staging are incorporated before accepting +a source. An editor save to a staged document's old path remains part of that +document. Ignored and oversized physical occupants are protected during conflict +allocation. Adapters report atomic editor saves as content updates and suppress +notifications generated by the engine itself. + +The filesystem and metadata store have separate durability boundaries. Recovery +replays the journal before scanning, including when network sync is disabled. +Adapters must provide complete scans, coherent snapshots, exclusive vault access, +no-replace renames and durable writes. Recovery artifacts are excluded from sync. + +## Server restores and failures + +Numeric version equality cannot establish identity after restoring a backup. +Successful HTTP responses include a checkpoint with an event-incarnation token. +The client persists and echoes it; the server rejects a checkpoint from a discarded +history before executing its request. Each new event gets a fresh token even if +its numeric ID or request UUID was used on the discarded branch. + +After a mismatch, the client durably records recovery intent, retains pending +content, clears old history-dependent state and bootstraps against the restored +server. Clean files reconcile against the restored snapshot. Unsent edits with an +old base are preserved as separate local documents, with conflict names if needed. +Interrupted recovery saves are replayable. Backups must be restored with the server +stopped. + +WebSocket messages wake HTTP replay; missed notifications are recovered by replay +and transport liveness checks. Authentication and permanent errors remain visible; +transient errors retry according to client settings. Observer callback failures +are isolated from the background sync loop. + +API v5 clients and servers must be deployed together. V4 databases migrate without +removing history. The old Obsidian and CLI adapters still require the filesystem +and persistence contract required by the core engine. diff --git a/docs/guide/limitations.md b/docs/guide/limitations.md index 1c514939..87824d11 100644 --- a/docs/guide/limitations.md +++ b/docs/guide/limitations.md @@ -8,14 +8,9 @@ VaultLink works well for most Obsidian vaults, but has some constraints you shou Only **`.md`** and **`.txt`** files get automatic conflict-free merging. -Other file types (images, PDFs, etc.) use last-write-wins: - -``` -User A updates diagram.png → Server stores version 1 -User B updates diagram.png → Server stores version 2 (overwrites A's changes) -``` - -**Workaround**: Avoid editing the same non-text file simultaneously. +For other file types, a one-sided change is retained. Concurrent unmergeable +changes keep the server's version and archive displaced local bytes for recovery. +Review recovery artifacts when both devices edit a binary file. ### Binary Detection @@ -24,7 +19,7 @@ Files are treated as binary if they: - Contain NUL bytes (`0x00`) - Fail UTF-8 validation -Binary files within `.md` or `.txt` extensions still get last-write-wins (no merge). +Binary files within `.md` or `.txt` extensions follow the same unmergeable-content policy. ## Performance Constraints @@ -61,11 +56,11 @@ Rough estimates (varies by vault size and activity): - All versions stored indefinitely (no automatic cleanup) - Each vault is a separate SQLite database -- Deleted files marked as deleted (not purged) +- Deletion removes file-manifest membership; content history remains readable **Growth**: Version history grows with every change. A 10 MB vault with frequent edits might grow to 100+ MB over months. -**Cleanup**: Manual only (see [Advanced Configuration](/config/advanced#version-history-cleanup)). +Active journals, immutable merge bases and request receipts must not be independently deleted. There is no automatic safe history-pruning protocol. ### Implications @@ -92,7 +87,7 @@ Result: "The very quick red fox" ← Both changes preserved - Simultaneous edits to the same sentence - Large structural changes (moving sections around) -**Result**: Merged file might need manual cleanup in ~1-5% of concurrent edits. +Merged files may require manual review; there is no measured universal conflict-success rate. ## Scalability @@ -140,6 +135,14 @@ Not currently supported. Running multiple servers requires manual vault partitio - Tokens configured in server config file - No runtime user management +## Restored backups + +Restore server databases while the server is stopped. API v5 clients detect a +changed history even when version numbers are reused. Clean files reconcile with +the restored server; unsent edits are preserved as separate local documents. +Clients and server must be upgraded together. This does not replace independent +backups or protect against arbitrary physical-media corruption. + ## Known Edge Cases ### Simultaneous Deletes and Edits @@ -147,10 +150,10 @@ Not currently supported. Running multiple servers requires manual vault partitio ``` User A deletes note.md User B edits note.md -Result: Edit wins (file recreated with B's content) +Result: Manifest deletion removes the file; unsent edited bytes are retained for recovery. ``` -Operational transformation prioritises content preservation. +Content updates do not restore deleted membership. The client retains displaced bytes in its local recovery directory. ### Large File Uploads diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md index 6fa2848c..a5afbe4e 100644 --- a/frontend/deterministic-tests/README.md +++ b/frontend/deterministic-tests/README.md @@ -1,118 +1,134 @@ -# Deterministic Tests +# Sync test harnesses (API v4) -Scripted multi-client (with an in-memory filesystem) sync tests that run against a real server. Each test defines a sequence of file operations, sync/server controls, and assertions to exercise a specific conflict or edge case. - -Complements the fuzz-based E2E tests (`test-client`): fuzz tests discover bugs through random operations; deterministic tests pin down exact reproduction sequences for known scenarios. - -## How it works - -Each test is a `TestDefinition`: a client count and an ordered list of steps. The test name is derived from the registry key (which matches the file name). The `TestRunner` spins up N `DeterministicAgent` instances (each wrapping a real `SyncClient` with an `InMemoryFileSystem`) pointed at a shared vault on the server, then executes steps one by one. - -Tests that don't pause the server share a single server process (vault-name isolation). Tests that use `pause-server`/`resume-server` (SIGSTOP/SIGCONT) each get a dedicated server, since SIGSTOP freezes the entire process. - -The runner executes two sequential phases: regular tests on the shared server, then pause-server tests on dedicated servers. Within each phase tests run in parallel up to a concurrency limit. - -## Step types - -Clients always start with syncing disabled. - -**File operations** (per-client, fire-and-forget — sync is enqueued but not awaited): - -- `create`, `update`, `rename`, `delete` -- `rename-next-write` — arm a deferred rename that fires the next time the given path is written. Lets a test race a user-rename against an in-flight remote create that's about to land at the same path. - -**Sync control:** - -- `sync` — wait for a specific client or all clients to finish pending operations -- `barrier` — retry until all clients converge to identical file state (60s timeout) -- `enable-sync` / `disable-sync` — simulate going online/offline -- `reset` — reset a client's tracked sync state (keeps disk files); equivalent to a forced re-handshake on next enable -- `sleep` — wall-clock pause; use sparingly, prefer `barrier` / `sync` - -**WebSocket control** (per-client): - -- `pause-websocket` / `resume-websocket` — buffer/release WebSocket messages for a specific client - -**Server control:** - -- `pause-server` / `resume-server` — SIGSTOP/SIGCONT the server process -- `resume-server-until-history-then-pause` — resume the server, wait until a specific client observes a matching history entry (`CREATE`/`UPDATE`/`DELETE` for a path), then re-pause. Used to land exactly one operation across the wire. - -**Fault injection** (per-client): - -- `drop-next-create-response` — arm a one-shot interceptor that lets the next `POST /documents` reach the server (commit happens) but throws `SyncResetError` before the client sees the response, simulating connection loss after server commit. -- `wait-for-dropped-create-response` — wait until the armed drop has fired. - -**Assertions:** - -- `assert-consistent` — all clients have identical files; optionally takes a custom `verify(state: AssertableState)` callback - -## Running +Run the complete harness check from **any working directory**: ```sh -# Build server first -cd sync-server && cargo build --release && cd - - -# Run all tests -cd frontend && npm run build -w sync-client && npm run test -w deterministic-tests - -# Filter by name -npm run test -w deterministic-tests -- --filter=rename - -# Control parallelism (default: number of CPU cores) -npm run test -w deterministic-tests -- -j 4 +E2E_WORKERS=2 /path/to/vault-link/scripts/e2e.sh ``` -## Adding a test +The `E2E_WORKERS` variable is the number of fuzz workers / simultaneous scripted tests. +The runner builds the server and the three in-scope frontend packages, then runs +the harness self-tests, recovery regressions, protocol/real-filesystem tests, +scripted scenarios, and seeded fuzz workers. It does not build the Obsidian plugin +or CLI application. Install frontend dependencies with `npm ci` first. Node's +built-in WebSocket, Rust, and a C compiler (`cc`, for test-only exclusive rename +and OS-lock primitives on macOS/Linux) are required. -1. Create `src/tests/my-scenario.test.ts`: +CI installs those dependencies in `frontend` before invoking the runner. It sets +`E2E_ARTIFACTS=logs/e2e` and uploads that same directory even when a suite fails. +The runner owns all test servers; no separate server or legacy cleanup script is +needed in the workflow. -```typescript -import type { TestDefinition } from "../test-definition"; +Each server gets a fresh temporary database, working directory, and port. Only +owned processes are terminated. There is no `pkill`, fixed database deletion, +or shared server PID file. Failing tests and teardown errors give nonzero exits; +the remaining suites still run. Logs, concrete fuzz traces, results, and failure +snapshots are retained in the printed artifact directory. Snapshots are captured +before teardown so cleanup failures retain the original evidence. -export const myScenarioTest: TestDefinition = { - description: - "Client 0 creates A.md offline. After syncing, both clients should have the file.", - clients: 2, - steps: [ - { type: "create", client: 0, path: "A.md", content: "hello" }, - { type: "enable-sync", client: 0 }, - { type: "enable-sync", client: 1 }, - { type: "barrier" }, - { - type: "assert-consistent", - verify: (s) => { - s.assertFileCount(1).assertContent("A.md", "hello"); - } - } - ] -}; +Environment options: + +- `E2E_WORKERS=1`: number of fuzz workers / simultaneous scripted tests (max 32). +- `E2E_SEED=1`: first uint32 seed; workers use successive seeds. +- `E2E_ITERATIONS=25`: generated conflict scenarios per worker. +- `E2E_TIMEOUT_SECONDS=900`: timeout for each build/suite/worker command. +- `E2E_ARTIFACTS=/absolute/path`: optional output directory; defaults to `mkdtemp`. + +## Running individual layers + +From `frontend`: + +```sh +# Rebuild protocol peers and run scripted tests (name filter optional). +npm run test --workspace deterministic-tests -- --filter v4- --concurrency 2 + +# Adapter/oracle/lifecycle self-tests and strict recovery regressions. +npm run test:harness + +# After rebuilding: real worker death, server SIGKILL/restart, CAS/WS ordering, +# and host filesystem contract checks. +npm run test:protocol + +npm run build --workspace sync-client --workspace test-client +node test-client/dist/cli.js --seed 42 --iterations 25 +node test-client/dist/cli.js --replay /path/to/trace-42.json ``` -The `verify` callback receives an `AssertableState` object with chainable assertion methods: +`test:harness` includes strict regressions for **engine** bugs. +A failing default run is not an expected-failure test or a harness skip. See +[`../test-support/README.md`](../test-support/README.md) for their scope. -```typescript -s.assertFileCount(n); // exact file count -s.assertFileExists("path"); // file must exist -s.assertFileNotExists("path"); // file must not exist -s.assertContent("path", "expected"); // exact content match -s.assertContains("path", "a", "b"); // all substrings present in file -s.assertContainsAny("path", "a", "b"); // at least one substring present -s.assertAnyFileContains("text"); // substring present in some file -s.assertNoFileContains("text"); // substring absent from every file -s.assertSubstringCount("path", "x", 3); // substring appears exactly N times -s.assertContentInAtMostOneFile("text"); // no duplicate content -s.ifFileExists("path", (s) => { /* … */ }); // conditional block -s.getContent("path"); // raw content (or "" if missing) -``` +## What the scripted runner checks -2. Register it in `src/test-registry.ts`: +Real `SyncClient` instances use the shared v4 adapter in `../test-support`. +External editor operations and engine filesystem operations are separate: engine +writes never generate self-notifications. Editor identity notifications continue +while transport is disabled. Delayed notifications can be explicitly queued and +flushed. Snapshots and persisted state are deep copies, never shared references. -```typescript -import { myScenarioTest } from "./tests/my-scenario.test"; +Scripted and seeded clients explicitly set `syncIntervalMs: 0`, with assertions +that it stays disabled. Barriers observe convergence; they do not wake the engine. +Network retry timers remain enabled. Separate fake-clock tests in +`test-support/polling.test.ts` exercise optional periodic reconciliation. -const TESTS = { - // ... - "my-scenario": myScenarioTest -}; -``` +A barrier checks byte-for-byte replica equality **and** the server's canonical +UUID/path map, file bytes, independently computed base/observed hashes, content +and manifest CAS parents, event watermark, +contiguous event log, and absence of pending requests/application journals. +Internal recovery files are excluded from the user-visible file list, but are +present in failure disk images. Scenario assertions separately check intended +content, document identity, or preservation markers; equality alone is not an +adequate data-loss oracle. + +Clients start disabled. `enable-sync` waits for the initial sync attempt/retries, +except while the server is intentionally paused: that attempt is tracked so a +later script step can resume the server. Thus sequential enables really establish +a winner. To test _independent creates_, first bootstrap both clients empty, then +disable them and create files. Initial adoption of pre-existing files during +bootstrap is a different policy from two already-initialized clients creating +distinct UUIDs at one path. + +## Controls and assertions + +- `create`, `update`, `create-bytes`, `rename`, `delete`: external editor actions. + Editor rename may explicitly replace its target; engine rename never may. +- `sync`, `barrier`, `enable-sync`, `disable-sync`. +- `reset`: restart transports; **not** a crash and not a database reset. +- `pause-websocket`, `resume-websocket`: buffer only WS delivery. HTTP catchup + remains active; this is deliberately not called "disconnect". +- `pause-observation`, `wait-for-observation`, `resume-observation`: explicitly + gate HTTP catchup as well as WS, including replies already in flight. The + checkpoint must be exercised and released, and held reads respect cancellation. +- `delay-notifications`, `flush-notifications`: delay editor notifications without + disabling preservation assertions. +- `drop-response`: `{client, kind: "create" | "content" | "manifest", point: +"before" | "after"}`. After faults fire only on an `Accepted` CAS response. +- `wait-for-response-drop`: explicit checkpoint; every armed fault must fire. + Every interrupted request must be retried with an identical request ID, target + and body before a different CAS is sent. Fault kinds match exactly. +- `rename-next-write`: inject an editor rename after the engine's durable install + rename. The historical name is retained for fixtures; v4 never writes in place. +- `pause-server`, `resume-server`: SIGSTOP/SIGCONT, not a crash. +- `crash-server`, `restart-server`: SIGKILL and restart with the same database/port. + Server-control scenarios have dedicated processes. +- `remember-identity`, `assert-identity`, `assert-files`, `assert-markers`. +- `assert-documents`: a complete expected set of `{key, path, content, conflict?}` + records. Content is text or a byte array. Keys bind UUIDs on the first successful + check and must retain them on later checks; new keys cannot reuse deleted UUIDs. + `conflict: true` resolves the UUID-derived conflict name for `path`. +- `assert-consistent` can provide `verify(state)`: exact text/bytes, UUID lookups, + `assertIdentity`, and `conflictPath(original)` for UUID-derived conflict names. + +Legacy timing scenarios are retained. Prefer explicit fault checkpoints and +barriers for new tests. Only known coherent-scan races and actually injected +transient network errors may retry; they still must converge before the deadline. +Other background errors, unhandled rejections, and cleanup failures fail the run. + +## Reproducibility limits + +Seeds and JSON traces reproduce harness choices and logical actions. They do not +control the OS scheduler, server thread scheduling, or production-generated UUIDs. +The recovery sweep enumerates adapter/save boundaries; worker termination proves +abrupt runtime loss. Neither is a proof covering every real filesystem or hardware +power-loss behavior. Run multiple seeds, repeat timing-sensitive cases, and keep +the scenario's independent oracle when reducing a failing trace. diff --git a/frontend/deterministic-tests/package.json b/frontend/deterministic-tests/package.json index 4bd82c74..4e7ee563 100644 --- a/frontend/deterministic-tests/package.json +++ b/frontend/deterministic-tests/package.json @@ -8,7 +8,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "npm run build && node dist/cli.js" + "test": "cargo build --release --manifest-path ../../sync-server/Cargo.toml && npm run build --workspace sync-client && npm run build && node dist/cli.js" }, "devDependencies": { "commander": "^14.0.2", diff --git a/frontend/deterministic-tests/src/cli.ts b/frontend/deterministic-tests/src/cli.ts index 1f5f4706..2c1f495c 100644 --- a/frontend/deterministic-tests/src/cli.ts +++ b/frontend/deterministic-tests/src/cli.ts @@ -9,10 +9,11 @@ import { runWithConcurrency } from "./run-with-concurrency"; import { TOKEN, SERVER_BINARY_PATH, CONFIG_PATH } from "./consts"; import * as path from "node:path"; import * as fs from "node:fs"; -import { debugging, Logger } from "sync-client"; +import * as os from "node:os"; +import { Logger } from "sync-client"; const logger = new Logger(); -debugging.logToConsole(logger, { useColors: true }); +logger.onLogEmitted.add((line) => console.log(line.message)); process.on("unhandledRejection", (reason) => { logger.error(`Unhandled Rejection: ${reason}`); @@ -32,6 +33,8 @@ function testUsesPauseServer(test: TestDefinition): boolean { (step) => step.type === "pause-server" || step.type === "resume-server" || + step.type === "crash-server" || + step.type === "restart-server" || step.type === "resume-server-until-history-then-pause" ); } @@ -42,7 +45,7 @@ function testUsesPauseServer(test: TestDefinition): boolean { */ function findProjectRoot(): string { let dir = path.dirname(__filename); - const {root} = path.parse(dir); + const { root } = path.parse(dir); while (dir !== root) { if ( fs.existsSync(path.join(dir, "sync-server")) && @@ -114,11 +117,7 @@ async function runDedicatedServerTest( } return { name, result }; } finally { - try { - await server.stop(); - } catch { - // best-effort cleanup - } + await server.stop(); serverManager.untrack(server); } } @@ -138,6 +137,11 @@ async function main(): Promise { } const { filter, concurrency } = parseArgs(process.argv); + const artifacts = + process.env.E2E_ARTIFACTS ?? + fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-deterministic-")); + fs.mkdirSync(artifacts, { recursive: true }); + logger.info(`Artifacts: ${artifacts}`); const testsToRun: [string, TestDefinition][] = []; for (const [key, test] of Object.entries(TESTS)) { @@ -193,13 +197,7 @@ async function main(): Promise { allResults.push(...results); } finally { - try { - await sharedServer.stop(); - } catch (error) { - logger.warn( - `Error stopping shared server: ${error instanceof Error ? error.message : String(error)}` - ); - } + await sharedServer.stop(); serverManager.untrack(sharedServer); } } @@ -221,6 +219,10 @@ async function main(): Promise { const passed = allResults.filter((r) => r.result.success); const failed = allResults.filter((r) => !r.result.success); + fs.writeFileSync( + path.join(artifacts, "deterministic-results.json"), + JSON.stringify(allResults, null, 2) + ); logger.info( `\n--- Results: ${passed.length}/${allResults.length} passed ---` diff --git a/frontend/deterministic-tests/src/crash-worker.ts b/frontend/deterministic-tests/src/crash-worker.ts new file mode 100644 index 00000000..c015fac6 --- /dev/null +++ b/frontend/deterministic-tests/src/crash-worker.ts @@ -0,0 +1,93 @@ +import { parentPort, workerData } from "node:worker_threads"; +import assert from "node:assert/strict"; +import { SyncClient } from "sync-client"; +import { + MemoryDisk, + MemoryPersistence, + type DiskImage, + type StoredClient +} from "../../test-support/storage"; +import { + classifyRequest, + type RequestRecord, + type RequestKind +} from "../../test-support/network"; + +export interface CrashWorkerInput { + disk: DiskImage; + stored: StoredClient; + stopAt?: RequestKind; + editAfterSync?: { path: string; content: string }; +} +const input = workerData as CrashWorkerInput; +const disk = new MemoryDisk(); +disk.restore(input.disk); +const persistence = new MemoryPersistence(input.stored); +const requests: RequestRecord[] = []; +let editedBase: { documentId: string; vaultUpdateId: number } | undefined; +const snapshot = () => ({ + disk: disk.image(), + stored: persistence.snapshot(), + requests, + editedBase +}); + +async function main() { + const client = await SyncClient.create({ + fs: disk.session(), + persistence, + fetch: async (url, init) => { + const record = classifyRequest(url, init); + if (record) requests.push(record); + const response = await fetch(url, init); + if (record && response.ok && input.stopAt === record.kind) { + const receipt = (await response.clone().json()) as { + type: string; + }; + if (receipt.type === "Accepted") { + parentPort!.postMessage({ + type: "checkpoint", + ...snapshot() + }); + // Parent terminates this worker. No destroy(), no state flush, + // no finally block, and no old runtime survives the restart. + await new Promise(() => {}); + } + } + return response; + } + }); + assert.equal( + client.getSettings().syncIntervalMs, + 0, + "Crash recovery must preserve the disabled polling setting" + ); + await client.start(); + await client.waitUntilFinished(); + if (input.editAfterSync) { + const state = persistence.snapshot().database!; + const documentId = Object.keys(state.local!).find( + (id) => state.local![id] === input.editAfterSync!.path + ); + assert(documentId, "Edit must target an established document"); + editedBase = { + documentId, + vaultUpdateId: state.documents![documentId].base!.vaultUpdateId + }; + await disk.userWrite( + input.editAfterSync.path, + new TextEncoder().encode(input.editAfterSync.content) + ); + await client.syncLocallyUpdatedFile({ + relativePath: input.editAfterSync.path + }); + await client.waitUntilFinished(); + } + await client.destroy(); + parentPort!.postMessage({ type: "finished", ...snapshot() }); + parentPort!.close(); +} +main().catch((error: unknown) => { + parentPort!.postMessage({ type: "error", error: String(error) }); + process.exit(1); +}); diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts index fb7fe582..36a029e5 100644 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -1,483 +1,340 @@ -import type { - HistoryEntry, - StoredDatabase, - SyncSettings, - RelativePath, - TextWithCursors -} from "sync-client"; import { SyncClient, - SyncResetError, - debugging, LogLevel, - utils + type HistoryEntry, + type SyncSettings } from "sync-client"; +import { MemoryDisk, MemoryPersistence } from "../../test-support/storage"; +import { NetworkFaults, type RequestKind } from "../../test-support/network"; import { assert } from "./utils/assert"; import { sleep } from "./utils/sleep"; import { withTimeout } from "./utils/with-timeout"; -import { - IS_SYNC_ENABLED_BY_DEFAULT, - WAIT_TIMEOUT_MS, - WEBSOCKET_CONNECT_TIMEOUT_MS, - WEBSOCKET_POLL_INTERVAL_MS -} from "./consts"; +import { WAIT_TIMEOUT_MS, WEBSOCKET_CONNECT_TIMEOUT_MS } from "./consts"; import { ManagedWebSocketFactory } from "./managed-websocket"; -export class DeterministicAgent extends debugging.InMemoryFileSystem { - public readonly clientId: number; - private readonly logger: (msg: string) => void; - private client!: SyncClient; - private data: Partial<{ - settings: Partial; - database: Partial; - }> = {}; - private isSyncEnabled = IS_SYNC_ENABLED_BY_DEFAULT; - private readonly syncErrors: Error[] = []; - private readonly pendingSyncOperations = new Set>(); +/** User actions live here; the engine only receives disk.session(). */ +export class DeterministicAgent { + public readonly disk = new MemoryDisk(); + public readonly persistence: MemoryPersistence; + public readonly network = new NetworkFaults(); + private client?: SyncClient; + private readonly errors: Error[] = []; + private readonly pending = new Set>(); private readonly wsFactory = new ManagedWebSocketFactory(); - private nextWriteRename: - | { - oldPath: RelativePath; - newPath: RelativePath; - } - | undefined; - private nextCreateResponseDrop: - | { - dropped: Promise; - resolveDropped: () => void; - } - | undefined; + private nextWriteRename?: { oldPath: string; newPath: string }; + private delayedNotifications = false; + private disposed = false; + private readonly notifications: (() => Promise)[] = []; public constructor( - clientId: number, - initialSettings: Partial, - logger: (msg: string) => void + public readonly clientId: number, + settings: Partial, + private readonly logger: (msg: string) => void ) { - super(); - this.clientId = clientId; - this.logger = logger; - this.data.settings = { ...initialSettings }; + this.persistence = new MemoryPersistence({ + settings: { + networkRetryIntervalMs: 25, + webSocketRetryIntervalMs: 50, + requestTimeoutMs: 5_000, + ...settings, + // Scripted and seeded scenarios must exercise notification delivery. + syncIntervalMs: 0 + } + }); + this.disk.boundary = async (label) => { + const rename = this.nextWriteRename; + // v4 installs through an exclusive rename, never write-in-place. + if ( + rename && + label.startsWith("durable:rename:") && + label.endsWith(`->${rename.oldPath}`) + ) { + this.nextWriteRename = undefined; + await this.rename(rename.oldPath, rename.newPath); + } + }; } public async init( - fetchImplementation: typeof globalThis.fetch + fetchImplementation: typeof globalThis.fetch = fetch ): Promise { this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: this.wrapFetch(fetchImplementation), + fs: this.disk.session(), + persistence: this.persistence, + fetch: this.network.wrap(fetchImplementation), webSocket: this.wsFactory.constructorFn }); - - this.client.logger.onLogEmitted.add((line) => { - const prefix = `[Client ${this.clientId}]`; - switch (line.level) { - case LogLevel.ERROR: - this.logger(`${prefix} ERROR: ${line.message}`); - break; - case LogLevel.WARNING: - this.logger(`${prefix} WARN: ${line.message}`); - break; - case LogLevel.INFO: - this.logger(`${prefix} INFO: ${line.message}`); - break; - case LogLevel.DEBUG: - this.logger(`${prefix} DEBUG: ${line.message}`); - break; - } - }); - - await this.client.start(); - - const connectionCheck = await this.client.checkConnection(); assert( - connectionCheck.isSuccessful, + this.client.getSettings().syncIntervalMs === 0, + "Correctness scenarios must run with periodic polling disabled" + ); + this.client.logger.onLogEmitted.add((line) => { + this.logger( + `[Client ${this.clientId}] ${line.level}: ${line.message}` + ); + if (this.disposed && line.level === LogLevel.ERROR) + throw new Error( + `Client ${this.clientId} error after cleanup: ${line.message}` + ); + if ( + line.level === LogLevel.ERROR && + !this.isRetryableFailure(line.message) + ) + this.errors.push(new Error(line.message)); + }); + await this.client.start(); + assert( + (await this.client.checkConnection()).isSuccessful, `Client ${this.clientId} connection check failed` ); + } - if (this.isSyncEnabled) { - await this.waitForWebSocket(); + public database() { + return this.persistence.snapshot().database; + } + public files(): Map { + return this.disk.userFiles(); + } + public async listFilesRecursively(): Promise { + return [...this.files().keys()].sort(); + } + public async getFileContent(path: string): Promise { + const bytes = this.files().get(path); + assert(bytes !== undefined, `Missing file: ${path}`); + return new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes); + } + public async read(path: string): Promise { + const snapshot = await this.disk.readSnapshot(path); + assert(snapshot !== undefined, `Missing file: ${path}`); + return snapshot.content; + } + public async write(path: string, content: Uint8Array): Promise { + const created = !(await this.disk.exists(path)); + await this.disk.userWrite(path, content); + this.notify(() => + created + ? this.client!.syncLocallyCreatedFile(path) + : this.client!.syncLocallyUpdatedFile({ relativePath: path }) + ); + } + public async rename(oldPath: string, relativePath: string): Promise { + await this.disk.userRename(oldPath, relativePath); + this.notify(() => + this.client!.syncLocallyUpdatedFile({ oldPath, relativePath }) + ); + } + public async delete(path: string): Promise { + await this.disk.userDelete(path); + this.notify(() => this.client!.syncLocallyDeletedFile(path)); + } + public delayNotifications(): void { + this.delayedNotifications = true; + } + public flushNotifications(): void { + this.delayedNotifications = false; + for (const notify of this.notifications.splice(0)) this.enqueue(notify); + } + private notify(operation: () => Promise): void { + // Offline means transport disabled, not that editor identity events vanish. + if (this.delayedNotifications) this.notifications.push(operation); + else this.enqueue(operation); + } + private enqueue(operation: () => Promise): void { + const promise = operation().catch((error: unknown) => { + if (!this.isRetryableFailure(String(error))) + this.errors.push( + error instanceof Error ? error : new Error(String(error)) + ); + }); + this.pending.add(promise); + void promise.then(() => this.pending.delete(promise)); + } + private async drain(): Promise { + while (this.pending.size) await Promise.all([...this.pending]); + } + private isRetryableFailure(message: string): boolean { + // A coherent scan rejecting an editor race is a required adapter + // behavior. It must subsequently converge; never blanket-ignore errors. + return ( + this.network.isExpectedFailure(message) || + /^(Sync paused for retry: )?Error: File changed during scan: .+$/.test( + message + ) + ); + } + private assertHealthy(): void { + if (this.errors.length) { + const errors = this.errors.splice(0); + throw new AggregateError( + errors, + `Client ${this.clientId}: ${errors.map(String).join("; ")}` + ); } } - - public pauseWebSocket(): void { - this.log("Pausing WebSocket message delivery"); - this.wsFactory.pause(); - } - - public resumeWebSocket(): void { - this.log("Resuming WebSocket message delivery"); - this.wsFactory.resume(); - } - - public dropNextCreateResponse(): void { + public async waitForSync(): Promise { assert( - this.nextCreateResponseDrop === undefined, - `Client ${this.clientId} already has a create response drop armed` - ); - let resolveDropped!: () => void; - const dropped = new Promise((resolve) => { - resolveDropped = resolve; - }); - this.nextCreateResponseDrop = { - dropped, - resolveDropped - }; - this.log("Armed next create response drop"); - } - - public async waitForDroppedCreateResponse(): Promise { - assert( - this.nextCreateResponseDrop !== undefined, - `Client ${this.clientId} has no create response drop armed` + this.client!.getSettings().syncIntervalMs === 0, + "Periodic polling was enabled during a correctness scenario" ); await withTimeout( - this.nextCreateResponseDrop.dropped, + (async () => { + await this.drain(); + const deadline = Date.now() + WAIT_TIMEOUT_MS; + for (;;) { + try { + await this.client!.waitUntilFinished(); + break; + } catch (error) { + if ( + !this.isRetryableFailure(String(error)) || + Date.now() >= deadline + ) + throw error; + await sleep(30); + } + } + this.assertHealthy(); + })(), WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for create response drop` + `Client ${this.clientId} sync timeout` ); - this.log("Create response was dropped after server commit"); } - + public async disableSync(): Promise { + await this.drain(); + await this.client!.setSetting("isSyncEnabled", false); + this.assertHealthy(); + } + public async enableSync(wait = true): Promise { + // setSetting waits for a sync attempt in v4. Track it, but don't block + // the script from resuming a paused server or releasing a checkpoint. + this.enqueue(() => this.client!.setSetting("isSyncEnabled", true)); + const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; + while ( + !this.persistence.snapshot().settings?.isSyncEnabled && + Date.now() < deadline + ) + await sleep(1); + assert( + !!this.persistence.snapshot().settings?.isSyncEnabled, + "Enable did not persist" + ); + if (wait) await this.waitForSync(); + } + public async reset(): Promise { + await this.drain(); + await this.client!.reset(); + } + public pauseWebSocket(): void { + this.wsFactory.pause(); + } + public pauseObservation(): void { + this.pauseWebSocket(); + this.network.pauseObservation(); + } + public resumeObservation(): void { + this.network.resumeObservation(); + this.resumeWebSocket(); + } + public async waitForObservation(): Promise { + await withTimeout( + this.network.waitForObservation(), + WAIT_TIMEOUT_MS, + "HTTP observation checkpoint never reached" + ); + } + public resumeWebSocket(): void { + this.wsFactory.resume(); + } + public dropNextCreateResponse(): void { + this.network.arm("create"); + } + public dropNextResponse( + kind: RequestKind, + point: "before" | "after" = "after" + ): void { + this.network.arm(kind, point); + } + public async waitForDroppedCreateResponse(): Promise { + await withTimeout( + this.network.wait(), + WAIT_TIMEOUT_MS, + "Armed response drop never fired" + ); + } + public renameNextWrite(oldPath: string, newPath: string): void { + assert( + !this.nextWriteRename, + "Previous install/rename hook never fired" + ); + this.nextWriteRename = { oldPath, newPath }; + } public async waitForHistoryEntry( matches: (entry: HistoryEntry) => boolean, onMatch?: (entry: HistoryEntry) => void ): Promise { - const existing = this.client.getHistoryEntries().find(matches); - if (existing !== undefined) { - onMatch?.(existing); - return; - } - - await withTimeout( - new Promise((resolve) => { - const unsubscribe = this.client.onSyncHistoryUpdated.add(() => { - const entry = this.client - .getHistoryEntries() - .find(matches); - if (entry === undefined) { - return; - } - - unsubscribe(); - onMatch?.(entry); - resolve(); - }); - }), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} timed out waiting for history entry` - ); - } - - public async waitForSync(): Promise { - this.log("Waiting for sync to complete..."); - // Drain agent-level sync operations first. These are the fire-and-forget - // promises from enqueueSync() that call into the SyncClient's methods. - // Without this, waitUntilFinished() might return before the SyncClient - // has even been told about the operation. - await this.drainPendingSyncOperations(); - await withTimeout( - this.client.waitUntilFinished(), - WAIT_TIMEOUT_MS, - `Client ${this.clientId} waitForSync timed out after ${WAIT_TIMEOUT_MS}ms` - ); - if (this.syncErrors.length > 0) { - const errors = this.syncErrors.splice(0); - throw new Error( - `Client ${this.clientId} had ${errors.length} sync error(s):\n${errors.map((e) => e.message).join("\n")}` - ); - } - this.log("Sync complete"); - } - - public async reset(): Promise { - this.log("Resetting client (clears tracked state, keeps disk files)"); - await this.drainPendingSyncOperations(); - await this.client.reset(); - if (this.isSyncEnabled) { - await this.waitForWebSocket(); - } - } - - public async disableSync(): Promise { - this.log("Disabling sync"); - // Drain pending enqueued operations before disabling so the SyncClient - // knows about all operations that were enqueued while sync was enabled. - await this.drainPendingSyncOperations(); - await this.client.setSetting("isSyncEnabled", false); - this.isSyncEnabled = false; - // Wait for in-flight operations to drain. Disabling sync triggers - // a reset, which aborts in-flight fetches with SyncResetError. + let unsubscribe = () => {}; try { await withTimeout( - this.client.waitUntilFinished(), + new Promise((resolve) => { + const check = () => { + const entry = + this.client!.getHistoryEntries().find(matches); + if (entry) { + onMatch?.(entry); + resolve(); + } + }; + unsubscribe = this.client!.onSyncHistoryUpdated.add(check); + check(); + }), WAIT_TIMEOUT_MS, - `Client ${this.clientId} disableSync drain timed out` + "History checkpoint not reached" ); - } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log("Disable sync drain interrupted by reset (expected)"); - } else { - throw error; - } + } finally { + unsubscribe(); } } - - public async enableSync(): Promise { - this.log("Enabling sync"); - await this.client.setSetting("isSyncEnabled", true); - this.isSyncEnabled = true; - await this.waitForWebSocket(); - } - - public async getFileContent(path: string): Promise { - const bytes = await this.read(path); - return new TextDecoder().decode(bytes); - } - - public renameNextWrite(oldPath: RelativePath, newPath: RelativePath): void { - assert( - this.nextWriteRename === undefined, - `Client ${this.clientId} already has a next-write rename armed` - ); - this.nextWriteRename = { oldPath, newPath }; - this.log(`Armed next write rename: ${oldPath} -> ${newPath}`); - } - public async cleanup(): Promise { - this.log("Cleaning up..."); - // Guard against uninitialized client (init() failed partway). - // The class field uses `!:` so TS thinks this is always defined, - // but at runtime it can be undefined when init() throws partway. - const maybeClient = this.client as SyncClient | undefined; - if (maybeClient === undefined) { - this.log("Client not initialized, nothing to clean up"); - return; + if (!this.client) return; + const errors: unknown[] = []; + this.resumeObservation(); + this.flushNotifications(); + try { + await this.waitForSync(); + } catch (error) { + errors.push(error); + } + try { + this.network.assertConsumed(); + assert(!this.nextWriteRename, "Install/rename hook never fired"); + } catch (error) { + errors.push(error); } try { - await this.drainPendingSyncOperations(); await withTimeout( - this.client.waitUntilFinished(), + this.client.destroy(), WAIT_TIMEOUT_MS, - `Client ${this.clientId} cleanup waitUntilFinished timed out` + "Destroy timed out" + ); + await withTimeout( + this.wsFactory.finish(), + WAIT_TIMEOUT_MS, + "WebSocket callbacks did not finish during cleanup" ); } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Cleanup interrupted by reset (expected): ${error}`); - } else { - this.log(`Cleanup waitUntilFinished failed: ${error}`); - } + errors.push(error); } - // Surface any background sync errors that arrived after the last - // waitForSync (e.g. between the final assert-consistent and here). - // Without this, regressions that fault the engine during the very - // last step of a test would be silently swallowed. - const pendingErrors = this.syncErrors.splice(0); - await this.client.destroy(); - this.log("Cleanup complete"); - if (pendingErrors.length > 0) { - throw new Error( - `Client ${this.clientId} had ${pendingErrors.length} background sync error(s) during cleanup:\n${pendingErrors.map((e) => e.message).join("\n")}` - ); - } - } - - public override async read(path: RelativePath): Promise { - await Promise.resolve(); - return super.read(path); - } - - public override async write( - path: RelativePath, - content: Uint8Array - ): Promise { - await Promise.resolve(); - const isNew = !this.files.has(path); - await super.write(path, content); - - if (this.isSyncEnabled && isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyCreatedFile(path); - }); - } - - const { nextWriteRename } = this; - if ( - nextWriteRename !== undefined && - nextWriteRename.oldPath === path - ) { - this.nextWriteRename = undefined; - await super.rename( - nextWriteRename.oldPath, - nextWriteRename.newPath - ); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath: nextWriteRename.oldPath, - relativePath: nextWriteRename.newPath - }); - }); - } - // The rename consumed `path`. Skip the post-update enqueue below - // — it would send a syncLocallyUpdatedFile for a path that no - // longer exists. - return; - } - - if (!this.isSyncEnabled) { - return; - } - - if (!isNew) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - } - - public override async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const result = await super.atomicUpdateText(path, updater); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ relativePath: path }); - }); - } - return result; - } - - public override async delete(path: RelativePath): Promise { - await super.delete(path); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyDeletedFile(path); - }); - } - } - - public override async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - await super.rename(oldPath, newPath); - if (this.isSyncEnabled) { - this.enqueueSync(async () => { - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - }); - } - } - - private async waitForWebSocket(): Promise { - const deadline = Date.now() + WEBSOCKET_CONNECT_TIMEOUT_MS; - while (!this.client.isWebSocketConnected && Date.now() < deadline) { - await sleep(WEBSOCKET_POLL_INTERVAL_MS); - } - assert( - this.client.isWebSocketConnected, - `Client ${this.clientId} WebSocket failed to connect within ${WEBSOCKET_CONNECT_TIMEOUT_MS}ms` - ); - } - - /** - * Wait until all agent-level enqueued sync operations have completed. - * Uses a loop because completing one operation can trigger new enqueues. - */ - private async drainPendingSyncOperations(): Promise { - while (this.pendingSyncOperations.size > 0) { - await utils.awaitAll([...this.pendingSyncOperations]); - } - } - - private enqueueSync(operation: () => Promise): void { - const promise = this.executeSyncOperation(operation).catch( - (error: unknown) => { - const err = - error instanceof Error ? error : new Error(String(error)); - this.log(`Background sync failed: ${err.message}`); - this.syncErrors.push(err); - } - ); - this.pendingSyncOperations.add(promise); - void promise.finally(() => { - this.pendingSyncOperations.delete(promise); - }); - } - - private async executeSyncOperation( - operation: () => Promise - ): Promise { + this.client = undefined; + this.disposed = true; try { - await operation(); + this.assertHealthy(); } catch (error) { - if (error instanceof Error && error.name === "SyncResetError") { - this.log(`Sync operation interrupted by reset: ${error}`); - return; - } - if ( - error instanceof Error && - error.message.includes("has been destroyed") - ) { - this.log(`Sync operation interrupted by destroy: ${error}`); - return; - } - - throw error; + errors.push(error); } - } - - private log(message: string): void { - this.logger(`[Client ${this.clientId}] ${message}`); - } - - private wrapFetch( - fetchImplementation: typeof globalThis.fetch - ): typeof globalThis.fetch { - return async (input, init) => { - const response = await fetchImplementation(input, init); - const drop = this.nextCreateResponseDrop; - if ( - drop !== undefined && - DeterministicAgent.isCreateDocumentRequest(input, init) - ) { - this.nextCreateResponseDrop = undefined; - try { - await response.body?.cancel(); - } catch { - // Best-effort — body may already be consumed/closed. - } - drop.resolveDropped(); - throw new SyncResetError(); - } - return response; - }; - } - - private static isCreateDocumentRequest( - input: RequestInfo | URL, - init: RequestInit | undefined - ): boolean { - const method = - init?.method ?? - (typeof Request !== "undefined" && input instanceof Request - ? input.method - : "GET"); - if (method.toUpperCase() !== "POST") { - return false; - } - - const url = - input instanceof URL - ? input - : new URL(typeof input === "string" ? input : input.url); - return /\/documents\/?$/.test(url.pathname); + if (errors.length) + throw new AggregateError( + errors, + `Client ${this.clientId} cleanup failed: ${errors.map(String).join("; ")}` + ); } } diff --git a/frontend/deterministic-tests/src/managed-websocket.ts b/frontend/deterministic-tests/src/managed-websocket.ts index c759891b..7d1159b5 100644 --- a/frontend/deterministic-tests/src/managed-websocket.ts +++ b/frontend/deterministic-tests/src/managed-websocket.ts @@ -209,8 +209,13 @@ export class ManagedWebSocketFactory { // WS before the agent reconnects would silently see the new socket // start un-paused and miss the messages it meant to buffer. private currentlyPaused = false; + private disposed = false; public get constructorFn(): typeof globalThis.WebSocket { + const ensureActive = () => { + if (this.disposed) + throw new Error("WebSocket created after harness cleanup"); + }; const trackInstance = (instance: ManagedWebSocket): void => { this.instances.push(instance); if (this.currentlyPaused) { @@ -222,6 +227,7 @@ export class ManagedWebSocketFactory { url: string | URL, protocols?: string | string[] ) { + ensureActive(); super(url, protocols); trackInstance(this); } @@ -229,6 +235,21 @@ export class ManagedWebSocketFactory { return TrackedManagedWebSocket; } + public async finish(): Promise { + this.disposed = true; + await Promise.all( + this.instances.map((socket) => + socket.readyState === WebSocket.CLOSED + ? Promise.resolve() + : new Promise((resolve) => + socket.addEventListener("close", () => resolve(), { + once: true + }) + ) + ) + ); + } + public pause(): void { this.currentlyPaused = true; for (const ws of this.instances) { diff --git a/frontend/deterministic-tests/src/parse-args.ts b/frontend/deterministic-tests/src/parse-args.ts index 11c56f19..a6968d13 100644 --- a/frontend/deterministic-tests/src/parse-args.ts +++ b/frontend/deterministic-tests/src/parse-args.ts @@ -7,8 +7,8 @@ export interface CliArgs { } function parsePositiveInt(value: string): number { - const n = parseInt(value, 10); - if (isNaN(n) || n <= 0) { + const n = Number(value); + if (!/^\d+$/.test(value) || !Number.isSafeInteger(n) || n <= 0) { throw new InvalidArgumentError("must be a positive integer"); } return n; diff --git a/frontend/deterministic-tests/src/server-control.ts b/frontend/deterministic-tests/src/server-control.ts index d3c5387d..03c3f418 100644 --- a/frontend/deterministic-tests/src/server-control.ts +++ b/frontend/deterministic-tests/src/server-control.ts @@ -20,10 +20,13 @@ export class ServerControl { private _port: number | undefined; private tempDir: string | undefined; private _isPaused = false; + private preserveOnStop = false; + private expectedExit = false; + private unexpectedExit?: Error; public constructor(serverPath: string, configPath: string, logger: Logger) { - this.serverPath = serverPath; - this.baseConfigPath = configPath; + this.serverPath = path.resolve(serverPath); + this.baseConfigPath = path.resolve(configPath); this.logger = logger; } @@ -38,6 +41,15 @@ export class ServerControl { return `http://localhost:${this.port}`; } + public get databaseDirectory(): string { + if (!this.tempDir) throw new Error("Server has no database directory"); + return path.join(this.tempDir, "databases"); + } + + public get isPaused(): boolean { + return this._isPaused; + } + public async start(): Promise { if (this.process !== null) { throw new Error("Server is already running"); @@ -66,14 +78,25 @@ export class ServerControl { } private async startOnce(): Promise { - const reservation = await findFreePort(); - this._port = reservation.port; + const reservation = + this._port === undefined ? await findFreePort() : undefined; + this._port ??= reservation!.port; const tmpBase = os.tmpdir(); - this.tempDir = fs.mkdtempSync(path.join(tmpBase, "vault-link-test-")); - const tempConfigPath = path.join(this.tempDir, "config.yml"); - const dbDir = path.join(this.tempDir, "databases"); - - this.writeConfigFile(tempConfigPath, dbDir); + let tempConfigPath: string; + try { + this.tempDir ??= fs.mkdtempSync( + path.join(tmpBase, "vault-link-test-") + ); + tempConfigPath = path.join(this.tempDir, "config.yml"); + this.writeConfigFile( + tempConfigPath, + path.join(this.tempDir, "databases") + ); + } catch (error) { + reservation?.release(); + this.cleanupTempDir(); + throw error; + } this.logger.info( `Starting server: ${this.serverPath} (port ${this._port})` @@ -81,12 +104,15 @@ export class ServerControl { // Release the port reservation right before spawning to minimize // the TOCTOU window between port discovery and server binding. - reservation.release(); + reservation?.release(); this.process = spawn(this.serverPath, [tempConfigPath], { + cwd: this.tempDir, stdio: ["ignore", "pipe", "pipe"], detached: false }); + this.expectedExit = false; + this.unexpectedExit = undefined; this.process.stdout?.on("data", (data: Buffer) => { this.logger.info(`[SERVER] ${data.toString().trim()}`); @@ -102,6 +128,10 @@ export class ServerControl { const currentProcess = this.process; currentProcess.on("exit", (code, signal) => { + if (!this.expectedExit) + this.unexpectedExit = new Error( + `Owned server exited unexpectedly (code ${code}, signal ${signal})` + ); this.logger.info( `Server exited with code ${code}, signal ${signal}` ); @@ -139,7 +169,9 @@ export class ServerControl { ); } try { - const response = await fetch(pingUrl); + const response = await fetch(pingUrl, { + signal: AbortSignal.timeout(1_000) + }); if (response.ok) { this.logger.info("[SERVER] Ready"); return; @@ -194,11 +226,13 @@ export class ServerControl { public async stop(): Promise { const proc = this.process; if (proc?.pid === undefined) { - this.cleanupTempDir(); + if (!this.preserveOnStop) this.cleanupTempDir(); + if (this.unexpectedExit) throw this.unexpectedExit; return; } + this.expectedExit = true; - // Resume if paused — a SIGSTOP'd process ignores SIGKILL + // SIGKILL terminates even a stopped process; SIGCONT is harmless here. if (this._isPaused) { try { process.kill(proc.pid, "SIGCONT"); @@ -212,7 +246,7 @@ export class ServerControl { // Set up a promise that resolves when the process actually exits. const exitPromise = new Promise((resolve) => { - if (proc.exitCode !== null) { + if (proc.exitCode !== null || proc.signalCode !== null) { resolve(); return; } @@ -229,11 +263,49 @@ export class ServerControl { // Wait for the process to actually exit before cleaning up, // with a 5s safety timeout to avoid hanging forever. - await Promise.race([exitPromise, sleep(STOP_TIMEOUT_MS)]); + let timer: ReturnType | undefined; + try { + await Promise.race([ + exitPromise, + new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + "Owned server did not exit after SIGKILL" + ) + ), + STOP_TIMEOUT_MS + ); + }) + ]); + } finally { + clearTimeout(timer); + } this.process = null; this._isPaused = false; - this.cleanupTempDir(); + if (!this.preserveOnStop) this.cleanupTempDir(); + if (this.unexpectedExit) throw this.unexpectedExit; + } + + /** A real process death: retain the exact database and port for restart. */ + public async crash(): Promise { + if (!this.isRunning()) throw new Error("Cannot crash a stopped server"); + this.preserveOnStop = true; + try { + await this.stop(); + } finally { + this.preserveOnStop = false; + } + } + + public async restart(): Promise { + if (!this.tempDir || this.isRunning()) + throw new Error( + "Restart requires a crashed server with retained storage" + ); + await this.startOnce(); } public isRunning(): boolean { @@ -273,23 +345,27 @@ export class ServerControl { // one `databases_directory_path:` (under `server:` and `database:` // respectively) const baseConfig = fs.readFileSync(this.baseConfigPath, "utf-8"); + for (const key of ["port", "databases_directory_path"]) { + if ( + (baseConfig.match(new RegExp(`^ ${key}:`, "gm")) ?? []) + .length !== 1 + ) + throw new Error(`Expected exactly one config key: ${key}`); + } const config = baseConfig .replace(/^\s*port:\s*\d+/m, ` port: ${this._port}`) .replace( /^\s*databases_directory_path:\s*.+/m, - ` databases_directory_path: ${dbDir}` + ` databases_directory_path: ${JSON.stringify(dbDir)}` ); fs.writeFileSync(destPath, config); } private cleanupTempDir(): void { if (this.tempDir !== undefined) { - try { - fs.rmSync(this.tempDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup - } + fs.rmSync(this.tempDir, { recursive: true, force: true }); this.tempDir = undefined; + this._port = undefined; } } } diff --git a/frontend/deterministic-tests/src/server-manager.ts b/frontend/deterministic-tests/src/server-manager.ts index 76c624f7..53862fae 100644 --- a/frontend/deterministic-tests/src/server-manager.ts +++ b/frontend/deterministic-tests/src/server-manager.ts @@ -59,9 +59,8 @@ export class ServerManager { // Last-resort synchronous cleanup. Runs even when the process is // exiting via process.exit() from unhandledRejection / // uncaughtException — paths where async stopAll() cannot complete. - // SIGSTOP'd servers MUST receive SIGCONT before SIGKILL or the - // kernel keeps them as zombies holding the test's tmpdir, and the - // next CI run can't reuse the port. + // Kill only the server processes owned by this runner. SIGKILL also + // terminates a SIGSTOP'd process; forceKillSync's SIGCONT is harmless. process.on("exit", () => { for (const server of this.activeServers) { server.forceKillSync(); diff --git a/frontend/deterministic-tests/src/test-definition.ts b/frontend/deterministic-tests/src/test-definition.ts index bd832a50..3e4c5fe0 100644 --- a/frontend/deterministic-tests/src/test-definition.ts +++ b/frontend/deterministic-tests/src/test-definition.ts @@ -3,6 +3,18 @@ import type { AssertableState } from "./utils/assertable-state"; export interface ClientState { files: Map; clientFiles: Map[]; + manifests: Record[]; + canonical: Record; + bytes: Map; +} + +/** A scenario's independent, complete expected state. Keys survive moves; + * a new key must receive a previously unseen UUID, including after deletion. */ +export interface ExpectedDocument { + key: string; + path: string; + conflict?: boolean; + content: string | number[]; } export type TestStep = @@ -31,10 +43,41 @@ export type TestStep = | { type: "assert-consistent"; verify?: (state: AssertableState) => void } | { type: "pause-websocket"; client: number } | { type: "resume-websocket"; client: number } + | { + type: + | "pause-observation" + | "resume-observation" + | "wait-for-observation"; + client: number; + } | { type: "drop-next-create-response"; client: number } | { type: "wait-for-dropped-create-response"; client: number } | { type: "sleep"; ms: number } - | { type: "reset"; client: number }; + | { type: "reset"; client: number } + | { type: "create-bytes"; client: number; path: string; bytes: number[] } + | { + type: "drop-response"; + client: number; + kind: "create" | "content" | "manifest"; + point?: "before" | "after"; + } + | { type: "wait-for-response-drop"; client: number } + | { type: "delay-notifications" | "flush-notifications"; client: number } + | { + type: "remember-identity" | "assert-identity"; + path: string; + key: string; + } + | { + type: "assert-files"; + expected: Record; + absent?: string[]; + count?: number; + } + | { type: "assert-markers"; markers: string[]; removed?: string[] } + | { type: "assert-documents"; expected: ExpectedDocument[] } + | { type: "crash-server" } + | { type: "restart-server" }; export interface TestDefinition { description?: string; @@ -46,4 +89,5 @@ export interface TestResult { success: boolean; error?: string; duration?: number; + diagnostics?: unknown; } diff --git a/frontend/deterministic-tests/src/test-registry.ts b/frontend/deterministic-tests/src/test-registry.ts index 1a07b411..322fe049 100644 --- a/frontend/deterministic-tests/src/test-registry.ts +++ b/frontend/deterministic-tests/src/test-registry.ts @@ -1,4 +1,5 @@ import type { TestDefinition } from "./test-definition"; +import { V4_TESTS } from "./v4-regressions"; import { renameCreateConflictTest } from "./tests/rename-create-conflict.test"; import { renameChainTest } from "./tests/rename-chain.test"; import { renameUpdateConflictTest } from "./tests/rename-update-conflict.test"; @@ -106,6 +107,7 @@ import { remoteQuickWriteRenameBeforeRecordTest } from "./tests/remote-quick-wri import { selfMergePendingRenameAliasesSecondCreateTest } from "./tests/self-merge-pending-rename-aliases-second-create.test"; export const TESTS: Partial> = { + ...V4_TESTS, "rename-create-conflict": renameCreateConflictTest, "rename-chain": renameChainTest, "rename-update-conflict": renameUpdateConflictTest, diff --git a/frontend/deterministic-tests/src/test-runner.ts b/frontend/deterministic-tests/src/test-runner.ts index cf99f7f2..2ea7db3a 100644 --- a/frontend/deterministic-tests/src/test-runner.ts +++ b/frontend/deterministic-tests/src/test-runner.ts @@ -13,6 +13,8 @@ import { IS_SYNC_ENABLED_BY_DEFAULT } from "./consts"; import { randomUUID } from "node:crypto"; +import { assertCanonical } from "../../test-support/canonical"; +import { ContentLedger } from "../../test-support/oracles"; export class TestRunner { private agents: DeterministicAgent[] = []; @@ -20,6 +22,8 @@ export class TestRunner { private readonly token: string; private readonly remoteUri: string; private readonly logger: Logger; + private vaultName = ""; + private readonly identities = new Map(); public constructor( serverControl: ServerControl, @@ -44,6 +48,8 @@ export class TestRunner { } this.logger.info(`Clients: ${test.clients}`); this.logger.info(`Steps: ${test.steps.length}`); + this.identities.clear(); + let diagnostics: TestResult["diagnostics"]; try { assert( @@ -61,6 +67,11 @@ export class TestRunner { await this.executeStep(step); } + assert( + this.serverControl.isRunning(), + "Server died before test completion" + ); + diagnostics = this.captureDiagnostics(); await this.cleanup(); const duration = Date.now() - startTime; @@ -71,25 +82,46 @@ export class TestRunner { duration }; } catch (error) { + diagnostics ??= this.captureDiagnostics(); const duration = Date.now() - startTime; const errorMessage = error instanceof Error ? error.message : String(error); this.logger.info(`\n✗ Test failed: ${name}`); this.logger.info(`Error: ${errorMessage}`); - await this.cleanup(); + try { + await this.cleanup(); + } catch (cleanupError) { + return { + success: false, + error: `${errorMessage}\nCleanup: ${String(cleanupError)}`, + duration, + diagnostics + }; + } return { success: false, error: errorMessage, + diagnostics, duration }; } } + private captureDiagnostics(): unknown { + return this.agents.map((agent) => ({ + client: agent.clientId, + database: agent.database(), + disk: agent.disk.image(), + requests: structuredClone(agent.network.requests) + })); + } + private async initializeAgents(count: number): Promise { assert(count > 0, `Client count must be positive, got ${count}`); const vaultName = `test-${randomUUID()}`; + this.vaultName = vaultName; this.logger.info( `Initializing ${count} agents with vault: ${vaultName}` ); @@ -129,6 +161,87 @@ export class TestRunner { private async executeStep(step: TestStep): Promise { switch (step.type) { + case "pause-observation": + this.getAgent(step.client).pauseObservation(); + break; + case "resume-observation": + this.getAgent(step.client).resumeObservation(); + break; + case "wait-for-observation": + await this.getAgent(step.client).waitForObservation(); + break; + case "create-bytes": + await this.getAgent(step.client).write( + step.path, + new Uint8Array(step.bytes) + ); + break; + case "drop-response": + this.getAgent(step.client).dropNextResponse( + step.kind, + step.point + ); + break; + case "wait-for-response-drop": + await this.getAgent(step.client).waitForDroppedCreateResponse(); + break; + case "delay-notifications": + this.getAgent(step.client).delayNotifications(); + break; + case "flush-notifications": + this.getAgent(step.client).flushNotifications(); + break; + case "remember-identity": + case "assert-identity": + await this.assertConsistent((state) => { + if (step.type === "remember-identity") + this.identities.set( + step.key, + state.documentId(step.path) + ); + else { + const id = this.identities.get(step.key); + assert( + id !== undefined, + `Identity ${step.key} was never recorded` + ); + state.assertIdentity(step.path, id); + } + }); + break; + case "assert-files": + await this.assertConsistent((state) => { + for (const [path, value] of Object.entries(step.expected)) + state.assertContent(path, value); + for (const path of step.absent ?? []) + state.assertFileNotExists(path); + if (step.count !== undefined) + state.assertFileCount(step.count); + }); + break; + case "assert-documents": + await this.assertConsistent((state) => + state.assertDocuments(step.expected, this.identities) + ); + break; + case "assert-markers": { + const ledger = new ContentLedger(); + for (const marker of step.markers) ledger.add(marker); + for (const marker of step.removed ?? []) + ledger.removedBy( + marker, + "trace records explicit deletion/overwrite" + ); + for (const agent of this.agents) + ledger.assertPreserved(agent.files()); + break; + } + case "crash-server": + await this.serverControl.crash(); + break; + case "restart-server": + await this.serverControl.restart(); + break; case "create": case "update": await this.getAgent(step.client).write( @@ -170,7 +283,9 @@ export class TestRunner { break; case "enable-sync": - await this.getAgent(step.client).enableSync(); + await this.getAgent(step.client).enableSync( + !this.serverControl.isPaused + ); break; case "pause-server": @@ -190,7 +305,9 @@ export class TestRunner { (entry) => entry.details.type === step.syncType && entry.details.relativePath === step.path, - () => { this.serverControl.pause(); } + () => { + this.serverControl.pause(); + } ); this.serverControl.resume(); await historySeen; @@ -305,6 +422,11 @@ export class TestRunner { // Snapshot all agents' file states upfront to minimize the window // where background sync could mutate state between reads. + const canonical = await assertCanonical( + this.agents, + `${this.remoteUri}/vaults/${encodeURIComponent(this.vaultName)}`, + this.token + ); const clientFiles: Map[] = []; for (const agent of this.agents) { const sortedFiles = (await agent.listFilesRecursively()).sort(); @@ -360,7 +482,12 @@ export class TestRunner { verify( new AssertableState({ files: clientFiles[0], - clientFiles + clientFiles, + manifests: this.agents.map( + (agent) => agent.database()!.local! + ), + canonical: canonical.fileManifest.entries, + bytes: this.agents[0].files() }) ); } catch (error) { @@ -384,10 +511,12 @@ export class TestRunner { } this.logger.info("\nCleaning up agents..."); - for (const agent of this.agents) { + const errors: unknown[] = []; + for (const agent of this.agents.splice(0)) { try { await agent.cleanup(); } catch (error) { + errors.push(error); this.logger.warn( `Agent cleanup error: ${error instanceof Error ? error.message : String(error)}` ); @@ -395,5 +524,10 @@ export class TestRunner { } this.agents = []; this.logger.info("Cleanup complete"); + if (errors.length) + throw new AggregateError( + errors, + `Cleanup failed: ${errors.map(String).join("; ")}` + ); } } diff --git a/frontend/deterministic-tests/src/utils/assertable-state.ts b/frontend/deterministic-tests/src/utils/assertable-state.ts index 7c6f192c..c208fa13 100644 --- a/frontend/deterministic-tests/src/utils/assertable-state.ts +++ b/frontend/deterministic-tests/src/utils/assertable-state.ts @@ -1,12 +1,103 @@ -import type { ClientState } from "../test-definition"; +import assert from "node:assert/strict"; +import type { ClientState, ExpectedDocument } from "../test-definition"; export class AssertableState { public readonly files: Map; public readonly clientFiles: Map[]; + public readonly manifests: Record[]; + public readonly canonical: Record; + public readonly bytes: Map; public constructor(state: ClientState) { this.files = state.files; this.clientFiles = state.clientFiles; + this.manifests = state.manifests; + this.canonical = state.canonical; + this.bytes = state.bytes; + } + + public documentId(path: string): string { + const id = Object.keys(this.canonical).find( + (id) => this.canonical[id] === path + ); + if (!id) throw new Error(`No document identity for ${path}`); + return id; + } + + public assertDocuments( + expected: readonly ExpectedDocument[], + identities: Map + ): this { + this.assertFileCount(expected.length); + assert.equal( + new Set(expected.map((doc) => doc.key)).size, + expected.length, + "Duplicate expected identity key" + ); + const next = new Map(identities); + const paths = new Set(); + for (const doc of expected) { + const path = doc.conflict ? this.conflictPath(doc.path) : doc.path; + assert( + !paths.has(path), + `Two expected documents resolved to ${path}` + ); + paths.add(path); + this.assertBytes( + path, + typeof doc.content === "string" + ? new TextEncoder().encode(doc.content) + : new Uint8Array(doc.content) + ); + const id = this.documentId(path); + const recorded = next.get(doc.key); + if (recorded !== undefined) this.assertIdentity(path, recorded); + else { + assert( + ![...next.values()].includes(id), + `New document ${doc.key} reused an existing or deleted UUID` + ); + next.set(doc.key, id); + } + } + // A failed check must not establish a partial identity oracle. + for (const [key, id] of next) identities.set(key, id); + return this; + } + + public conflictPath(original: string): string { + const dot = original.lastIndexOf("."); + const split = + dot > original.lastIndexOf("/") + 1 ? dot : original.length; + const matches = Object.entries(this.canonical).filter( + ([id, path]) => + path === + `${original.slice(0, split)} (conflict ${id})${original.slice(split)}` + ); + if (matches.length !== 1) + throw new Error( + `Expected one UUID-derived conflict path for ${original}, found ${matches.length}` + ); + return matches[0][1]; + } + + public assertIdentity(path: string, id: string): this { + if (this.documentId(path) !== id) + throw new Error( + `Document identity changed at ${path}: expected ${id}, got ${this.documentId(path)}` + ); + return this; + } + + public assertBytes(path: string, expected: Uint8Array): this { + const actual = this.bytes.get(path); + if ( + !actual || + actual.length !== expected.length || + actual.some((byte, i) => byte !== expected[i]) + ) + throw new Error(`Byte mismatch at ${path}`); + return this; } public assertFileCount(expected: number): this { diff --git a/frontend/deterministic-tests/src/v4-regressions.ts b/frontend/deterministic-tests/src/v4-regressions.ts new file mode 100644 index 00000000..75841fc4 --- /dev/null +++ b/frontend/deterministic-tests/src/v4-regressions.ts @@ -0,0 +1,253 @@ +import type { TestDefinition, TestStep } from "./test-definition"; + +const online: TestStep[] = [ + { type: "enable-sync", client: 0 }, + { type: "enable-sync", client: 1 }, + { type: "barrier" } +]; +const offline: TestStep[] = [ + { type: "disable-sync", client: 0 }, + { type: "disable-sync", client: 1 } +]; +export const V4_TESTS: Record = { + "v4-text-byte-and-component-boundaries": { + clients: 2, + steps: [ + ...online, + { type: "create", client: 0, path: "empty.md", content: "" }, + { + type: "create", + client: 0, + path: "bom.md", + content: "\ufefffirst\r\nlast" + }, + { + type: "create", + client: 0, + path: `${"a".repeat(252)}.md`, + content: "255 ASCII bytes" + }, + { + type: "create", + client: 0, + path: `${"é".repeat(126)}.md`, + content: "255 UTF-8 bytes" + }, + { type: "barrier" }, + { + type: "assert-files", + count: 4, + expected: { + "empty.md": "", + "bom.md": "\ufefffirst\r\nlast", + [`${"a".repeat(252)}.md`]: "255 ASCII bytes", + [`${"é".repeat(126)}.md`]: "255 UTF-8 bytes" + } + } + ] + }, + "v4-binary-bytes-and-independent-identities": { + clients: 2, + steps: [ + ...online, + ...offline, + { + type: "create-bytes", + client: 0, + path: "nested/blob.bin", + bytes: [0, 255, 254, 128, 13, 10] + }, + { + type: "create-bytes", + client: 1, + path: "nested/blob.bin", + bytes: [0, 254, 255, 129, 10, 13] + }, + ...online, + { + type: "assert-consistent", + verify: (state) => { + state + .assertFileCount(2) + .assertBytes( + "nested/blob.bin", + new Uint8Array([0, 255, 254, 128, 13, 10]) + ); + state.assertBytes( + state.conflictPath("nested/blob.bin"), + new Uint8Array([0, 254, 255, 129, 10, 13]) + ); + } + } + ] + }, + "v4-three-file-cycle-preserves-identities": { + clients: 2, + steps: [ + ...online, + { type: "create", client: 0, path: "a.md", content: "A" }, + { type: "create", client: 0, path: "b.md", content: "B" }, + { type: "create", client: 0, path: "c.md", content: "C" }, + { type: "barrier" }, + { type: "remember-identity", key: "a", path: "a.md" }, + { type: "remember-identity", key: "b", path: "b.md" }, + { type: "remember-identity", key: "c", path: "c.md" }, + { type: "disable-sync", client: 0 }, + { type: "delay-notifications", client: 0 }, + { type: "rename", client: 0, oldPath: "a.md", newPath: "temp.md" }, + { type: "rename", client: 0, oldPath: "b.md", newPath: "a.md" }, + { type: "rename", client: 0, oldPath: "c.md", newPath: "b.md" }, + { type: "rename", client: 0, oldPath: "temp.md", newPath: "c.md" }, + { + type: "update", + client: 0, + path: "c.md", + content: "A edited offline" + }, + { type: "flush-notifications", client: 0 }, + { type: "enable-sync", client: 0 }, + { type: "barrier" }, + { type: "assert-identity", key: "a", path: "c.md" }, + { type: "assert-identity", key: "b", path: "a.md" }, + { type: "assert-identity", key: "c", path: "b.md" }, + { + type: "assert-files", + count: 3, + expected: { + "a.md": "B", + "b.md": "C", + "c.md": "A edited offline" + }, + absent: ["temp.md"] + } + ] + }, + "v4-case-alias-and-file-directory-conflicts": { + clients: 2, + steps: [ + ...online, + ...offline, + { + type: "create", + client: 0, + path: "folder/note.md", + content: "nested file" + }, + { + type: "create", + client: 0, + path: "README.md", + content: "uppercase file" + }, + { + type: "create", + client: 1, + path: "folder", + content: "blocking file" + }, + { + type: "create", + client: 1, + path: "readme.md", + content: "lowercase file" + }, + ...online, + { + type: "assert-consistent", + verify: (state) => { + state + .assertFileCount(4) + .assertContent("folder/note.md", "nested file") + .assertContent("README.md", "uppercase file"); + state.assertAnyFileContains( + "blocking file", + "lowercase file" + ); + } + }, + { + type: "assert-markers", + markers: [ + "nested file", + "blocking file", + "uppercase file", + "lowercase file" + ] + } + ] + }, + "v4-server-sigkill-retains-acknowledged-state": { + clients: 2, + steps: [ + ...online, + { + type: "create", + client: 0, + path: "a.md", + content: "durable bytes" + }, + { type: "barrier" }, + { type: "remember-identity", key: "a", path: "a.md" }, + ...offline, + { type: "crash-server" }, + { type: "restart-server" }, + ...online, + { type: "assert-identity", key: "a", path: "a.md" }, + { + type: "assert-files", + count: 1, + expected: { "a.md": "durable bytes" } + } + ] + } +}; + +for (const kind of ["create", "content", "manifest"] as const) { + for (const point of ["before", "after"] as const) { + V4_TESTS[`v4-${kind}-${point}-commit-network-loss`] = { + clients: 2, + steps: [ + ...online, + ...(kind === "content" + ? ([ + { + type: "create", + client: 0, + path: "file.md", + content: "old bytes" + }, + { type: "barrier" }, + { + type: "remember-identity", + key: "updated-file", + path: "file.md" + } + ] as TestStep[]) + : []), + { type: "drop-response", client: 0, kind, point }, + { + type: kind === "content" ? "update" : "create", + client: 0, + path: "file.md", + content: "must survive retry" + }, + { type: "wait-for-response-drop", client: 0 }, + { type: "barrier" }, + ...(kind === "content" + ? [ + { + type: "assert-identity", + key: "updated-file", + path: "file.md" + } as TestStep + ] + : []), + { + type: "assert-files", + count: 1, + expected: { "file.md": "must survive retry" } + } + ] + }; + } +} diff --git a/frontend/deterministic-tests/tsconfig.json b/frontend/deterministic-tests/tsconfig.json index 7558871d..47e92325 100644 --- a/frontend/deterministic-tests/tsconfig.json +++ b/frontend/deterministic-tests/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "baseUrl": ".", "strict": true, + "skipLibCheck": true, "target": "ES2022", "module": "CommonJS", "esModuleInterop": true, diff --git a/frontend/deterministic-tests/webpack.config.js b/frontend/deterministic-tests/webpack.config.js index 6aee1547..12c1653f 100644 --- a/frontend/deterministic-tests/webpack.config.js +++ b/frontend/deterministic-tests/webpack.config.js @@ -2,8 +2,9 @@ const path = require("path"); const webpack = require("webpack"); module.exports = { - entry: "./src/cli.ts", + entry: { cli: "./src/cli.ts", "crash-worker": "./src/crash-worker.ts" }, target: "node", + externals: { "sync-client": "commonjs sync-client" }, mode: "production", optimization: { minimize: false @@ -21,7 +22,7 @@ module.exports = { }, output: { globalObject: "this", - filename: "cli.js", + filename: "[name].js", path: path.resolve(__dirname, "dist") }, plugins: [ diff --git a/frontend/local-client-cli/src/node-filesystem.test.ts b/frontend/local-client-cli/src/node-filesystem.test.ts index 85fb7a0f..55b286a4 100644 --- a/frontend/local-client-cli/src/node-filesystem.test.ts +++ b/frontend/local-client-cli/src/node-filesystem.test.ts @@ -51,7 +51,7 @@ test("NodeFileSystemOperations - exists with forward slashes", async () => { } }); -test("NodeFileSystemOperations - delete with forward slashes", async () => { +test("NodeFileSystemOperations - deleteFile with forward slashes", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); @@ -59,8 +59,10 @@ test("NodeFileSystemOperations - delete with forward slashes", async () => { await fsOps.write("test.txt", new TextEncoder().encode("test")); assert.equal(await fsOps.exists("test.txt"), true); - await fsOps.delete("test.txt"); + await fsOps.deleteFile("test.txt"); assert.equal(await fsOps.exists("test.txt"), false); + + await fsOps.deleteFile("test.txt"); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -154,9 +156,46 @@ test("NodeFileSystemOperations - handles paths with forward slashes on all platf const readContent = await fsOps.read(testPath); assert.equal(new TextDecoder().decode(readContent), "test"); - await fsOps.delete(testPath); + await fsOps.deleteFile(testPath); assert.equal(await fsOps.exists(testPath), false); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test("NodeFileSystemOperations - delete prunes empty directory trees", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); + const fsOps = new NodeFileSystemOperations(tempDir); + + try { + await fs.mkdir(path.join(tempDir, "parent", "child"), { + recursive: true + }); + + await fsOps.delete("parent"); + assert.equal(await fsOps.exists("parent"), false); + + await fsOps.delete("parent"); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test("NodeFileSystemOperations - delete does not remove files", async () => { + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); + const fsOps = new NodeFileSystemOperations(tempDir); + + try { + await fsOps.write( + "parent/file.txt", + new TextEncoder().encode("preserve me") + ); + + await assert.rejects(async () => fsOps.delete("parent")); + assert.equal(await fsOps.exists("parent/file.txt"), true); + await assert.rejects(async () => fsOps.delete("parent/file.txt")); + assert.equal(await fsOps.exists("parent/file.txt"), true); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index ba95ab6a..521f76bd 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -15,7 +15,7 @@ import { toUnixPath } from "./path-utils"; export const VAULTLINK_DIR = ".vaultlink"; export class NodeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly basePath: string) { } + public constructor(private readonly basePath: string) {} public async listFilesRecursively( directory: RelativePath | undefined @@ -105,9 +105,36 @@ export class NodeFileSystemOperations implements FileSystemOperations { } public async delete(relativePath: RelativePath): Promise { + if (!relativePath) { + throw new Error("Cannot delete the vault root"); + } + const fullPath = path.join(this.basePath, relativePath); try { - await fs.unlink(fullPath); + await this.deleteDirectoryTree(fullPath); + } catch (error) { + throw new Error( + `Failed to delete directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + public async deleteFile(relativePath: RelativePath): Promise { + const fullPath = path.join(this.basePath, relativePath); + try { + const entry = await fs.lstat(fullPath).catch((error: unknown) => { + if (this.isMissing(error)) return undefined; + throw error; + }); + if (!entry) return; + if (!entry.isFile() || entry.nlink !== 1) { + throw new Error("Cannot unlink a non-regular or linked file"); + } + + await fs.unlink(fullPath).catch((error: unknown) => { + if (!this.isMissing(error)) throw error; + }); + await this.syncDirectory(path.dirname(fullPath)); } catch (error) { throw new Error( `Failed to delete file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` @@ -171,6 +198,40 @@ export class NodeFileSystemOperations implements FileSystemOperations { } } + private async deleteDirectoryTree(fullPath: string): Promise { + const entry = await fs.lstat(fullPath).catch((error: unknown) => { + if (this.isMissing(error)) return undefined; + throw error; + }); + if (!entry) return; + if (!entry.isDirectory()) { + throw new Error("Cannot delete a regular file as a directory"); + } + + for (const child of await fs.readdir(fullPath, { + withFileTypes: true + })) { + if (!child.isDirectory()) { + throw new Error( + `Directory contains a non-directory entry: ${path.join(fullPath, child.name)}` + ); + } + await this.deleteDirectoryTree(path.join(fullPath, child.name)); + } + + await fs.rmdir(fullPath); + await this.syncDirectory(path.dirname(fullPath)); + } + + private isMissing(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ); + } + private async walkDirectory( relativePath: string, files: RelativePath[] diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index ceb8bc2a..fec25c59 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -152,9 +152,20 @@ export class ObsidianFileSystemOperations implements FileSystemOperations { } public async delete(path: RelativePath): Promise { - if (!(await this.vault.adapter.trashSystem(normalizePath(path)))) { - return this.vault.adapter.remove(normalizePath(path)); + if (!path) { + throw new Error("Cannot delete the vault root"); } + await this.deleteDirectoryTree(normalizePath(path)); + } + + public async deleteFile(path: RelativePath): Promise { + path = normalizePath(path); + const entry = await this.vault.adapter.stat(path); + if (!entry) return; + if (entry.type !== "file") { + throw new Error(`Cannot delete non-file: ${path}`); + } + await this.vault.adapter.remove(path); } public async rename( @@ -173,4 +184,21 @@ export class ObsidianFileSystemOperations implements FileSystemOperations { return file; } + + private async deleteDirectoryTree(path: string): Promise { + const entry = await this.vault.adapter.stat(path); + if (!entry) return; + if (entry.type !== "folder") { + throw new Error(`Cannot delete non-directory: ${path}`); + } + + const children = await this.vault.adapter.list(path); + if (children.files.length > 0) { + throw new Error(`Cannot delete non-empty directory: ${path}`); + } + for (const folder of children.folders) { + await this.deleteDirectoryTree(folder); + } + await this.vault.adapter.rmdir(path, false); + } } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4d8218ba..5aa0bee9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,6 +9,7 @@ "sync-client", "obsidian-plugin", "test-client", + "deterministic-tests", "local-client-cli" ], "devDependencies": { @@ -21,26 +22,102 @@ "typescript-eslint": "8.41.0" } }, + "deterministic-tests": { + "version": "0.14.0", + "bin": { + "deterministic-tests": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^25.0.2", + "commander": "^14.0.2", + "sync-client": "file:../sync-client", + "ts-loader": "^9.5.4", + "tslib": "2.8.1", + "typescript": "5.9.3", + "webpack": "^5.103.0", + "webpack-cli": "^6.0.1" + } + }, + "deterministic-tests/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "deterministic-tests/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "deterministic-tests/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "local-client-cli": { "version": "0.14.0", - "dependencies": { - "commander": "^14.0.2", - "watcher": "^2.3.1" - }, "bin": { "vaultlink": "dist/cli.js" }, "devDependencies": { - "@types/node": "^24.8.1", + "@types/node": "^25.0.2", + "commander": "^14.0.2", "sync-client": "file:../sync-client", - "ts-loader": "^9.5.2", + "ts-loader": "^9.5.4", "tslib": "2.8.1", - "tsx": "^4.20.6", - "typescript": "5.8.3", - "webpack": "^5.99.9", + "tsx": "^4.21.0", + "typescript": "5.9.3", + "watcher": "^2.3.1", + "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } }, + "local-client-cli/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "local-client-cli/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "local-client-cli/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "node_modules/@codemirror/state": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/@codemirror/state/-/state-6.5.0.tgz", @@ -52,20 +129,6 @@ "@marijn/find-cluster-break": "^1.0.0" } }, - "node_modules/@codemirror/view": { - "version": "6.38.1", - "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.1.tgz", - "integrity": "sha512-RmTOkE7hRU3OVREqFVITWHz6ocgBjv08GoePscAakgVQfciA3SGCEk7mb9IzwW61cKKmlTpHXG6DUE5Ubx+MGQ==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@codemirror/state": "^6.5.0", - "crelt": "^1.0.6", - "style-mod": "^4.1.0", - "w3c-keyname": "^2.2.4" - } - }, "node_modules/@discoveryjs/json-ext": { "version": "0.6.3", "dev": true, @@ -75,9 +138,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz", - "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", "cpu": [ "ppc64" ], @@ -92,9 +155,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz", - "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", "cpu": [ "arm" ], @@ -109,9 +172,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz", - "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", "cpu": [ "arm64" ], @@ -126,9 +189,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz", - "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", "cpu": [ "x64" ], @@ -143,9 +206,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz", - "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", "cpu": [ "arm64" ], @@ -160,9 +223,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz", - "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", "cpu": [ "x64" ], @@ -177,9 +240,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz", - "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", "cpu": [ "arm64" ], @@ -194,9 +257,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz", - "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", "cpu": [ "x64" ], @@ -211,9 +274,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz", - "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", "cpu": [ "arm" ], @@ -228,9 +291,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz", - "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", "cpu": [ "arm64" ], @@ -245,9 +308,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz", - "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", "cpu": [ "ia32" ], @@ -262,9 +325,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz", - "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", "cpu": [ "loong64" ], @@ -279,9 +342,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz", - "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", "cpu": [ "mips64el" ], @@ -296,9 +359,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz", - "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", "cpu": [ "ppc64" ], @@ -313,9 +376,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz", - "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", "cpu": [ "riscv64" ], @@ -330,9 +393,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz", - "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", "cpu": [ "s390x" ], @@ -347,9 +410,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz", - "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", "cpu": [ "x64" ], @@ -364,9 +427,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz", - "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", "cpu": [ "arm64" ], @@ -381,9 +444,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz", - "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", "cpu": [ "x64" ], @@ -398,9 +461,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz", - "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", "cpu": [ "arm64" ], @@ -415,9 +478,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz", - "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", "cpu": [ "x64" ], @@ -432,9 +495,9 @@ } }, "node_modules/@esbuild/openharmony-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz", - "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", "cpu": [ "arm64" ], @@ -449,9 +512,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz", - "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", "cpu": [ "x64" ], @@ -466,9 +529,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz", - "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", "cpu": [ "arm64" ], @@ -483,9 +546,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz", - "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", "cpu": [ "ia32" ], @@ -500,9 +563,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz", - "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", "cpu": [ "x64" ], @@ -723,6 +786,8 @@ }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", "engines": { @@ -730,7 +795,9 @@ } }, "node_modules/@jridgewell/source-map": { - "version": "0.3.6", + "version": "0.3.11", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.11.tgz", + "integrity": "sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==", "dev": true, "license": "MIT", "dependencies": { @@ -739,14 +806,16 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.0", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.30", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", - "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", "dev": true, "license": "MIT", "dependencies": { @@ -759,7 +828,8 @@ "resolved": "https://registry.npmjs.org/@marijn/find-cluster-break/-/find-cluster-break-1.0.2.tgz", "integrity": "sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", @@ -961,26 +1031,10 @@ "@types/tern": "*" } }, - "node_modules/@types/eslint": { - "version": "9.6.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/estree": "*", - "@types/json-schema": "*" - } - }, - "node_modules/@types/eslint-scope": { - "version": "3.7.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/eslint": "*", - "@types/estree": "*" - } - }, "node_modules/@types/estree": { - "version": "1.0.7", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", "dev": true, "license": "MIT" }, @@ -1013,7 +1067,6 @@ "integrity": "sha512-8fz6oa6wEKZrhXWro/S3n2eRJqlRcIa6SlDh59FXJ5Wp5XRZ8B9ixpJDcjadHq47hMx0u+HW6SNa6LjJQ6NLtw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.41.0", @@ -1052,7 +1105,6 @@ "integrity": "sha512-gTtSdWX9xiMPA/7MV9STjJOOYtWwIJIYxkQxnSV1U3xcE+mnJSH3f6zI0RYP+ew66WSlZ5ed+h0VCxsvdC1jJg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.41.0", "@typescript-eslint/types": "8.41.0", @@ -1453,7 +1505,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1485,7 +1536,6 @@ "version": "6.12.6", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -1763,6 +1813,19 @@ "dev": true, "license": "MIT" }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.25", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.25.tgz", + "integrity": "sha512-gMmEShwwq7FJqMwvfRwvCl00v4kN+KOfJqXn+f4nrufak5gNHJOksd/60Dvjuz7sI8Y5WiSFBa8FEYr+zoyqCw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/big.js": { "version": "5.2.2", "dev": true, @@ -1804,7 +1867,9 @@ } }, "node_modules/browserslist": { - "version": "4.24.4", + "version": "4.29.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", + "integrity": "sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==", "dev": true, "funding": [ { @@ -1821,12 +1886,12 @@ } ], "license": "MIT", - "peer": true, "dependencies": { - "caniuse-lite": "^1.0.30001688", - "electron-to-chromium": "^1.5.73", - "node-releases": "^2.0.19", - "update-browserslist-db": "^1.1.1" + "baseline-browser-mapping": "^2.11.23", + "caniuse-lite": "^1.0.30001810", + "electron-to-chromium": "^1.5.427", + "node-releases": "^2.0.55", + "update-browserslist-db": "^1.3.3" }, "bin": { "browserslist": "cli.js" @@ -1860,6 +1925,8 @@ }, "node_modules/buffer-from": { "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true, "license": "MIT" }, @@ -1983,7 +2050,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001707", + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", "dev": true, "funding": [ { @@ -2038,14 +2107,16 @@ } }, "node_modules/chokidar": { - "version": "4.0.3", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", "dev": true, "license": "MIT", "dependencies": { - "readdirp": "^4.0.1" + "readdirp": "^5.0.0" }, "engines": { - "node": ">= 14.16.0" + "node": ">= 20.19.0" }, "funding": { "url": "https://paulmillr.com/funding/" @@ -2232,6 +2303,7 @@ "version": "14.0.2", "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=20" @@ -2286,7 +2358,8 @@ "resolved": "https://registry.npmjs.org/crelt/-/crelt-1.0.6.tgz", "integrity": "sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/cross-spawn": { "version": "7.0.6", @@ -2442,10 +2515,15 @@ "node": ">=0.10" } }, + "node_modules/deterministic-tests": { + "resolved": "deterministic-tests", + "link": true + }, "node_modules/dettle": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/dettle/-/dettle-1.0.5.tgz", "integrity": "sha512-ZVyjhAJ7sCe1PNXEGveObOH9AC8QvMga3HJIghHawtG7mE4K5pW9nz/vDGAr/U7a3LWgdOzEE7ac9MURnyfaTA==", + "dev": true, "license": "MIT" }, "node_modules/dunder-proto": { @@ -2771,7 +2849,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.127", + "version": "1.5.433", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.433.tgz", + "integrity": "sha512-5lCAbyZBjtmUt/RAGHRqrL2q0oEFRThDAsZHHDn9XHa89Qw7gMYOeSicBTy+AHfvo0r6vwsZvqNJTQIQy1BLzA==", "dev": true, "license": "ISC" }, @@ -2889,12 +2969,14 @@ } }, "node_modules/enhanced-resolve": { - "version": "5.18.1", + "version": "5.25.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.25.1.tgz", + "integrity": "sha512-nGXts5znJzmWPu+mIE9izCOzdg63oJca2mDzGWWTth7sr4aCToKcoyFVBQwN75Ij5Pf6p510EwkTqViTRzDV+w==", "dev": true, "license": "MIT", "dependencies": { "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" + "tapable": "^2.3.3" }, "engines": { "node": ">=10.13.0" @@ -2928,7 +3010,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.6.0", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", + "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", "dev": true, "license": "MIT" }, @@ -2944,9 +3028,9 @@ } }, "node_modules/esbuild": { - "version": "0.25.9", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz", - "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==", + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -2957,32 +3041,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.9", - "@esbuild/android-arm": "0.25.9", - "@esbuild/android-arm64": "0.25.9", - "@esbuild/android-x64": "0.25.9", - "@esbuild/darwin-arm64": "0.25.9", - "@esbuild/darwin-x64": "0.25.9", - "@esbuild/freebsd-arm64": "0.25.9", - "@esbuild/freebsd-x64": "0.25.9", - "@esbuild/linux-arm": "0.25.9", - "@esbuild/linux-arm64": "0.25.9", - "@esbuild/linux-ia32": "0.25.9", - "@esbuild/linux-loong64": "0.25.9", - "@esbuild/linux-mips64el": "0.25.9", - "@esbuild/linux-ppc64": "0.25.9", - "@esbuild/linux-riscv64": "0.25.9", - "@esbuild/linux-s390x": "0.25.9", - "@esbuild/linux-x64": "0.25.9", - "@esbuild/netbsd-arm64": "0.25.9", - "@esbuild/netbsd-x64": "0.25.9", - "@esbuild/openbsd-arm64": "0.25.9", - "@esbuild/openbsd-x64": "0.25.9", - "@esbuild/openharmony-arm64": "0.25.9", - "@esbuild/sunos-x64": "0.25.9", - "@esbuild/win32-arm64": "0.25.9", - "@esbuild/win32-ia32": "0.25.9", - "@esbuild/win32-x64": "0.25.9" + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" } }, "node_modules/escalade": { @@ -3010,7 +3094,6 @@ "integrity": "sha512-t5aPOpmtJcZcz5UJyY2GbvpDlsK5E8JqRqoKtfiKE3cNh437KIqfJr3A3AKf5k64NPx6d0G3dno6XDY05PqPtw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -3179,11 +3262,6 @@ "node": ">=0.10.0" } }, - "node_modules/eventemitter3": { - "version": "5.0.1", - "dev": true, - "license": "MIT" - }, "node_modules/events": { "version": "3.3.0", "dev": true, @@ -3555,7 +3633,9 @@ } }, "node_modules/fs-extra": { - "version": "11.3.0", + "version": "11.4.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", + "integrity": "sha512-EQsFzMUJkCKGr1ePqlYADkIUmHW1s3ZXr5Yqy6wbGrfUCphpl2maM/kyOIRA2HpP3AaFQTZXD4ldjek+nccddA==", "dev": true, "license": "MIT", "dependencies": { @@ -3667,19 +3747,6 @@ "node": ">=6" } }, - "node_modules/get-tsconfig": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.1.tgz", - "integrity": "sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -3759,11 +3826,6 @@ "node": ">=0.10.0" } }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "dev": true, - "license": "BSD-2-Clause" - }, "node_modules/globals": { "version": "14.0.0", "dev": true, @@ -4240,7 +4302,9 @@ } }, "node_modules/immutable": { - "version": "5.1.1", + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.9.tgz", + "integrity": "sha512-m8nVez3rwrgmWxtLMt1ZYXB2Lv7OKYn/disyxAlSDYAlKSlFoPPfIAmAM/M5xqL4m4C/wAPw7S2/CNaUii1Hxg==", "dev": true, "license": "MIT" }, @@ -4548,6 +4612,21 @@ "node": ">=0.10.0" } }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, "node_modules/js-yaml": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", @@ -4566,11 +4645,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "dev": true, - "license": "MIT" - }, "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, @@ -4694,14 +4768,6 @@ "iconv-lite": "^0.4.15" } }, - "node_modules/loader-runner": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.11.5" - } - }, "node_modules/loader-utils": { "version": "2.0.4", "dev": true, @@ -4812,6 +4878,8 @@ }, "node_modules/merge-stream": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "dev": true, "license": "MIT" }, @@ -4838,24 +4906,15 @@ } }, "node_modules/mime-db": { - "version": "1.52.0", + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -4867,7 +4926,9 @@ } }, "node_modules/mini-css-extract-plugin": { - "version": "2.9.2", + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/mini-css-extract-plugin/-/mini-css-extract-plugin-2.10.2.tgz", + "integrity": "sha512-AOSS0IdEB95ayVkxn5oGzNQwqAi2J0Jb/kKm43t7H73s8+f5873g0yuj0PNvK4dO75mu5DHg4nlgp4k6Kga8eg==", "dev": true, "license": "MIT", "dependencies": { @@ -4889,7 +4950,6 @@ "version": "8.17.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4956,6 +5016,154 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minimizer-webpack-plugin": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/minimizer-webpack-plugin/-/minimizer-webpack-plugin-5.11.0.tgz", + "integrity": "sha512-2JVy2DY+iBtfRyrQB5cRM7lfurnJIi6jFTgLip/7FxDu0WxswrIRzr4fBySQ0X7HqM7BcwE28zm+HfQ5V6y4Cg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.3", + "terser": "^5.51.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, + "@napi-rs/image": { + "optional": true + }, + "@swc/core": { + "optional": true + }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "html-minifier-terser": { + "optional": true + }, + "imagemin": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, + "sharp": { + "optional": true + }, + "svgo": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/minimizer-webpack-plugin/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimizer-webpack-plugin/node_modules/schema-utils": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", @@ -5038,22 +5246,16 @@ "license": "MIT", "optional": true }, - "node_modules/node-gyp-build": { - "version": "4.8.4", + "node_modules/node-releases": { + "version": "2.0.56", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.56.tgz", + "integrity": "sha512-x0InOIyzgdk+eyaWaRJFH5snEtiImgBgblZ2CyPrLmqqcuMQkEvcDPHbzqbD8eDsSeJbVOjn+crzyzHaM4D+/A==", "dev": true, "license": "MIT", - "optional": true, - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" + "engines": { + "node": ">=18" } }, - "node_modules/node-releases": { - "version": "2.0.19", - "dev": true, - "license": "MIT" - }, "node_modules/normalize-path": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", @@ -5170,21 +5372,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obsidian": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.10.2.tgz", - "integrity": "sha512-bX03YCHf06OTzI/D+QK71ajCPCmwr/cjxzlVXjQa10DjK5iHRWhtJJpp83arSCyayFMp23u+UHcY7hxcEx2Mvg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/codemirror": "5.60.8", - "moment": "2.29.4" - }, - "peerDependencies": { - "@codemirror/state": "6.5.0", - "@codemirror/view": "6.38.1" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -5329,32 +5516,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-queue": { - "version": "8.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^5.0.1", - "p-timeout": "^6.1.2" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-timeout": { - "version": "6.1.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-try": { "version": "2.2.0", "dev": true, @@ -5531,7 +5692,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.8", "picocolors": "^1.1.1", @@ -5648,6 +5808,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/promise-make-counter/-/promise-make-counter-1.0.2.tgz", "integrity": "sha512-FJAxTBWQuQoAs4ZOYuKX1FHXxEgKLEzBxUvwr4RoOglkTpOjWuM+RXsK3M9q5lMa8kjqctUrhwYeZFT4ygsnag==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-naked": "^3.0.2" @@ -5657,6 +5818,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/promise-make-naked/-/promise-make-naked-3.0.2.tgz", "integrity": "sha512-B+b+kQ1YrYS7zO7P7bQcoqqMUizP06BOyNSBEnB5VJKDSWo8fsVuDkfSmwdjF0JsRtaNh83so5MMFJ95soH5jg==", + "dev": true, "license": "MIT" }, "node_modules/pseudomap": { @@ -5743,14 +5905,6 @@ ], "license": "MIT" }, - "node_modules/randombytes": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -5775,11 +5929,13 @@ "license": "MIT" }, "node_modules/readdirp": { - "version": "4.1.2", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", + "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", "dev": true, "license": "MIT", "engines": { - "node": ">= 14.18.0" + "node": ">= 20.19.0" }, "funding": { "type": "individual", @@ -5944,16 +6100,6 @@ "node": ">= 0.10" } }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, "node_modules/resolve-url-loader": { "version": "5.0.0", "dev": true, @@ -6039,22 +6185,21 @@ "license": "MIT" }, "node_modules/sass": { - "version": "1.91.0", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.91.0.tgz", - "integrity": "sha512-aFOZHGf+ur+bp1bCHZ+u8otKGh77ZtmFyXDo4tlYvT7PWql41Kwd8wdkPqhhT+h2879IVblcHFglIMofsFd1EA==", + "version": "1.104.1", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.104.1.tgz", + "integrity": "sha512-yDA+1aIG3EHgN4V/BvuhCvu61FF6hEd4e+9DxikUm9U0CAGuvdIZ/UYy7qbOxjhbbWQraoyLlMuzdRGOSV5Bmw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "chokidar": "^4.0.0", - "immutable": "^5.0.2", + "chokidar": "^5.0.0", + "immutable": "^5.1.5", "source-map-js": ">=0.6.2 <2.0.0" }, "bin": { "sass": "sass.js" }, "engines": { - "node": ">=14.0.0" + "node": ">=20.19.0" }, "optionalDependencies": { "@parcel/watcher": "^2.4.1" @@ -6129,14 +6274,6 @@ "node": ">=10" } }, - "node_modules/serialize-javascript": { - "version": "6.0.2", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" - } - }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -6326,6 +6463,17 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -6415,14 +6563,16 @@ "node_modules/stubborn-fs": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", - "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" + "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==", + "dev": true }, "node_modules/style-mod": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.3.tgz", "integrity": "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/supports-color": { "version": "8.1.1", @@ -6454,11 +6604,17 @@ "link": true }, "node_modules/tapable": { - "version": "2.2.1", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", "dev": true, "license": "MIT", "engines": { "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" } }, "node_modules/term-size": { @@ -6552,12 +6708,14 @@ } }, "node_modules/terser": { - "version": "5.39.0", + "version": "5.51.2", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.51.2.tgz", + "integrity": "sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==", "dev": true, "license": "BSD-2-Clause", "dependencies": { "@jridgewell/source-map": "^0.3.3", - "acorn": "^8.8.2", + "acorn": "^8.15.0", "commander": "^2.20.0", "source-map-support": "~0.5.20" }, @@ -6569,14 +6727,15 @@ } }, "node_modules/terser-webpack-plugin": { - "version": "5.3.14", + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.6.1.tgz", + "integrity": "sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==", "dev": true, "license": "MIT", "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "jest-worker": "^27.4.5", "schema-utils": "^4.3.0", - "serialize-javascript": "^6.0.2", "terser": "^5.31.1" }, "engines": { @@ -6590,22 +6749,50 @@ "webpack": "^5.1.0" }, "peerDependenciesMeta": { + "@minify-html/node": { + "optional": true + }, "@swc/core": { "optional": true }, + "@swc/css": { + "optional": true + }, + "@swc/html": { + "optional": true + }, + "clean-css": { + "optional": true + }, + "cssnano": { + "optional": true + }, + "csso": { + "optional": true + }, "esbuild": { "optional": true }, + "html-minifier-terser": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "postcss": { + "optional": true + }, "uglify-js": { "optional": true } } }, "node_modules/terser-webpack-plugin/node_modules/ajv": { - "version": "8.17.1", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6617,8 +6804,28 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/terser-webpack-plugin/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/terser-webpack-plugin/node_modules/ajv-keywords": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "dev": true, "license": "MIT", "dependencies": { @@ -6628,32 +6835,23 @@ "ajv": "^8.8.2" } }, - "node_modules/terser-webpack-plugin/node_modules/jest-worker": { - "version": "27.5.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, "node_modules/terser-webpack-plugin/node_modules/json-schema-traverse": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, "node_modules/terser-webpack-plugin/node_modules/schema-utils": { - "version": "4.3.0", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "ajv-keywords": "^5.1.0" }, "engines": { @@ -6671,15 +6869,6 @@ "dev": true, "license": "MIT" }, - "node_modules/terser/node_modules/source-map-support": { - "version": "0.5.21", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "node_modules/test-client": { "resolved": "test-client", "link": true @@ -6720,6 +6909,7 @@ "version": "2.7.4", "resolved": "https://registry.npmjs.org/tiny-readdir/-/tiny-readdir-2.7.4.tgz", "integrity": "sha512-721U+zsYwDirjr8IM6jqpesD/McpZooeFi3Zc6mcjy1pse2C+v19eHPFRqz4chGXZFw7C3KITDjAtHETc2wj7Q==", + "dev": true, "license": "MIT", "dependencies": { "promise-make-counter": "^1.0.2" @@ -6795,22 +6985,41 @@ } }, "node_modules/ts-loader": { - "version": "9.5.2", + "version": "9.6.2", + "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.6.2.tgz", + "integrity": "sha512-R4iuczmtgxvtuI556s+hTZ6/7Ee03VCAk/l/M8LY1OAsUgB7YydsCxkgq9D9pKRaD7GJqUi2u8fp9zZP/ufjKA==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.0", - "enhanced-resolve": "^5.0.0", - "micromatch": "^4.0.0", - "semver": "^7.3.4", + "picomatch": "^4.0.0", "source-map": "^0.7.4" }, "engines": { "node": ">=12.0.0" }, "peerDependencies": { + "loader-utils": "*", "typescript": "*", - "webpack": "^5.0.0" + "webpack": "^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "loader-utils": { + "optional": true + } + } + }, + "node_modules/ts-loader/node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, "node_modules/ts-loader/node_modules/source-map": { @@ -6827,14 +7036,13 @@ "license": "0BSD" }, "node_modules/tsx": { - "version": "4.20.6", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", - "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", + "version": "4.23.13", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.13.tgz", + "integrity": "sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "~0.25.0", - "get-tsconfig": "^4.7.5" + "esbuild": "~0.28.0" }, "bin": { "tsx": "dist/cli.mjs" @@ -6861,7 +7069,6 @@ "version": "5.8.3", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6931,7 +7138,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.1.3", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", "dev": true, "funding": [ { @@ -7088,12 +7297,14 @@ "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/watcher": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/watcher/-/watcher-2.3.1.tgz", "integrity": "sha512-d3yl+ey35h05r5EFP0TafE2jsmQUJ9cc2aernRVyAkZiu0J3+3TbNugNcqdUJDoWOfL2p+bNsN427stsBC/HnA==", + "dev": true, "dependencies": { "dettle": "^1.0.2", "stubborn-fs": "^1.2.5", @@ -7101,11 +7312,12 @@ } }, "node_modules/watchpack": { - "version": "2.4.2", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.5.2.tgz", + "integrity": "sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==", "dev": true, "license": "MIT", "dependencies": { - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.1.2" }, "engines": { @@ -7113,35 +7325,29 @@ } }, "node_modules/webpack": { - "version": "5.99.9", + "version": "5.111.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.111.1.tgz", + "integrity": "sha512-cNypaz0RP+S4cvQVi1M/3bRUkvNP0xZpL0TjFx+c6jg64VbxD+2weWDgY9UnjRI6dhZgtaj8DU76GGFcJ79xPQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { - "@types/eslint-scope": "^3.7.7", - "@types/estree": "^1.0.6", + "@types/estree": "^1.0.8", "@types/json-schema": "^7.0.15", "@webassemblyjs/ast": "^1.14.1", "@webassemblyjs/wasm-edit": "^1.14.1", "@webassemblyjs/wasm-parser": "^1.14.1", - "acorn": "^8.14.0", - "browserslist": "^4.24.0", + "browserslist": "^4.28.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^5.17.1", - "es-module-lexer": "^1.2.1", - "eslint-scope": "5.1.1", + "enhanced-resolve": "^5.25.0", + "es-module-lexer": "^2.1.0", "events": "^3.2.0", - "glob-to-regexp": "^0.4.1", "graceful-fs": "^4.2.11", - "json-parse-even-better-errors": "^2.3.1", - "loader-runner": "^4.2.0", - "mime-types": "^2.1.27", - "neo-async": "^2.6.2", - "schema-utils": "^4.3.2", - "tapable": "^2.1.1", - "terser-webpack-plugin": "^5.3.11", - "watchpack": "^2.4.1", - "webpack-sources": "^3.2.3" + "mime-db": "^1.54.0", + "minimizer-webpack-plugin": "^5.7.0", + "schema-utils": "^4.5.0", + "tapable": "^2.3.0", + "watchpack": "^2.5.2", + "webpack-sources": "^3.5.1" }, "bin": { "webpack": "bin/webpack.js" @@ -7163,7 +7369,6 @@ "version": "6.0.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -7228,7 +7433,9 @@ } }, "node_modules/webpack-sources": { - "version": "3.2.3", + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.5.1.tgz", + "integrity": "sha512-jyuiGJdtvY434z5bUZrjz67v76/ePNvFZTp9Mdz29IlH4+GPsgyGjiv0fKI+M7BdkU6ADjulUcKAd3tUK3WlEw==", "dev": true, "license": "MIT", "engines": { @@ -7236,10 +7443,11 @@ } }, "node_modules/webpack/node_modules/ajv": { - "version": "8.17.1", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -7251,8 +7459,28 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/webpack/node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/webpack/node_modules/ajv-keywords": { "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", "dev": true, "license": "MIT", "dependencies": { @@ -7262,39 +7490,23 @@ "ajv": "^8.8.2" } }, - "node_modules/webpack/node_modules/eslint-scope": { - "version": "5.1.1", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/webpack/node_modules/estraverse": { - "version": "4.3.0", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, "node_modules/webpack/node_modules/json-schema-traverse": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, "node_modules/webpack/node_modules/schema-utils": { - "version": "4.3.2", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.5.0.tgz", + "integrity": "sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==", "dev": true, "license": "MIT", "dependencies": { - "@types/json-schema": "^7.0.9", - "ajv": "^8.9.0", - "ajv-formats": "^2.1.1", + "@types/json-schema": "^7.0.15", + "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "ajv-keywords": "^5.1.0" }, "engines": { @@ -7467,28 +7679,95 @@ "version": "0.14.0", "license": "MIT", "devDependencies": { - "@types/node": "^24.8.1", + "@types/node": "^25.0.2", "css-loader": "^7.1.2", "date-fns": "^4.1.0", "file-loader": "^6.2.0", - "fs-extra": "^11.3.0", - "mini-css-extract-plugin": "^2.9.2", - "obsidian": "1.10.2", - "reconcile-text": "^0.8.0", + "fs-extra": "^11.3.2", + "mini-css-extract-plugin": "^2.9.4", + "obsidian": "1.11.0", + "reconcile-text": "^0.11.0", "resolve-url-loader": "^5.0.0", - "sass": "^1.91.0", + "sass": "^1.96.0", "sass-loader": "^16.0.6", "sync-client": "file:../sync-client", - "terser-webpack-plugin": "^5.3.14", - "ts-loader": "^9.5.2", + "terser-webpack-plugin": "^5.3.16", + "ts-loader": "^9.5.4", "tslib": "2.8.1", - "tsx": "^4.20.6", - "typescript": "5.8.3", + "tsx": "^4.21.0", + "typescript": "5.9.3", "url": "^0.11.4", - "webpack": "^5.99.9", + "webpack": "^5.103.0", "webpack-cli": "^6.0.1" } }, + "obsidian-plugin/node_modules/@codemirror/view": { + "version": "6.38.6", + "resolved": "https://registry.npmjs.org/@codemirror/view/-/view-6.38.6.tgz", + "integrity": "sha512-qiS0z1bKs5WOvHIAC0Cybmv4AJSkAXgX5aD6Mqd2epSLlVJsQl8NG23jCVouIgkh4All/mrbdsf2UOLFnJw0tw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@codemirror/state": "^6.5.0", + "crelt": "^1.0.6", + "style-mod": "^4.1.0", + "w3c-keyname": "^2.2.4" + } + }, + "obsidian-plugin/node_modules/@types/node": { + "version": "25.9.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.8.tgz", + "integrity": "sha512-VfMrScDmMhUJQmd5hArdQnFvK0OIeD36uN2Va1FcpYaLB/BgkgM8Ulc50XtYISPMz7APJ30+N0T5EM0jlcdfRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "obsidian-plugin/node_modules/obsidian": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/obsidian/-/obsidian-1.11.0.tgz", + "integrity": "sha512-lVqN9AmDWHzhNATi2tDnjqVgI6WUYKeT+lIsAycAyLt4XCC6zRsWzb+tFCiB7Rn3PpttefjoovilhYwvS4Iqxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/codemirror": "5.60.8", + "moment": "2.29.4" + }, + "peerDependencies": { + "@codemirror/state": "6.5.0", + "@codemirror/view": "6.38.6" + } + }, + "obsidian-plugin/node_modules/reconcile-text": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/reconcile-text/-/reconcile-text-0.11.0.tgz", + "integrity": "sha512-a3sy3obazoc1BMEHx6IQn8ESZKnakVWZuRLi7OSEB56E8evRtrXBMj7yuo10fMoG4JkcZC6tokOfzpwZAKX+PQ==", + "dev": true, + "license": "MIT" + }, + "obsidian-plugin/node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "obsidian-plugin/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, "sync-client": { "version": "0.14.0", "devDependencies": { @@ -7496,7 +7775,6 @@ "@types/node": "^24.8.1", "byte-base64": "^1.1.0", "minimatch": "^10.0.1", - "p-queue": "^8.1.0", "reconcile-text": "^0.8.0", "ts-loader": "^9.5.2", "tslib": "2.8.1", diff --git a/frontend/package.json b/frontend/package.json index 0dd9057d..0bbf94af 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -18,6 +18,9 @@ "build": "npm run build --workspaces", "dev": "concurrently --kill-others \"npm run dev -w sync-client\" \"npm run dev -w obsidian-plugin\"", "test": "npm run test --workspaces", + "check:harness": "tsc --noEmit -p test-support/tsconfig.json && tsc --noEmit -p deterministic-tests/tsconfig.json && tsc --noEmit -p test-client/tsconfig.json", + "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/journal-interleavings.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", + "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/journal-protocol.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts'", "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md new file mode 100644 index 00000000..06d680ad --- /dev/null +++ b/frontend/sync-client/README.md @@ -0,0 +1,137 @@ +# Sync client (API v5) + +The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket +messages wake it; complete disk scans and the durable server event log are the +sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a +positive interval to repair missed notifications. Unset or `0` disables polling. +`syncLocallyCreatedFile`, `syncLocallyUpdatedFile`, and `syncLocallyDeletedFile` +durably record logical file changes before resolving, including while offline. +Shutdown waits for those saves. Notifications describe logical external changes: adapters suppress +self-generated create/delete/move notifications, and report atomic editor saves +as content updates. This distinguishes a deliberate delete/recreate from an edit. +Use `waitUntilFinished()` to await the active attempt; errors leave +pending work durable and are reported to the caller. Transient failures retry; +authentication and protocol errors wait for an explicit wake or settings change. + +## State and policy + +The persisted state contains the incorporated server file manifest, the current +local identity map, and the exact pending submission. A received server file +manifest is not a merge base until its local application is durable. Successful +submissions advance the base to the submitted content snapshot, preserving later +local edits. Unknown outcomes are retried unchanged before incorporating newer +remote state. Each contiguous event batch is folded into its final manifest and +content heads before merging. Intermediate changes that the server has already +reverted must not overwrite local edits. + +A permanent upload rejection is recorded locally, with content retained for +recovery. An unchanged rejected payload is skipped so other files can sync; the +failure remains visible in history and `waitUntilFinished()`. Editing it, resetting +the client, or changing settings permits a fresh attempt. Earlier attempts that +committed despite a lost reply are incorporated through normal event replay. + +For each document ID (absence means deletion), use the server path if local is +unchanged, use local if the server is unchanged, and use the server when both +changed. Then resolve cross-document path conflicts: a retained server path wins, +UUID ordering breaks other ties, and displaced documents receive deterministic +`(conflict UUID)` names. This also handles file/ancestor and portable-name aliases. +Nonportable local names are repaired client-side before a file manifest is published. +Allocated filename components fit within 255 UTF-8 bytes, including conflict suffixes. +The protocol does not impose a universal total-path byte limit; filesystem-specific +limits surface from the adapter instead of silently flattening a name. Directories +are implicit; empty directories are not synced. + +- Content is uploaded before new membership; the client chooses and persists UUIDs. +- Text uses three-way `reconcile`. First contact at a matching path adopts the + remote ID and reconciles differing text with an empty parent. +- Concurrent binary/unmergeable edits keep the server version and retain + displaced local bytes in recovery. If only one side changed, keep that side. +- Deletion follows the file manifest decision. Displaced edited bytes are retained + locally, without automatically restoring membership. +- Notified moves preserve IDs. Offline inference requires a unique nonempty hash + match on both sides; uncertain cases become delete/create. +- Ignored and oversized files preserve their disk paths and remote membership. + The original manifest base is retained while excluded, so becoming eligible + resumes reconciliation without losing local renames. Not-yet-downloaded files + are not mistaken for local deletions. +- Bytes and line endings are preserved. File modification times are not tracked + or synchronized. Optional editor cursors are repositioned when text is reconciled. + +## Restored server histories + +Successful HTTP responses carry an `X-Vault-Link-History` checkpoint. The client +persists it before consuming the response and supplies it on subsequent reads and +writes. WebSocket events wake HTTP replay, which validates that checkpoint. A +restored database cannot silently reuse a version number: a missing or different +event incarnation returns a history mismatch before executing the request. + +On mismatch the client first recovers its filesystem journal and scans local +work. It durably records a reset before clearing the old checkpoint, archives +pending content requests, and clears old receipts, cursors, bases and diff-cache +entries. Clean files reconcile against the restored snapshot. Unsent edits with +an old base become separate local documents; path conflicts preserve them under +conflict names instead of applying a diff to an unrelated restored version. +Recovery resumes after interrupted state saves. Existing API v4 client state +without a checkpoint takes this same conservative bootstrap on upgrade. + +Status observers cannot stop the background engine by throwing or rejecting. +Their failures are reported, and remaining observers continue receiving updates. + +## Required adapter contract + +`FileSystemOperations` and `PersistenceProvider` define the adapter operations +needed by the recovery model. The Obsidian and CLI adapters have intentionally +not been updated in this rewrite. + +The sync client does not coordinate separate processes accessing the same local +vault. + +The filesystem adapter must provide: + +- Complete scans and coherent content snapshots. Missing and unreadable must + be distinguished; reject read races rather than return a partial snapshot. +- Vault-relative operations that reject symlinks, hardlinks, special files, and + ancestor symlink traversal. Staging and visible files share one filesystem. +- Atomic, complete, exclusive file creation (a crash leaves absent or complete + bytes, never a partial file) and atomic renames that **never replace** destinations. + Writes apply optional cursor metadata. +- `flushPaths` flushes existing files and ancestor directories, including parents + of absent paths. Recovery uses it to complete interrupted rename durability. +- Durable recursive directory creation, and directory pruning using only `rmdir` + operations (never unlink descendant files). All affected data/directory entries + must be flushed before a mutation resolves, including intermediate ancestors. + +Keep the persistence store outside the user file namespace or inside the reserved +`.vault-link-sync` directory, so writing sync state cannot generate user edits. + +Persistence `save` atomically replaces the complete state and resolves only after +its data and directory entry are durable. A failed save may leave either complete +value; the engine reloads before retrying. Settings saves are assumed to succeed +and do not reload after errors. Settings and engine saves are serialized. +Changing a client's vault once it holds identities, offline notifications, +pending requests, or bootstrap/application state requires a separate state store +and client. Lifecycle operations are serialized. + +Before changing visible files, the engine persists an application journal. It +stages every affected source before installing destinations, so swaps and cycles +work. Recovery runs before scanning, including while the server is unavailable. +Prepared content is persisted before installation, preventing repeated merges +following a crash. Unexpected occupants are preserved and given conflict names. +Notifications received while staging rebind the actual source identity before +content is merged. Editor saves to a temporarily staged path continue the original +document's journal, including saves racing preparation and installation. Ignored +or oversized occupants are reserved before conflict allocation. + +`.vault-link-sync/transactions` contains only active recovery artifacts. Staged +sources and prepared output are removed after every install is durably recorded; +completed journals are not archived. `.vault-link-sync/recovery` retains displaced +original bytes and their document/path metadata before staging files are deleted. +`.vault-link-sync/requests` retains rejected submission snapshots for explicit +recovery. These directories are never scanned +as user files. Pending requests and active recovery journals must not be +independently discarded. + +These guarantees assume the adapters/storage honor their durability contracts. +Arbitrary external writers can change a file between observations; detected races +are preserved/reconciled, but unobserved writes and offline identity cannot be +reconstructed perfectly. There is no automated v2/v3 persisted-state migration. diff --git a/frontend/sync-client/package.json b/frontend/sync-client/package.json index aa369fa7..bdc094cb 100644 --- a/frontend/sync-client/package.json +++ b/frontend/sync-client/package.json @@ -15,7 +15,6 @@ "devDependencies": { "byte-base64": "^1.1.0", "minimatch": "^10.0.1", - "p-queue": "^8.1.0", "reconcile-text": "^0.8.0", "uuid": "^13.0.0", "@types/node": "^24.8.1", diff --git a/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts b/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts new file mode 100644 index 00000000..22ed4c8e --- /dev/null +++ b/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts @@ -0,0 +1,428 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../persistence/database"; +import type { ServerConfig } from "../services/server-config"; +import { toStoredSnapshot } from "../sync-operations/content"; +import { isInternalPath } from "../utils/portable-path"; +import { Logger } from "../tracing/logger"; +import { FileOperations, type FileWrite } from "./file-operations"; +import type { + FileSnapshot, + FileSystemOperations +} from "./filesystem-operations"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const text = (snapshot: FileSnapshot) => + new TextDecoder().decode(snapshot.content); +const parent = (path: string) => path.split("/").slice(0, -1).join("/"); + +/** Atomic mutations with an injectable interruption before or after durability. */ +class MemoryFiles implements FileSystemOperations { + public readonly files = new Map(); + public readonly directories = new Set([""]); + public boundary: (label: string) => void = () => {}; + + public change(label: string, action: () => void): void { + this.boundary(`before ${label}`); + action(); + this.boundary(`after ${label}`); + } + + public put(path: string, value: string): void { + for ( + let directory = parent(path); + directory; + directory = parent(directory) + ) + this.directories.add(directory); + this.files.set(path, { content: bytes(value) }); + } + + public async listFilesRecursively(root = ""): Promise { + assert(this.directories.has(root), `Cannot list non-directory ${root}`); + return [...this.files.keys()].filter( + (path) => !root || path.startsWith(`${root}/`) + ); + } + + public async readSnapshot(path: string): Promise { + assert(!this.directories.has(path), `Cannot read directory ${path}`); + return structuredClone(this.files.get(path)); + } + + public async stat(path: string) { + for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor)) + assert( + !this.files.has(ancestor), + `Cannot traverse file ${ancestor}` + ); + const snapshot = this.files.get(path); + if (snapshot) + return { kind: "file" as const, size: snapshot.content.length }; + if (this.directories.has(path)) + return { kind: "directory" as const, size: 0 }; + return undefined; + } + + public async exists(path: string): Promise { + return this.files.has(path) || this.directories.has(path); + } + + public async createDirectory(path: string): Promise { + this.change(`mkdir ${path}`, () => { + for ( + let directory = path; + directory; + directory = parent(directory) + ) { + assert( + !this.files.has(directory), + `File blocks directory ${directory}` + ); + this.directories.add(directory); + } + }); + } + + public async write(path: string, snapshot: FileSnapshot): Promise { + this.change(`write ${path}`, () => { + assert( + !this.files.has(path) && !this.directories.has(path), + `Destination exists: ${path}` + ); + assert( + this.directories.has(parent(path)), + `Parent missing: ${path}` + ); + this.files.set(path, structuredClone(snapshot)); + }); + } + + public async rename(from: string, to: string): Promise { + this.change(`rename ${from} -> ${to}`, () => { + assert(this.files.has(from), `Source missing: ${from}`); + assert( + !this.files.has(to) && !this.directories.has(to), + `Destination exists: ${to}` + ); + assert(this.directories.has(parent(to)), `Parent missing: ${to}`); + this.files.set(to, this.files.get(from)!); + this.files.delete(from); + }); + } + + public async flushPaths(paths: readonly string[]): Promise { + this.change(`flush ${paths.join(",")}`, () => {}); + } + + public async delete(path: string): Promise { + this.change(`rmdir ${path}`, () => { + assert(!this.files.has(path)); + assert( + ![...this.files.keys()].some((file) => + file.startsWith(`${path}/`) + ) + ); + for (const directory of this.directories) + if (directory === path || directory.startsWith(`${path}/`)) + this.directories.delete(directory); + }); + } + + public async deleteFile(path: string): Promise { + this.change(`unlink ${path}`, () => { + assert(!this.directories.has(path)); + this.files.delete(path); + }); + } +} + +async function setup( + contents: Record, + local: Record +) { + const fs = new MemoryFiles(); + for (const [path, value] of Object.entries(contents)) fs.put(path, value); + let durable: StoredDatabase = emptyState("test"); + durable.local = local; + for (const [id, path] of Object.entries(local)) + durable.documents[id] = { + materialized: true, + observedHash: (await toStoredSnapshot(fs.files.get(path)!)).hash + }; + const config = { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig; + const restart = () => { + const database = new Database( + new Logger(), + structuredClone(durable), + async (next) => { + fs.change("save", () => { + durable = structuredClone(next); + }); + }, + "test", + async () => structuredClone(durable) + ); + return { + database, + operations: new FileOperations(fs, database, config) + }; + }; + return { fs, restart, ...restart() }; +} + +async function scenario() { + const fixture = await setup( + { + "a.md": "A", + "b.md": "B", + "c.md": "C", + "d.md": "Unmanaged", + "e.md": "E", + "folder/keep.md": "Keep", + "deleted.md": "Deleted" + }, + { a: "a.md", b: "b.md", c: "c.md", e: "e.md", deleted: "deleted.md" } + ); + const next = structuredClone(fixture.database.state); + next.local = { + a: "b.md", + b: "a.md", + c: "d.md", + e: "folder", + added: "new.md" + }; + next.documents.added = { materialized: false }; + const writes: Record = { + a: { + expected: await fixture.operations.snapshot("a.md"), + replacement: await toStoredSnapshot({ content: bytes("A updated") }) + }, + added: { + replacement: await toStoredSnapshot({ + content: bytes("Downloaded") + }) + } + }; + return { ...fixture, next, writes }; +} + +async function verifyScenario(fs: MemoryFiles, database: Database) { + assert.equal(database.state.application, undefined); + for (const [id, expected] of Object.entries({ + a: "A updated", + b: "B", + c: "C", + e: "E", + added: "Downloaded" + })) { + const snapshot = fs.files.get(database.state.local[id])!; + assert.equal(text(snapshot), expected, id); + assert.equal(database.state.documents[id].materialized, true, id); + assert.equal( + database.state.documents[id].observedHash, + (await toStoredSnapshot(snapshot)).hash, + id + ); + } + assert.equal(database.state.local.a, "b.md"); + assert.equal(database.state.local.b, "a.md"); + assert.equal(database.state.local.c, "d.md"); + assert.notEqual(database.state.local.e, "folder"); + assert.equal(text(fs.files.get("folder/keep.md")!), "Keep"); + assert.equal(fs.files.has("deleted.md"), false); + assert.equal(database.state.local.deleted, undefined); + const visible = [...fs.files].filter(([path]) => !isInternalPath(path)); + assert.deepEqual( + visible.map(([, value]) => text(value)).sort(), + ["A updated", "B", "C", "Downloaded", "E", "Keep", "Unmanaged"].sort() + ); + assert.equal( + [...fs.files.keys()].some((path) => + path.startsWith(".vault-link-sync/transactions/") + ), + false + ); +} + +describe("filesystem journal recovery", () => { + it("stages swaps, preserves unexpected files/directories, and cleans recovery artifacts", async () => { + const { operations, database, fs, next, writes } = await scenario(); + await operations.apply(next, writes); + await verifyScenario(fs, database); + fs.boundary = () => { + throw new Error("Completed recovery must not write"); + }; + await operations.recover(); + }); + + it("resumes after interruption at every filesystem and metadata mutation boundary", async (t) => { + const baseline = await scenario(); + const boundaries: string[] = []; + baseline.fs.boundary = (label) => boundaries.push(label); + await baseline.operations.apply(baseline.next, baseline.writes); + for (const [index, label] of boundaries.entries()) { + await t.test(`${index}: ${label}`, async () => { + const fixture = await scenario(); + const interrupted = new Error("Interrupted"); + let count = 0; + fixture.fs.boundary = () => { + if (count++ === index) throw interrupted; + }; + await assert.rejects( + fixture.operations.apply(fixture.next, fixture.writes), + (error) => error === interrupted + ); + fixture.fs.boundary = () => {}; + const { database, operations } = fixture.restart(); + if ( + !database.state.application && + database.state.local.a === "a.md" + ) + await operations.apply( + structuredClone(fixture.next), + fixture.writes + ); + else await operations.recover(); + await verifyScenario(fixture.fs, database); + }); + } + }); + + it("merges an edit made after planning exactly once across an interrupted install", async () => { + const base = "First paragraph.\n\nLast paragraph.\n"; + const local = "Local first paragraph.\n\nLast paragraph.\n"; + const remote = "First paragraph.\n\nRemote last paragraph.\n"; + const fixture = await setup({ "note.md": base }, { note: "note.md" }); + const expected = await fixture.operations.snapshot("note.md"); + let edited = false; + const interrupted = new Error("Interrupted after install"); + fixture.fs.boundary = (label) => { + if (!edited && label === "after save") { + fixture.fs.put("note.md", local); + edited = true; + } + if ( + label.startsWith("after rename ") && + label.endsWith(" -> note.md") + ) + throw interrupted; + }; + await assert.rejects( + fixture.operations.apply(structuredClone(fixture.database.state), { + note: { + expected, + replacement: await toStoredSnapshot({ + content: bytes(remote) + }) + } + }), + (error) => error === interrupted + ); + fixture.fs.boundary = () => {}; + const { operations, database } = fixture.restart(); + await operations.recover(); + assert.equal( + text(fixture.fs.files.get("note.md")!), + "Local first paragraph.\n\nRemote last paragraph.\n" + ); + assert.equal( + database.state.documents.note.observedHash, + (await operations.snapshot("note.md"))!.hash + ); + assert.equal( + [...fixture.fs.files.keys()].some((path) => + path.startsWith(".vault-link-sync/transactions/") + ), + false + ); + }); + + it("preserves an external deletion between planning and staging", async () => { + const fixture = await setup( + { "note.md": "Local" }, + { note: "note.md" } + ); + const expected = await fixture.operations.snapshot("note.md"); + fixture.fs.boundary = (label) => { + if (label === "after save") fixture.fs.files.delete("note.md"); + }; + await fixture.operations.apply( + structuredClone(fixture.database.state), + { + note: { + expected, + replacement: await toStoredSnapshot({ + content: bytes("Remote") + }) + } + } + ); + assert.equal(fixture.database.state.local.note, undefined); + assert.equal(fixture.fs.files.has("note.md"), false); + }); + + it("preserves a file created after the destination check", async () => { + const fixture = await setup( + { "source.md": "Incoming" }, + { doc: "source.md" } + ); + const next = structuredClone(fixture.database.state); + next.local.doc = "target.md"; + fixture.fs.boundary = (label) => { + if ( + label.startsWith("before rename ") && + label.endsWith(" -> target.md") + ) + fixture.fs.put("target.md", "Concurrent"); + }; + await assert.rejects( + fixture.operations.apply(next), + /Destination exists/ + ); + fixture.fs.boundary = () => {}; + const { operations, database } = fixture.restart(); + await operations.recover(); + assert.equal(text(fixture.fs.files.get("target.md")!), "Incoming"); + const displaced = Object.entries(database.state.local).find( + ([id]) => id !== "doc" + )!; + assert.equal(text(fixture.fs.files.get(displaced[1])!), "Concurrent"); + }); + + it("preserves an unexpected file blocking a destination's ancestor", async () => { + const fixture = await setup( + { "source.md": "Incoming", target: "Occupant" }, + { doc: "source.md" } + ); + const next = structuredClone(fixture.database.state); + next.local.doc = "target/child.md"; + await fixture.operations.apply(next); + assert.equal( + text(fixture.fs.files.get("target/child.md")!), + "Incoming" + ); + const displaced = Object.entries(fixture.database.state.local).find( + ([id]) => id !== "doc" + )!; + assert.equal(text(fixture.fs.files.get(displaced[1])!), "Occupant"); + }); + + it("handles paths becoming directories and directories becoming files", async () => { + const fixture = await setup( + { a: "A", "b/deep/child": "B" }, + { a: "a", b: "b/deep/child" } + ); + const next = structuredClone(fixture.database.state); + next.local = { a: "b", b: "a/deep/child" }; + await fixture.operations.apply(next); + assert.equal(text(fixture.fs.files.get("b")!), "A"); + assert.equal(text(fixture.fs.files.get("a/deep/child")!), "B"); + }); +}); diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 2864bd20..81d1a0f6 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,291 +1,916 @@ -import type { Logger } from "../tracing/logger"; -import type { FileSystemOperations } from "./filesystem-operations"; -import type { Database, RelativePath } from "../persistence/database"; -import { SafeFileSystemOperations } from "./safe-filesystem-operations"; -import type { TextWithCursors } from "reconcile-text"; -import { reconcile } from "reconcile-text"; -import { isFileTypeMergable } from "../utils/is-file-type-mergable"; -import { isBinary } from "../utils/is-binary"; +import { v4 as uuid } from "uuid"; +import type { + FileSnapshot, + FileSystemOperations +} from "./filesystem-operations"; +import type { + ApplicationJournal, + Database, + EngineState, + FileStep, + StoredSnapshot +} from "../persistence/database"; +import { + allocatePortablePath, + INTERNAL_DIRECTORY, + isInternalPath +} from "../utils/portable-path"; +import { + mergeContent, + fromStoredSnapshot, + toStoredSnapshot +} from "../sync-operations/content"; import type { ServerConfig } from "../services/server-config"; +import { FileNotFoundError } from "../errors/errors"; +import { + applyLocalChange, + unappliedChanges, + type LocalChange +} from "../sync-operations/local-changes"; +import { resolvePaths } from "../sync-operations/file-manifest"; +export interface FileWrite { + expected?: StoredSnapshot; + replacement: StoredSnapshot; +} + +/** The persisted journal bridges the filesystem and the metadata store. Every + * source is staged before any destination is installed. A phase is saved BEFORE + * an ambiguous rename, so recovery never reapplies a content merge to its output. + */ export class FileOperations { - private static readonly PARENTHESES_REGEX = / \((?\d+)\)$/; - private readonly fs: SafeFileSystemOperations; - public constructor( - private readonly logger: Logger, + public readonly fs: FileSystemOperations, private readonly database: Database, - fs: FileSystemOperations, private readonly serverConfig: ServerConfig, - private readonly nativeLineEndings = "\n" - ) { - this.fs = new SafeFileSystemOperations(fs, logger); + private readonly filesApplied: ( + paths: readonly string[] + ) => void = () => {}, + private readonly localChanges: { + entries: () => readonly LocalChange[]; + flush: () => Promise; + } = { entries: () => [], flush: async () => {} }, + private readonly protectedFile: ( + path: string, + size: number + ) => boolean = () => false + ) {} + + public async read(path: string): Promise { + const snapshot = await this.fs.readSnapshot(path); + if (!snapshot) + throw new FileNotFoundError(`File not found: ${path}`, path); + return snapshot.content; } - private static getParentDirAndFile( - path: RelativePath - ): [RelativePath, RelativePath] { - const pathParts = path.split("/"); - const fileName = pathParts.pop(); - if (fileName == null || fileName === "") { - throw new Error(`Path '${path}' cannot be empty`); - } - - return [pathParts.join("/"), fileName]; + public async listFilesRecursively(): Promise { + return (await this.fs.listFilesRecursively()).filter( + (path) => !isInternalPath(path) + ); } - public async listFilesRecursively( - root: RelativePath | undefined = undefined - ): Promise { - return this.fs.listFilesRecursively(root); + public async snapshot(path: string): Promise { + const value = await this.fs.readSnapshot(path); + return value && toStoredSnapshot(value); } - public async read(path: RelativePath): Promise { - return this.fromNativeLineEndings(await this.fs.read(path)); - } - - /** - * Create a file at the specified path. - * - * If a file with the same name already exists, it is moved before creating the new one. - * Parent directories are created if necessary. - */ - public async create( - path: RelativePath, - newContent: Uint8Array - ): Promise { - await this.ensureClearPath(path); - return this.fs.write(path, this.toNativeLineEndings(newContent)); - } - - public async ensureClearPath(path: RelativePath): Promise { - if (await this.fs.exists(path)) { - const deconflictedPath = await this.deconflictPath(path); - try { - this.logger.debug( - `Didn't expect ${path} to exist, deconflicting by moving it to '${deconflictedPath}'` - ); - - this.database.move(path, deconflictedPath); - await this.fs.rename(path, deconflictedPath, true); - } finally { - this.fs.unlock(deconflictedPath); + /** Project journal phases onto the physical namespace before binding an + * editor notification. Content hashes must never choose a notified identity. */ + private async physicalState( + journal = this.database.state.application + ): Promise { + const state = structuredClone(journal?.next ?? this.database.state); + if (journal) { + for (const step of journal.steps) { + if (step.superseded) continue; + const origin = step.replacement ? step.output : step.staged; + const path = + step.phase === "installed" + ? step.to + : step.phase === "installing" && + !(await this.fs.exists(origin)) + ? step.to + : (await this.fs.exists(step.staged)) + ? step.staged + : step.from; + if (path) state.local[step.documentId] = path; + else delete state.local[step.documentId]; } - } else { - await this.createParentDirectories(path); } + return state; } - /** - * Update the file at the given path. - * - * Performs a 3-way merge before writing if the file's content differs from `expectedContent`. - * Does not recreate the file if it no longer exists, returning an empty array instead. - */ - public async write( - path: RelativePath, - expectedContent: Uint8Array, - newContent: Uint8Array + public async captureChange( + change: LocalChange, + previous: readonly LocalChange[] ): Promise { - if (!(await this.fs.exists(path))) { - this.logger.debug( - `The caller assumed ${path} exists, but it no longer, so we wont recreate it` - ); - return; - } - + const state = await this.physicalState(); + // No notifications may bind provisional identities before the initial + // snapshot is available. Replay the whole ordered queue at bootstrap, + // including moves/deletions after creates. + if (!state.initialized) return; + for (const pending of unappliedChanges(state, previous)) + applyLocalChange(state, pending, () => false); if ( - !isFileTypeMergable( - path, - (await this.serverConfig.getConfig()).mergeableFileExtensions - ) || - isBinary(expectedContent) || - isBinary(newContent) + change.type === "update" && + !Object.values(state.local).includes(change.path) ) { - this.logger.debug( - `The expected content is not mergable, so we won't perform a 3-way merge, just overwrite it` + // An editor may save to the logical path while its original is in + // staging. Namespace creates/deletes still bind to physical paths. + const step = this.database.state.application?.steps.find( + (step) => + step.from === change.path && step.phase !== "installed" ); - await this.fs.write( - path, - // `newContent` might not be binary so we still have to ensure the line endings are correct - this.toNativeLineEndings(newContent) + if (step && isInternalPath(state.local[step.documentId] ?? "")) + state.local[step.documentId] = change.path; + } + applyLocalChange(state, change, () => false); + } + + private async incorporateLocalChanges( + journal: ApplicationJournal + ): Promise { + await this.localChanges.flush(); + const changes = unappliedChanges( + journal.next, + this.localChanges.entries() + ); + if (!changes.length) return; + const physical = await this.physicalState(journal); + for (const change of changes) { + if (change.type === "update") { + const info = await this.fs.stat(change.path); + for (const id of Object.keys(change.identities ?? {})) { + // A save immediately before the rename has already moved + // to staging by the time its notification is incorporated. + const stagedPath = physical.local[id]; + const saved = + info ?? + (stagedPath && isInternalPath(stagedPath) + ? await this.fs.stat(stagedPath) + : undefined); + if ( + saved?.kind === "file" && + this.protectPath( + journal, + physical, + id, + change.path, + saved.size + ) + ) + journal.next.local[id] = change.path; + } + await this.incorporateContentSave(journal, physical, change); + journal.next.excluded = physical.excluded; + journal.next.lastAppliedLocalChangeId = change.changeId; + continue; + } + const before = { ...physical.local }; + const path = + change.type === "move" ? change.relativePath : change.path; + const info = await this.fs.stat(path); + if (info?.kind === "file" && this.protectedFile(path, info.size)) + journal.next.protectedPaths = [ + ...new Set([...(journal.next.protectedPaths ?? []), path]) + ]; + applyLocalChange( + physical, + change, + (path) => + isInternalPath(path) || + this.protectedFile(path, 0) || + journal.next.protectedPaths?.includes(path) === true || + Object.keys(journal.next.excluded ?? {}).some( + (id) => before[id] === path + ) ); - return; - } - - const expectedText = new TextDecoder().decode(expectedContent); // this comes from a previous read which must only have \n line endings - const newText = new TextDecoder().decode(newContent); // this comes from the server which stores text with \n line endings - - await this.fs.atomicUpdateText( - path, - ({ text, cursors }: TextWithCursors): TextWithCursors => { - this.logger.debug( - `Performing a 3-way merge for ${path} with the expected content` - ); - - text = text.replaceAll(this.nativeLineEndings, "\n"); - const merged = reconcile( - expectedText, - { text, cursors }, - newText - ); - - const resultText = merged.text.replaceAll( - "\n", - this.nativeLineEndings - ); - - return { - text: resultText, - cursors: merged.cursors - }; + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(physical.local) + ])) { + if (before[id] === physical.local[id]) continue; + if (physical.local[id] === undefined) + delete journal.next.local[id]; + else { + const path = physical.local[id]; + journal.next.local[id] = path; + const info = await this.fs.stat(path); + this.protectPath( + journal, + physical, + id, + path, + info?.size ?? 0 + ); + } + journal.next.documents[id] ??= physical.documents[id]; } + journal.next.excluded = physical.excluded; + journal.next.lastAppliedLocalChangeId = change.changeId; + } + journal.next.local = resolvePaths( + journal.next.local, + journal.next.fileManifest.entries, + Object.keys(journal.next.excluded ?? {}), + journal.next.protectedPaths ); - } - - public async delete(path: RelativePath): Promise { - if (await this.exists(path)) { - await this.fs.delete(path); - await this.deletingEmptyParentDirectoriesOfDeletedFile(path); - } else { - this.logger.debug(`No need to delete '${path}', it doesn't exist`); - } - } - - public async getFileSize(path: RelativePath): Promise { - return this.fs.getFileSize(path); - } - - public async exists(path: RelativePath): Promise { - return this.fs.exists(path); - } - - public async move( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - if (oldPath === newPath) { - return; - } - - await this.ensureClearPath(newPath); - - this.database.move(oldPath, newPath); - await this.fs.rename(oldPath, newPath); - await this.deletingEmptyParentDirectoriesOfDeletedFile(oldPath); - } - - public reset(): void { - this.fs.reset(); - } - - private async deletingEmptyParentDirectoriesOfDeletedFile( - path: RelativePath - ): Promise { - let directory = path; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - [directory] = FileOperations.getParentDirAndFile(directory); - if (directory.length === 0) { - break; - } - - const remainingContent = - await this.fs.listFilesRecursively(directory); - if (remainingContent.length === 0) { - this.logger.debug( - `Folder (${directory}) is now empty, deleting` - ); - await this.fs.delete(directory); - } else { - break; - } - } - } - - private fromNativeLineEndings(content: Uint8Array): Uint8Array { - if (isBinary(content)) { - return content; - } - - const decoder = new TextDecoder("utf-8"); - let text = decoder.decode(content); - text = text.replaceAll(this.nativeLineEndings, "\n"); - return new TextEncoder().encode(text); - } - - private toNativeLineEndings(content: Uint8Array): Uint8Array { - if (isBinary(content)) { - return content; - } - - const decoder = new TextDecoder("utf-8"); - let text = decoder.decode(content); - text = text.replaceAll("\n", this.nativeLineEndings); - return new TextEncoder().encode(text); - } - - private async createParentDirectories(path: string): Promise { - const components = path.split("/"); - if (components.length === 1) { - return; - } - for (let i = 1; i < components.length; i++) { - const parentDir = components.slice(0, i).join("/"); - if (!(await this.fs.exists(parentDir))) { - await this.fs.createDirectory(parentDir); - } - } - } - - /** - * Deconflicts the given path by appending (1), (2), etc. before the file extension until a non-existent path is found. - * The returned path has a lock acquired on it; it must be released by the caller when no longer needed. - * - * @param path The starting path to deconflict - * @returns a non-existent path with a lock acquired on it - */ - private async deconflictPath(path: RelativePath): Promise { - // eslint-disable-next-line prefer-const - let [directory, fileName] = FileOperations.getParentDirAndFile(path); - - if (directory) { - directory += "/"; - } - - const nameParts = fileName.split("."); - // Handle dotfiles: ".gitignore" should have no extension, ".config.json" should have ".json" - const isDotfile = fileName.startsWith(".") && nameParts[0] === ""; - const extension = - nameParts.length > 1 && !(isDotfile && nameParts.length === 2) - ? "." + nameParts[nameParts.length - 1] - : ""; - let stem = extension ? nameParts.slice(0, -1).join(".") : fileName; - let currentCount = Number.parseInt( - FileOperations.PARENTHESES_REGEX.exec(stem)?.groups?.count ?? "0" - ); - stem = stem.replace(FileOperations.PARENTHESES_REGEX, ""); - - let newName = path; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - currentCount++; - newName = `${directory}${stem} (${currentCount})${extension}`; - - // Avoid multiple deconflictPath calls returning the same path - if (this.fs.tryLock(newName)) { - const newDocument = - this.database.getLatestDocumentByRelativePath(newName); + for (const step of journal.steps) { + if (step.superseded) continue; + // A notified move can exclude an input after this transaction was + // planned. Preserve its bytes as well as its chosen physical path. + if (journal.next.excluded?.[step.documentId] !== undefined) { + const origin = step.replacement ? step.output : step.staged; if ( - newDocument?.isDeleted === false || // the document might have been confirmed by the server at a new path but haven't yet moved there locally - (await this.fs.exists(newName, true)) + step.phase === "installing" && + !(await this.fs.exists(origin)) + ) + step.phase = "installed"; + if (step.replacement && step.phase !== "installed") { + // contentWrite advanced the proposed merge context before + // applying bytes. Cancelling that application must retain + // the last incorporated base so reenabling the file still + // reconciles the deferred remote head. An installed output, + // including one whose acknowledgement was lost, keeps its + // new base for subsequent local edits. + const previous = + this.database.state.documents[step.documentId]; + Object.assign( + journal.next.documents[step.documentId], + structuredClone({ + base: previous?.base, + bootstrap: previous?.bootstrap, + recoveryBase: previous?.recoveryBase + }) + ); + } + step.replacement = undefined; + } + if (step.phase === "installed") { + step.to = physical.local[step.documentId]; + continue; + } + if (step.phase === "planned") { + const staged = await this.fs.exists(step.staged); + const from = physical.local[step.documentId]; + if (staged && from !== step.staged) { + // A notified move/delete and path reuse raced the source + // rename. Bind the staged object to the new occupant, then + // stage the original identity from its notified location. + const occupant = Object.keys(physical.local).find( + (id) => physical.local[id] === step.from + ); + const originalStaged = step.staged; + step.staged = `${step.staged}-${uuid()}`; + if (occupant) { + journal.steps.push({ + documentId: occupant, + from: step.from, + to: journal.next.local[occupant], + staged: originalStaged, + output: `${originalStaged}.output`, + phase: "staged" + }); + physical.local[occupant] = originalStaged; + } else { + // Retain an ambiguous object without merging it into a + // different document. Cleanup archives this source. + journal.steps.push({ + ...step, + staged: originalStaged, + to: undefined, + replacement: undefined, + phase: "installed", + superseded: true + }); + } + } + if ( + !staged || + (from !== undefined && !isInternalPath(from)) || + from === undefined ) { - this.fs.unlock(newName); - } else { - return newName; + if (step.from && !from) step.replacement = undefined; + step.from = from; } } + step.to = journal.next.local[step.documentId]; + } + for (const [id, from] of Object.entries(physical.local)) { + const to = journal.next.local[id]; + if ( + from === to || + isInternalPath(from) || + journal.steps.some( + (step) => + step.documentId === id && step.phase !== "installed" + ) + ) + continue; + const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${id}-${uuid()}`; + journal.steps.push({ + documentId: id, + from, + to, + expected: await this.snapshot(from), + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + } + await this.persist(journal); + } + + private protectPath( + journal: ApplicationJournal, + physical: EngineState, + id: string, + path: string, + size: number + ): boolean { + if ( + !this.protectedFile(path, size) && + !journal.next.protectedPaths?.includes(path) + ) + return false; + physical.excluded ??= {}; + physical.excluded[id] ??= + this.database.state.fileManifest.entries[id] ?? null; + journal.next.protectedPaths = [ + ...new Set([...(journal.next.protectedPaths ?? []), path]) + ]; + return true; + } + + private async incorporateContentSave( + journal: ApplicationJournal, + physical: EngineState, + change: LocalChange & { type: "update" } + ): Promise { + for (const id of Object.keys(change.identities ?? {})) { + const step = journal.steps.find( + (step) => + step.documentId === id && + step.phase !== "installed" && + !step.superseded + ); + if ( + !step || + !(await this.fs.exists(step.staged)) || + (await this.fs.stat(change.path))?.kind !== "file" + ) + continue; + const moved = + step.phase === "installing" && + !(await this.fs.exists( + step.replacement ? step.output : step.staged + )); + if (moved && step.to === change.path) continue; + // Stage the editor's save as a continuation of the SAME identity. + // The previous source stays in the journal until durable cleanup. + const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${id}-${uuid()}`; + if (moved && step.to) { + // Installation won the race with a save to the old editor path. + // Stage its output before reinstalling this identity's merge. + journal.steps.push({ + documentId: id, + from: step.to, + staged: `${prefix}.previous`, + output: `${prefix}.unused`, + phase: "planned", + superseded: true + }); + } + journal.steps.push({ + documentId: id, + from: change.path, + to: step.to, + expected: step.expected, + replacement: step.replacement, + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + step.to = undefined; + step.replacement = undefined; + step.phase = "installed"; + step.superseded = true; + physical.local[id] = change.path; } } + + private directory(path: string): string { + return path.split("/").slice(0, -1).join("/"); + } + + private async writeOnce( + path: string, + snapshot: FileSnapshot + ): Promise { + const existing = await this.fs.readSnapshot(path); + if (existing) { + const actual = await toStoredSnapshot(existing); + const expected = await toStoredSnapshot(snapshot); + if (actual.contentBase64 !== expected.contentBase64) + throw new Error(`Recovery artifact changed: ${path}`); + await this.fs.flushPaths([path]); + return; + } + await this.ensureParents(path); + await this.fs.write(path, snapshot); + } + + private async writeJsonOnce(path: string, value: unknown): Promise { + await this.writeOnce(path, { + content: new TextEncoder().encode(JSON.stringify(value)) + }); + } + + private async persist(journal: ApplicationJournal): Promise { + await this.database.commit({ + ...this.database.state, + application: journal + }); + } + + public async retainSnapshot( + requestId: string, + documentId: string, + path: string | undefined, + snapshot: StoredSnapshot + ): Promise { + const prefix = `${INTERNAL_DIRECTORY}/requests/${requestId}`; + await this.writeOnce(`${prefix}.content`, fromStoredSnapshot(snapshot)); + const metadata = await this.fs.readSnapshot(`${prefix}.json`); + if (metadata) { + const saved: unknown = JSON.parse( + new TextDecoder().decode(metadata.content) + ); + if ( + typeof saved !== "object" || + saved === null || + !("documentId" in saved) || + saved.documentId !== documentId || + !("hash" in saved) || + saved.hash !== snapshot.hash + ) + throw new Error(`Recovery artifact changed: ${prefix}.json`); + await this.fs.flushPaths([`${prefix}.json`]); + return; // originalPath is a hint recorded once, not request identity. + } + await this.writeJsonOnce(`${prefix}.json`, { + documentId, + originalPath: path, + hash: snapshot.hash + }); + } + + public async apply( + next: EngineState, + writes: Record = {}, + validatePlan: () => void = () => {} + ): Promise { + if (this.database.state.application) + throw new Error( + "Recover the existing filesystem transaction first" + ); + + const id = uuid(); + const steps: FileStep[] = []; + for (const documentId of new Set([ + ...Object.keys(this.database.state.local), + ...Object.keys(next.local) + ])) { + const oldPath = this.database.state.local[documentId]; + const newPath = next.local[documentId]; + const materialized = + this.database.state.documents[documentId]?.materialized; + const from = materialized ? oldPath : undefined; + const write = writes[documentId]; + if (!write && (from === undefined || from === newPath)) continue; + const expected = + write?.expected ?? + (from ? await this.snapshot(from) : undefined); + // An external deletion is a local change, not permission to resurrect + // the file using a stale network response. + if (from && !expected && !write) { + delete next.local[documentId]; + continue; + } + const prefix = `${INTERNAL_DIRECTORY}/transactions/${id}/${documentId}`; + steps.push({ + documentId, + from, + to: newPath, + expected, + replacement: write?.replacement, + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + } + validatePlan(); + if (steps.length === 0) { + const changedPaths = this.changedPaths( + this.database.state.local, + next.local + ); + await this.database.commit(next); + this.filesApplied(changedPaths); + return; + } + + const journal = { + id, + next: structuredClone(next), + steps, + extensions: (await this.serverConfig.getConfig()) + .mergeableFileExtensions + }; + validatePlan(); + await this.persist(journal); + await this.recover(); + } + + public async recover(): Promise { + const saved = this.database.state.application; + if (!saved) return; + + const journal = structuredClone(saved); + await this.fs.createDirectory( + `${INTERNAL_DIRECTORY}/transactions/${journal.id}` + ); + + for (;;) { + await this.incorporateLocalChanges(journal); + // Stage every pending source before installing anything, including + // occupants discovered during an earlier installation attempt. + await this.stageSources(journal); + const step = journal.steps.find( + (step) => step.phase !== "installed" + ); + if (!step) break; + await this.prepareContent(journal, step); + await this.incorporateLocalChanges(journal); + if (step.superseded && step.phase === "installed") continue; + try { + await this.install(journal, step); + } catch (error) { + const code = + typeof error === "object" && + error !== null && + "code" in error && + typeof error.code === "string" + ? error.code + : undefined; + if ( + step.to === undefined || + ![ + "ENAMETOOLONG", + "EINVAL", + "ENOTSUP", + "EOPNOTSUPP", + "EACCES", + "EPERM" + ].includes(code ?? "") + ) + throw error; + const origin = step.replacement ? step.output : step.staged; + // A rename that took effect must be acknowledged at its original + // destination. Never infer failure from a failed durability flush. + if ( + step.phase === "installing" && + !(await this.fs.exists(origin)) + ) + throw error; + if (step.replanned === true) throw error; + const extension = + /\.[^.]*$/u.exec(step.to.split("/").at(-1) ?? "")?.[0] ?? + ""; + const wanted = `Recovered ${step.documentId}${extension}`; + step.to = await this.conflictPath( + journal, + step.documentId, + wanted + ); + step.replanned = true; + journal.next.local[step.documentId] = step.to; + // The payload has already been prepared; persist the new target + // before retrying so crashes cannot reapply its content merge. + step.phase = "prepared"; + await this.persist(journal); + } + } + + // Once every install intent is durable, the journal can be replayed + // without its staged inputs. Remove transient bytes before committing + // the next engine state so an interrupted cleanup is retried. + await this.cleanup(journal); + await this.database.commit(journal.next); + this.filesApplied( + [ + ...new Set( + journal.steps.flatMap((step) => [step.from, step.to]) + ) + ].filter((path): path is string => path !== undefined) + ); + } + + private async stageSources(journal: ApplicationJournal): Promise { + for (;;) { + await this.incorporateLocalChanges(journal); + const step = journal.steps.find((step) => step.phase === "planned"); + if (!step) break; + // An existing staging file proves the move completed before a crash. + if (step.from && !(await this.fs.exists(step.staged))) { + const { from } = step, + { staged } = step; + const present = await this.fs.exists(from); + await this.incorporateLocalChanges(journal); + if ( + step.from !== from || + step.staged !== staged || + step.phase !== "planned" + ) + continue; + if (present) { + await this.fs.rename(from, staged); + await this.incorporateLocalChanges(journal); + if ( + step.from !== from || + step.staged !== staged || + step.phase !== "planned" + ) + continue; + } else { + // Preserve an external deletion instead of resurrecting it. + delete journal.next.local[step.documentId]; + step.to = undefined; + step.replacement = undefined; + } + } + if (step.from) await this.fs.flushPaths([step.from, step.staged]); + step.phase = "staged"; + await this.persist(journal); + } + // Directory cleanup is also retried if staging completed before a crash. + for (const step of journal.steps) + await this.removeEmptyParents(step.from); + } + + private async prepareContent( + journal: ApplicationJournal, + step: FileStep + ): Promise { + if (step.phase !== "staged") return; + const source = await this.snapshot(step.staged); + if (source) { + if (step.replacement && step.expected) + step.replacement = await mergeContent( + step.to ?? step.from ?? "", + step.expected, + source, + step.replacement, + journal.extensions + ); + // The prepared replacement now includes this source's edits. A + // later save must be merged relative to this source, or superseded + // local text is reintroduced from the already merged replacement. + step.expected = source; + } + // Save the merged result before writing it, so recovery never merges twice. + step.phase = "prepared"; + await this.persist(journal); + } + + private async install( + journal: ApplicationJournal, + step: FileStep + ): Promise { + if (step.to) { + const origin = step.replacement ? step.output : step.staged; + const alreadyMoved = + step.phase === "installing" && !(await this.fs.exists(origin)); + if (alreadyMoved) { + if (!(await this.fs.exists(step.to))) { + // The installed file was externally deleted before + // recovery. Preserve that deletion instead of failing or + // resurrecting bytes from metadata. + delete journal.next.local[step.documentId]; + step.to = undefined; + step.replacement = undefined; + step.phase = "installed"; + await this.persist(journal); + return; + } + await this.fs.flushPaths([origin, step.to]); + } else { + if (step.replacement) + await this.writeOnce( + step.output, + fromStoredSnapshot(step.replacement) + ); + await this.incorporateLocalChanges(journal); + if (step.superseded) return; + if (!(await this.clearDestination(journal, step, step.to))) + return; + await this.ensureParents(step.to); + await this.incorporateLocalChanges(journal); + if (step.superseded) return; + // Persist intent first; the no-replace rename protects against + // another writer taking the destination after we checked it. + step.phase = "installing"; + await this.persist(journal); + await this.fs.rename(origin, step.to); + await this.incorporateLocalChanges(journal); + if (step.superseded) return; + } + const document = journal.next.documents[step.documentId]; + if (document) { + document.materialized = true; + document.observedHash = + step.replacement?.hash ?? + (await this.snapshot(step.to))?.hash; + } + } + // Deletions and completed renames share this path whether their + // acknowledgement was received or lost. Transient bytes are removed + // only after every step reaches this phase. + step.phase = "installed"; + await this.persist(journal); + } + + private async clearDestination( + journal: ApplicationJournal, + step: FileStep, + path: string + ): Promise { + const obstruction = await this.obstruction(path); + if (!obstruction) return true; + if (obstruction.kind === "file") { + const info = await this.fs.stat(obstruction.path); + if ( + journal.next.protectedPaths?.includes(obstruction.path) === + true || + (info !== undefined && + this.protectedFile(obstruction.path, info.size)) + ) { + (journal.next.protectedPaths ??= []).push(obstruction.path); + step.to = await this.conflictPath( + journal, + step.documentId, + path + ); + journal.next.local[step.documentId] = step.to; + await this.persist(journal); + } else await this.displace(journal, obstruction.path); + } else if ( + (await this.fs.listFilesRecursively(obstruction.path)).length + ) { + // Preserve unmanaged descendants by changing the incoming file's name. + step.to = await this.conflictPath(journal, step.documentId, path); + journal.next.local[step.documentId] = step.to; + await this.persist(journal); + } else { + // The adapter only removes empty directories, including on a race. + await this.fs.delete(obstruction.path); + return true; + } + // The journal now contains a new source or destination. Retry it through + // the main loop so newly discovered sources are staged before installing. + return false; + } + + private async obstruction( + path: string + ): Promise<{ path: string; kind: "file" | "directory" } | undefined> { + const parts = path.split("/"); + for (let i = 0; i < parts.length; i++) { + const prefix = parts.slice(0, i + 1).join("/"); + const info = await this.fs.stat(prefix); + if (info && (info.kind === "file" || i === parts.length - 1)) + return { path: prefix, kind: info.kind }; + } + return undefined; + } + + private async conflictPath( + journal: ApplicationJournal, + documentId: string, + path: string + ): Promise { + const occupied = { ...journal.next.local }; + delete occupied[documentId]; + for (const [index, diskPath] of ( + await this.listFilesRecursively() + ).entries()) + occupied[`disk-${index}`] = diskPath; + return allocatePortablePath(path, documentId, occupied); + } + + private async displace( + journal: ApplicationJournal, + path: string + ): Promise { + const unexpected = await this.snapshot(path); + if (!unexpected) + throw new Error(`Destination changed during recovery: ${path}`); + const documentId = uuid(); + const to = await this.conflictPath(journal, documentId, path); + const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${documentId}`; + journal.steps.push({ + documentId, + from: path, + to, + expected: unexpected, + staged: `${prefix}.source`, + output: `${prefix}.output`, + phase: "planned" + }); + journal.next.local[documentId] = to; + journal.next.documents[documentId] = { + materialized: true, + observedHash: unexpected.hash + }; + await this.persist(journal); + } + + private async ensureParents(path: string): Promise { + const parent = this.directory(path); + if (parent) await this.fs.createDirectory(parent); + } + + private async removeEmptyParents(path?: string): Promise { + if (!path) return; + let parent = this.directory(path); + while (parent && !isInternalPath(parent)) { + // An earlier installation may have turned an ancestor into a file. + const info = await this.obstruction(parent); + if (!info) { + parent = this.directory(parent); + continue; + } + if (info.kind !== "directory") break; + if ((await this.fs.listFilesRecursively(parent)).length) break; + // delete() must reject non-empty directories, including unmanaged entries. + try { + await this.fs.delete(parent); + } catch { + break; + } + parent = this.directory(parent); + } + } + + private async cleanup(journal: ApplicationJournal): Promise { + for (const step of journal.steps) { + const source = await this.snapshot(step.staged); + if ( + source && + (!step.to || + (step.replacement && source.hash !== step.replacement.hash)) + ) { + const prefix = `${INTERNAL_DIRECTORY}/recovery/${journal.id}-${step.staged.split("/").at(-1)}`; + await this.writeOnce( + `${prefix}.content`, + fromStoredSnapshot(source) + ); + await this.writeJsonOnce(`${prefix}.json`, { + documentId: step.documentId, + originalPath: step.from, + hash: source.hash + }); + } + for (const path of [ + step.staged, + step.output, + `${step.staged}.recovery`, + `${step.staged}.json` + ]) { + if (await this.fs.exists(path)) await this.fs.deleteFile(path); + } + } + const completed = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/completed.json`; + if (await this.fs.exists(completed)) + await this.fs.deleteFile(completed); + await this.fs.delete( + `${INTERNAL_DIRECTORY}/transactions/${journal.id}` + ); + } + + private changedPaths( + before: Readonly>, + after: Readonly> + ): string[] { + const result = new Set(); + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(after) + ])) { + if (before[id] === after[id]) continue; + if (before[id] !== undefined) result.add(before[id]); + if (after[id] !== undefined) result.add(after[id]); + } + return [...result]; + } } diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index 36dddfe6..f4e79397 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -1,37 +1,45 @@ -import type { RelativePath } from "../persistence/database"; +import type { CursorPosition } from "reconcile-text"; -import type { TextWithCursors } from "reconcile-text"; - +export interface FileSnapshot { + content: Uint8Array; + /** Optional editor selections, moved with the file and applied with its bytes. */ + cursors?: CursorPosition[]; +} + +/** Required adapter contract for API v4; there is deliberately no weak fallback. + * Paths are vault-relative. Never follow symlinks (including ancestor symlinks). + * Only independent regular files are supported; reject hardlinks/special files. + * Every mutation must flush file data and affected directories before resolving. + * A successful scan is complete: unreadable directories must reject, not vanish. + */ export interface FileSystemOperations { - // List all files under root that should be synced. If root is undefined, return every file. - listFilesRecursively: ( - root: RelativePath | undefined - ) => Promise; - - // Read the content of a file. - read: (path: RelativePath) => Promise; - - // Create or overwrite a file with the given content. - write: (path: RelativePath, content: Uint8Array) => Promise; - - // Atomically update the content of a text file. - atomicUpdateText: ( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ) => Promise; - - // Get the size of a file in bytes. - getFileSize: (path: RelativePath) => Promise; - - // Check if a file exists. - exists: (path: RelativePath) => Promise; - - // Create a directory at the specified path. All parent directories must already exist. - createDirectory: (path: RelativePath) => Promise; - - // Delete a file. It is expected that the path points to an existing file. - delete: (path: RelativePath) => Promise; - - // Rename a file. It is expected that the oldPath points to an existing file and the newPath does not exist. - rename: (oldPath: RelativePath, newPath: RelativePath) => Promise; + listFilesRecursively: (root?: string) => Promise; + /** Coherent content snapshot; undefined only for an absent file. Reject read races. */ + readSnapshot: (path: string) => Promise; + /** Reject symlinks and special files rather than following them. */ + stat: ( + path: string + ) => Promise<{ kind: "file" | "directory"; size: number } | undefined>; + /** Includes directories. */ + exists: (path: string) => Promise; + /** Recursive, idempotent, durable directory creation. */ + createDirectory: (path: string) => Promise; + /** Atomically create a complete file: never replace an existing object. + * A crash leaves the path absent or complete, never a partial file. + * Flush data and directory entries before resolving. */ + write: (path: string, snapshot: FileSnapshot) => Promise; + /** Atomic, durable rename on the same filesystem; fail if destination exists. + * Flush the source's data and both affected directories before resolving. */ + rename: (from: string, to: string) => Promise; + /** Flush existing files and all affected ancestor directories, even when a + * named file is absent. Recovery uses this after a mutation became visible + * before its caller learned that it was durable. */ + flushPaths: (paths: readonly string[]) => Promise; + /** Idempotently unlink one regular file and flush its parent directory. + * The sync engine uses this only for its own internal recovery artifacts. */ + deleteFile: (path: string) => Promise; + /** Prune an implicit directory tree using rmdir only, durably. Reject + * regular files and never unlink descendant files; reject if files remain. + * Partial removal of empty child directories on failure is harmless. */ + delete: (path: string) => Promise; } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 86b2845c..750eb667 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,374 +1,235 @@ +import type { CursorPosition } from "reconcile-text"; import type { Logger } from "../tracing/logger"; -import { EMPTY_HASH } from "../utils/hash"; -import { CoveredValues } from "../utils/data-structures/min-covered"; -import { awaitAll } from "../utils/await-all"; -import { removeFromArray } from "../utils/remove-from-array"; +import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { FileManifestUpdateResponse } from "../services/types/FileManifestUpdateResponse"; +import type { PushFileManifest } from "../services/types/PushFileManifest"; +import type { PutFileContent } from "../services/types/PutFileContent"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import type { EventRecord } from "../services/types/EventRecord"; +import type { FileManifestEntries } from "../types/file-manifest-entries"; export type VaultUpdateId = number; export type DocumentId = string; export type RelativePath = string; +export interface StoredSnapshot { + contentBase64: string; + hash: string; + cursors?: CursorPosition[]; +} + export interface DocumentMetadata { - parentVersionId: VaultUpdateId; + parentVersionId: number; hash: string; - remoteRelativePath?: RelativePath; + remoteRelativePath?: string; } -export interface StoredDocumentMetadata { - relativePath: RelativePath; - documentId: DocumentId; - parentVersionId: VaultUpdateId; - remoteRelativePath?: RelativePath; - hash: string; +export interface DocumentState { + /** Local clean base retained while rebasing after a server restore. */ + recoveryBase?: StoredSnapshot; + base?: DocumentVersionWithoutContent & { hash: string }; + observedHash?: string; + materialized: boolean; + /** First contact with existing local content: there is no common ancestor. */ + bootstrap?: boolean; + /** Do not resubmit an unchanged payload rejected by the server. */ + rejected?: { hash: string; message: string }; } -export interface StoredDatabase { - documents: StoredDocumentMetadata[]; - lastSeenUpdateId: VaultUpdateId | undefined; - hasInitialSyncCompleted: boolean; +export type PendingRequest = { rejection?: string } & ( + | { + type: "content"; + documentId: string; + request: PutFileContent; + snapshot: StoredSnapshot; + response?: DocumentUpdateResponse; + } + | { + type: "fileManifest"; + request: PushFileManifest; + response?: FileManifestUpdateResponse; + } +); + +export interface EngineState { + /** Durable reset intent; clear the transport checkpoint before bootstrap. */ + historyRecovery?: boolean; + vaultKey: string; + initialized: boolean; + fileManifest: FileManifest; + local: FileManifestEntries; + documents: Record; + lastSeenUpdateId: number; + remoteHeads: Record; + bootstrap?: VaultSnapshot; + /** Fold pages durably without applying intermediate remote snapshots. */ + eventReplay?: { + after: number; + manifest: FileManifest; + heads: Record; + receipts: EventRecord[]; + }; + pending?: PendingRequest; + /** A rejected retry does not prove an earlier attempt failed to commit. */ + unconfirmed?: PendingRequest[]; + /** Commits consumption of the durable notification queue with the identity map. */ + lastAppliedLocalChangeId?: string; + /** Physical exclusions, including files with no sync identity. */ + protectedPaths?: string[]; + /** Original server paths while local files are excluded; null means absent. */ + excluded?: Record; + rejectedManifest?: { entries: FileManifestEntries; message: string }; } -/** - * Represents a document in the database. - * - * It is mutable and its content should always represent the latest - * state of the document on disk based on the update events we have seen. - */ +export interface FileStep { + documentId: string; + from?: string; + to?: string; + expected?: StoredSnapshot; + replacement?: StoredSnapshot; + staged: string; + output: string; + /** A permanent destination error has already selected a root fallback. */ + replanned?: boolean; + /** Retained input of a later step for this identity; never projected as live. */ + superseded?: boolean; + phase: "planned" | "staged" | "prepared" | "installing" | "installed"; +} + +export interface ApplicationJournal { + id: string; + extensions: string[]; + next: EngineState; + steps: FileStep[]; +} + +export interface StoredDatabase extends EngineState { + application?: ApplicationJournal; +} + +// Read-only view used by cursor tracking and status reporting. export interface DocumentRecord { - relativePath: RelativePath; - documentId: DocumentId; - metadata: DocumentMetadata | undefined; + documentId: string; + relativePath: string; + metadata?: DocumentMetadata; isDeleted: boolean; - updates: Promise[]; - parallelVersion: number; +} + +export function emptyState(vaultKey: string): StoredDatabase { + return { + vaultKey, + initialized: false, + fileManifest: { fileManifestId: 0, entries: {} }, + local: {}, + documents: {}, + remoteHeads: {}, + lastSeenUpdateId: 0 + }; +} + +function canRebindEmpty(state: Partial | undefined): boolean { + return ( + state !== undefined && + !state.initialized && + !state.pending && + !state.application && + !state.bootstrap && + !state.eventReplay && + (state.unconfirmed?.length ?? 0) === 0 && + Object.keys(state.local ?? {}).length === 0 + ); } export class Database { - private documents: DocumentRecord[]; - private lastSeenUpdateIds: CoveredValues; - private hasInitialSyncCompleted: boolean; - + public state: StoredDatabase; + private needsReload = false; public constructor( private readonly logger: Logger, - initialState: Partial | undefined, - private readonly saveData: (data: StoredDatabase) => Promise + initial: Partial | undefined, + private readonly saveData: (data: StoredDatabase) => Promise, + vaultKey: string, + private readonly loadData: () => Promise ) { - initialState ??= {}; - - this.documents = - initialState.documents?.map( - ({ relativePath, documentId, ...metadata }) => ({ - relativePath, - documentId, - metadata, - isDeleted: false, - updates: [], - parallelVersion: 0 - }) - ) ?? []; - - this.ensureConsistency(); - this.logger.debug(`Loaded ${this.documents.length} documents`); - - const { lastSeenUpdateId } = initialState; - this.logger.debug(`Loaded last seen update id: ${lastSeenUpdateId}`); - this.lastSeenUpdateIds = new CoveredValues( - Math.max(0, lastSeenUpdateId ?? 0) // the first updateId will be 1 which is the first integer after -1 - ); - - this.documents.forEach((doc) => { - this.lastSeenUpdateIds.add(doc.metadata?.parentVersionId); - }); - - this.hasInitialSyncCompleted = - initialState.hasInitialSyncCompleted ?? false; - this.logger.debug( - `Loaded hasInitialSyncCompleted: ${this.hasInitialSyncCompleted}` - ); - } - - public get length(): number { - return this.documents.length; - } - - public get resolvedDocuments(): DocumentRecord[] { - const paths = new Map(); - this.documents - // eslint-disable-next-line no-restricted-syntax -- Type narrowing, not removing a specific item - .filter(({ metadata }) => metadata !== undefined) - .forEach((record) => - paths.set(record.relativePath, [ - record, - ...(paths.get(record.relativePath) ?? []) - ]) - ); - - return Array.from(paths.values()).map((records) => { - records.sort( - (a, b) => b.parallelVersion - a.parallelVersion // descending - ); - - if ( - records.length > 1 && - records.some((current, i) => - i === 0 - ? false - : records[i - 1].parallelVersion === - current.parallelVersion - ) - ) { - throw new Error( - `Multiple documents with the same parallel version and path at ${records[0].relativePath}` - ); - } - return records[0]; - }); - } - - public updateDocumentMetadata( - metadata: { - parentVersionId: VaultUpdateId; - hash: string; - remoteRelativePath: RelativePath; - }, - toUpdate: DocumentRecord - ): void { - if (!this.documents.includes(toUpdate)) { - throw new Error("Document not found in database"); - } - - toUpdate.metadata = metadata; - - this.saveInTheBackground(); - } - - public removeDocumentPromise(promise: Promise): void { - const entry = this.documents.find(({ updates }) => - updates.includes(promise) - ); - - if (entry === undefined) { - // This method should be idempotent and tolerant of - // stragglers calling it after the databse has been reset. - return; - } - - removeFromArray(entry.updates, promise); - // No need to save as Promises don't get serialized - } - - public removeDocument(find: DocumentRecord): void { - removeFromArray(this.documents, find); - this.saveInTheBackground(); - } - - public getLatestDocumentByRelativePath( - find: RelativePath - ): DocumentRecord | undefined { - const candidates = this.documents.filter( - ({ relativePath }) => relativePath === find - ); - candidates.sort((a, b) => b.parallelVersion - a.parallelVersion); // descending - return candidates[0]; - } - - public async getResolvedDocumentByRelativePath( - relativePath: RelativePath, - promise: Promise - ): Promise { - const entry = this.getLatestDocumentByRelativePath(relativePath); - - if (entry === undefined) { + const hasState = initial && Object.keys(initial).length > 0; + if ( + hasState && + initial.vaultKey !== vaultKey && + !canRebindEmpty(initial) + ) { throw new Error( - `Document not found by relative path: ${relativePath}, ${JSON.stringify( - this.documents, - null, - 2 - )}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); } - - const currentPromises = entry.updates; - entry.updates = [...currentPromises, promise]; - await awaitAll(currentPromises); - - return entry; + this.state = hasState + ? { ...(initial as StoredDatabase), vaultKey } + : emptyState(vaultKey); } - public createNewPendingDocument( - documentId: DocumentId, - relativePath: RelativePath, - promise: Promise - ): DocumentRecord { - this.logger.debug( - `Creating new pending document: ${relativePath} (${documentId})` - ); - const previousEntry = - this.getLatestDocumentByRelativePath(relativePath); - - const entry = { - relativePath, - documentId, - metadata: undefined, - isDeleted: false, - updates: [promise], - parallelVersion: - previousEntry?.parallelVersion === undefined - ? 0 - : previousEntry.parallelVersion + 1 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public createNewEmptyDocument( - documentId: DocumentId, - parentVersionId: VaultUpdateId, - relativePath: RelativePath - ): DocumentRecord { - const entry = { - relativePath, - documentId, - metadata: { - parentVersionId, - hash: EMPTY_HASH, - remoteRelativePath: relativePath - }, - isDeleted: false, - updates: [], - parallelVersion: 0 - }; - - this.documents.push(entry); - this.saveInTheBackground(); - - return entry; - } - - public getDocumentByDocumentId( - find: DocumentId - ): DocumentRecord | undefined { - return this.documents.find(({ documentId }) => documentId === find); - } - - public move( - oldRelativePath: RelativePath, - newRelativePath: RelativePath - ): void { - const oldDocument = - this.getLatestDocumentByRelativePath(oldRelativePath); - - if (oldDocument === undefined) { - return; - } - - const newDocument = - this.getLatestDocumentByRelativePath(newRelativePath); - if (newDocument?.isDeleted === false) { + /** Bind an empty database to the configured vault. */ + public async bindVault(vaultKey: string): Promise { + if (this.state.vaultKey === vaultKey) return; + if (!canRebindEmpty(this.state)) throw new Error( - `Document already exists at new location: ${newRelativePath}` + "Incompatible vault identity. Preserve it for recovery and initialize a separate state store." ); - } - - oldDocument.relativePath = newRelativePath; - // We're in a strange state where the target of the move has just got deleted, - // however, its metadata might already have a bunch of updates queued up for - // the document at the new location. We need to keep these updates. - oldDocument.parallelVersion = - newDocument !== undefined ? newDocument.parallelVersion + 1 : 0; - - this.saveInTheBackground(); + await this.commit({ ...this.state, vaultKey }); } - public delete(relativePath: RelativePath): void { - const candidate = this.getLatestDocumentByRelativePath(relativePath); - if (candidate === undefined) { + public async commit(next: StoredDatabase): Promise { + const snapshot = structuredClone(next); + if (this.needsReload) throw new Error( - `Document not found by relative path: ${relativePath}` + "Reload durable state before retrying an uncertain save" ); - } - candidate.isDeleted = true; + this.needsReload = true; + await this.saveData(snapshot); + this.state = snapshot; + this.needsReload = false; } - public getHasInitialSyncCompleted(): boolean { - return this.hasInitialSyncCompleted; - } - - public setHasInitialSyncCompleted(value: boolean): void { - this.hasInitialSyncCompleted = value; - this.saveInTheBackground(); - } - - public getLastSeenUpdateId(): VaultUpdateId { - return this.lastSeenUpdateIds.min; - } - - public addSeenUpdateId(value: number): void { - const previousMin = this.lastSeenUpdateIds.min; - this.lastSeenUpdateIds.add(value); - if (previousMin !== this.lastSeenUpdateIds.min) { - this.saveInTheBackground(); - } - } - - public setLastSeenUpdateId(value: number): void { - this.lastSeenUpdateIds.min = value; - this.saveInTheBackground(); - } - - public reset(): void { - this.documents = []; - this.lastSeenUpdateIds = new CoveredValues( - 0 // the first updateId will be 1 which is the first integer after -1 - ); - this.hasInitialSyncCompleted = false; - this.saveInTheBackground(); + public async recoverPersistence(): Promise { + if (!this.needsReload) return; + const saved = await this.loadData(); + if (saved) this.state = saved; + this.needsReload = false; } public async save(): Promise { - return this.saveData({ - documents: this.resolvedDocuments.map( - ({ relativePath, documentId, metadata }) => ({ - documentId, - relativePath, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ...metadata! // `resolvedDocuments` only returns docs with metadata set - }) - ), - lastSeenUpdateId: this.lastSeenUpdateIds.min, - hasInitialSyncCompleted: this.hasInitialSyncCompleted - }); + await this.commit(this.state); } - private ensureConsistency(): void { - const idToPath = new Map(); - - this.resolvedDocuments.forEach(({ relativePath, documentId }) => { - idToPath.set(documentId, [ - ...(idToPath.get(documentId) ?? []), - relativePath - ]); - }); - - const duplicates = Array.from(idToPath.entries()) - .filter(([_, paths]) => paths.length > 1) - .map(([id, paths]) => `${id} (${paths.join(", ")})`); - - if (duplicates.length > 0) { - throw new Error( - "Document IDs are not unique, found duplicates: " + - duplicates.join("; ") - ); - } + public get length(): number { + return Object.keys(this.state.local).length; } - private saveInTheBackground(): void { - this.ensureConsistency(); - void this.save().catch((error: unknown) => { - this.logger.error(`Error saving data: ${error}`); - }); + public getLastSeenUpdateId(): number { + return this.state.lastSeenUpdateId; + } + + public getDocumentByDocumentId(id: string): DocumentRecord | undefined { + const relativePath = this.state.local[id]; + if (relativePath === undefined) return undefined; + const base = this.state.documents[id]?.base; + return { + documentId: id, + relativePath, + isDeleted: false, + metadata: base && { + parentVersionId: base.vaultUpdateId, + hash: base.hash, + remoteRelativePath: this.state.fileManifest.entries[id] + } + }; + } + + public getLatestDocumentByRelativePath( + path: string + ): DocumentRecord | undefined { + const id = Object.keys(this.state.local).find( + (id) => this.state.local[id] === path + ); + return id === undefined ? undefined : this.getDocumentByDocumentId(id); } } diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 3c5f604c..48629d49 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -3,6 +3,7 @@ import type { FetchController } from "./fetch-controller"; import type { Logger } from "../tracing/logger"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; import type { DocumentVersion } from "./types/DocumentVersion"; +import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { EventBatch } from "./types/EventBatch"; import type { FileManifest } from "./types/FileManifest"; import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse"; @@ -10,7 +11,11 @@ import type { PutFileContent } from "./types/PutFileContent"; import type { PushFileManifest } from "./types/PushFileManifest"; import type { VaultSnapshot } from "./types/VaultSnapshot"; import type { PingResponse } from "./types/PingResponse"; -import { AuthenticationError, PermanentSyncError } from "../errors/errors"; +import { + AuthenticationError, + PermanentSyncError, + ServerHistoryChangedError +} from "../errors/errors"; export class SyncService { private readonly client: typeof fetch; @@ -21,7 +26,11 @@ export class SyncService { fetchController: FetchController, private readonly settings: Settings, logger: Logger, - fetchImplementation: typeof fetch = globalThis.fetch + fetchImplementation: typeof fetch = globalThis.fetch, + private readonly history?: { + get: () => string | undefined; + save: (checkpoint: string | undefined) => Promise; + } ) { this.rawFetch = async (...args) => fetchImplementation(...args); this.client = fetchController.getControlledFetchImplementation( @@ -30,6 +39,42 @@ export class SyncService { ); } + public get verifiesHistory(): boolean { + return this.history !== undefined; + } + public get hasHistoryCheckpoint(): boolean { + return this.history?.get() !== undefined; + } + + public async resetHistory(): Promise { + await this.history?.save(undefined); + } + + private historyHeaders(): Record { + const checkpoint = this.history?.get(); + return checkpoint ? { "X-Vault-Link-History": checkpoint } : {}; + } + + private async recordHistory(response: Response): Promise { + const checkpoint = response.headers.get("x-vault-link-history"); + if (checkpoint && this.history) { + const previous = this.history.get(); + if ( + !previous || + Number(checkpoint.split(":")[0]) >= + Number(previous.split(":")[0]) + ) + await this.history.save(checkpoint); + } + } + + private checkHistory(response: Response): void { + if (response.headers.get("x-vault-link-history-mismatch") === "1") + throw new ServerHistoryChangedError( + "Server history changed; recovering local work" + ); + } + public getUrl(path: string): string { const { remoteUri, vaultName } = this.settings.getSettings(); return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`; @@ -49,6 +94,7 @@ export class SyncService { this.settings.getSettings().requestTimeoutMs ), headers: { + ...(raw ? {} : this.historyHeaders()), Authorization: `Bearer ${this.settings.getSettings().token}`, "Device-Id": this.deviceId, "Content-Type": "application/json" @@ -56,6 +102,7 @@ export class SyncService { } ); + this.checkHistory(response); if (!response.ok) { this.throwForErrorResponse( response, @@ -63,13 +110,11 @@ export class SyncService { ); } + if (!raw) await this.recordHistory(response); return (await response.json()) as T; } - private throwForErrorResponse( - response: Response, - message: string - ): never { + private throwForErrorResponse(response: Response, message: string): never { if (response.status === 401 || response.status === 403) { throw new AuthenticationError(message); } @@ -117,6 +162,10 @@ export class SyncService { return this.request(`/documents/${id}`); } + public async metadata(id: string): Promise { + return this.request(`/documents/${id}/metadata`); + } + public async getDocumentVersionContent({ documentId, vaultUpdateId @@ -133,11 +182,13 @@ export class SyncService { this.settings.getSettings().requestTimeoutMs ), headers: { + ...this.historyHeaders(), Authorization: `Bearer ${this.settings.getSettings().token}` } } ); + this.checkHistory(response); if (!response.ok) { this.throwForErrorResponse( response, @@ -145,6 +196,7 @@ export class SyncService { ); } + await this.recordHistory(response); return new Uint8Array(await response.arrayBuffer()); } } diff --git a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts index 572e2d9d..85dc56ab 100644 --- a/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts +++ b/frontend/sync-client/src/services/types/DocumentUpdateResponse.ts @@ -1,8 +1,7 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -import type { DocumentVersion } from "./DocumentVersion"; import type { DocumentVersionWithoutContent } from "./DocumentVersionWithoutContent"; /** * Response to an update document request. */ -export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersion; +export type DocumentUpdateResponse = { "type": "Accepted" } & DocumentVersionWithoutContent | { "type": "StaleBase" } & DocumentVersionWithoutContent; diff --git a/frontend/sync-client/src/services/types/EventBatch.ts b/frontend/sync-client/src/services/types/EventBatch.ts index f17f17ed..a3bfb573 100644 --- a/frontend/sync-client/src/services/types/EventBatch.ts +++ b/frontend/sync-client/src/services/types/EventBatch.ts @@ -1,4 +1,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { EventRecord } from "./EventRecord"; -export type EventBatch = { headEventId: number, events: Array, }; +export type EventBatch = { headEventId: number, +/** + * Last event in this page; `head_event_id` is the vault's current head. + */ +endEventId?: number, events: Array, }; diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts index 969a02eb..d5223b6a 100644 --- a/frontend/sync-client/src/services/websocket-manager.test.ts +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -1,9 +1,10 @@ /* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ import { describe, it, beforeEach } from "node:test"; import assert from "node:assert"; +import { createServer } from "node:net"; import { WebSocketManager } from "./websocket-manager"; -import type { Logger } from "../tracing/logger"; -import type { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { Settings } from "../persistence/settings"; // eslint-disable-next-line @typescript-eslint/no-require-imports const WebSocket = require("ws") as typeof globalThis.WebSocket; @@ -337,3 +338,179 @@ describe("WebSocketManager", () => { await manager.stop(); }); }); + +it( + "refused WebSocket connections and replacing a connecting socket do not emit unhandled errors", + { timeout: 5_000 }, + async () => { + const server = createServer(); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve) + ); + const address = server.address(); + assert(address && typeof address !== "string"); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ); + const logger = new Logger(); + const settings = new Settings( + logger, + { + remoteUri: `http://127.0.0.1:${address.port}`, + webSocketRetryIntervalMs: 10 + }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + WebSocket + ); + let closes = 0; + const disconnected = Promise.withResolvers(); + manager.onWebSocketStatusChanged.add((connected) => { + if (!connected && ++closes >= 2) disconnected.resolve(); + }); + try { + manager.start(); + manager.start(); + await disconnected.promise; + assert.equal(manager.isWebSocketConnected, false); + } finally { + await manager.stop(); + } + } +); + +for (const failedAttempt of [1, 2]) { + it(`recovers from a WebSocket constructor failure on attempt ${failedAttempt}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + let attempts = 0; + class FailingSocket extends MockWebSocket { + static OPEN = 1; + constructor(url: string) { + if (++attempts === failedAttempt) + throw new Error("socket allocation failed"); + super(url); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test", webSocketRetryIntervalMs: 1000 }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + FailingSocket as unknown as typeof WebSocket + ); + try { + assert.doesNotThrow(() => manager.start()); + context.mock.timers.tick(0); + if (failedAttempt === 2) { + const socket = ( + manager as unknown as { webSocket: MockWebSocket } + ).webSocket; + socket.close(); + } + assert.doesNotThrow(() => context.mock.timers.tick(1000)); + context.mock.timers.tick(1000); + assert.equal(attempts, failedAttempt + 1); + assert.equal(manager.isWebSocketConnected, true); + } finally { + await manager.stop(); + } + }); +} + +for (const closeBehavior of ["throws", "stays open"]) { + it(`stop retires a socket even when close ${closeBehavior}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + class BrokenCloseSocket extends MockWebSocket { + static OPEN = 1; + close() { + if (closeBehavior === "throws") throw new Error("close failed"); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test" }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + BrokenCloseSocket as unknown as typeof WebSocket + ); + let connected = false, + delivered = 0; + manager.onWebSocketStatusChanged.add((status) => { + connected = status; + }); + manager.onRemoteVaultUpdateReceived.add(async () => { + delivered++; + }); + manager.start(); + context.mock.timers.tick(0); + const socket = (manager as unknown as { webSocket: MockWebSocket }) + .webSocket; + const stopping = manager.stop(); + context.mock.timers.tick(10_000); + await stopping; + assert.equal(manager.isWebSocketConnected, false); + assert.equal(connected, false); + socket.simulateMessage({ + type: "vaultEvents", + headEventId: 0, + events: [] + }); + await manager.waitUntilFinished(); + assert.equal(delivered, 0); + }); +} + +it("a handshake send failure retires the connection without escaping the open callback", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + class BrokenSendSocket extends MockWebSocket { + static OPEN = 1; + send() { + throw new Error("transport closed during open"); + } + } + const logger = new Logger(); + const settings = new Settings( + logger, + { remoteUri: "http://test" }, + async () => {} + ); + const manager = new WebSocketManager( + "test", + logger, + settings, + BrokenSendSocket as unknown as typeof WebSocket + ); + manager.onWebSocketStatusChanged.add((connected) => { + if (connected) + manager.sendHandshakeMessage({ + type: "handshake", + token: "test", + deviceId: "test", + lastSeenVaultUpdateId: 0 + }); + }); + const statuses: boolean[] = []; + manager.onWebSocketStatusChanged.add((status) => statuses.push(status)); + try { + manager.start(); + assert.doesNotThrow(() => context.mock.timers.tick(0)); + assert.equal(manager.isWebSocketConnected, false); + assert.equal(statuses.at(-1), false); + } finally { + await manager.stop(); + } +}); diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index c2d323f7..3fb80947 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -26,6 +26,7 @@ export class WebSocketManager { private isStopped = true; private resolveDisconnectingPromise: null | (() => unknown) = null; + private receiveTimeoutId: ReturnType | undefined; private reconnectTimeoutId: ReturnType | undefined; private readonly outstandingPromises: Promise[] = []; @@ -56,12 +57,14 @@ export class WebSocketManager { public get isWebSocketConnected(): boolean { return ( - this.webSocket?.readyState === - this.webSocketFactoryImplementation.OPEN + this.webSocket !== undefined && + this.webSocket.readyState === + this.webSocketFactoryImplementation.OPEN ); } public start(): void { + if (!this.isStopped) return; this.isStopped = false; this.initializeWebSocket(); } @@ -71,13 +74,20 @@ export class WebSocketManager { this.resolveDisconnectingPromise = resolve; this.isStopped = true; + clearTimeout(this.receiveTimeoutId); if (this.reconnectTimeoutId !== undefined) { clearTimeout(this.reconnectTimeoutId); this.reconnectTimeoutId = undefined; } - this.webSocket?.close(1000, "WebSocketManager has been stopped"); + const socket = this.webSocket; + try { + socket?.close(1000, "WebSocketManager has been stopped"); + } catch (error) { + this.logger.warn(`Failed to close WebSocket: ${String(error)}`); + if (socket) this.reconnect(socket); + } // eslint-disable-next-line @typescript-eslint/init-declarations let timeoutId: ReturnType | undefined; @@ -107,6 +117,9 @@ export class WebSocketManager { if (timeoutId !== undefined) { clearTimeout(timeoutId); } + // A throwing, silent or timed-out close must still fence callbacks + // before a stopped client can release its state or restart. + if (socket) this.reconnect(socket); } await this.waitUntilFinished(); @@ -132,6 +145,7 @@ export class WebSocketManager { this.logger.error( `Failed to send handshake message: ${String(error)}` ); + this.reconnect(webSocket); throw error; } } @@ -162,7 +176,66 @@ export class WebSocketManager { } } + private expectResponse(socket: WebSocket): void { + clearTimeout(this.receiveTimeoutId); + // Cursor heartbeats are acknowledged every 15 seconds, including when + // there are no open editors. An OPEN TCP socket alone proves nothing. + this.receiveTimeoutId = setTimeout(() => { + if (this.webSocket !== socket || this.isStopped) return; + this.logger.warn("WebSocket receive deadline expired"); + this.reconnect(socket); + }, 45_000); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Browser timers have no unref method. + this.receiveTimeoutId.unref?.(); + } + + private reconnect(socket: WebSocket): void { + if (this.webSocket !== socket) return; + clearTimeout(this.receiveTimeoutId); + socket.onopen = null; + socket.onclose = null; + socket.onmessage = null; + socket.onerror = (): void => { + /* Retired transports may still emit errors. */ + }; + this.webSocket = undefined; + this.onWebSocketStatusChanged.trigger(false); + try { + socket.close(); + } catch { + /* The transport may already be gone. */ + } + if (this.isStopped) { + this.resolveDisconnectingPromise?.(); + this.resolveDisconnectingPromise = null; + } else { + this.scheduleReconnect(); + } + } + + private scheduleReconnect(): void { + if (this.isStopped) return; + clearTimeout(this.reconnectTimeoutId); + this.reconnectTimeoutId = setTimeout(() => { + this.reconnectTimeoutId = undefined; + this.initializeWebSocket(); + }, this.settings.getSettings().webSocketRetryIntervalMs); + } + private initializeWebSocket(): void { + if (this.isStopped) return; + try { + this.openWebSocket(); + } catch (error) { + this.logger.warn(`Cannot initialize WebSocket: ${String(error)}`); + this.webSocket = undefined; + clearTimeout(this.receiveTimeoutId); + this.onWebSocketStatusChanged.trigger(false); + this.scheduleReconnect(); + } + } + + private openWebSocket(): void { // Clean up old WebSocket handlers to prevent race conditions if (this.webSocket) { try { @@ -170,7 +243,9 @@ export class WebSocketManager { this.webSocket.onopen = null; this.webSocket.onclose = null; this.webSocket.onmessage = null; - this.webSocket.onerror = null; + this.webSocket.onerror = (): void => { + /* Retired transports may still emit errors. */ + }; this.webSocket.close(); } catch (e) { this.logger.error( @@ -188,7 +263,14 @@ export class WebSocketManager { this.logger.info(`Connecting to WebSocket at ${wsUri.toString()}`); - this.webSocket = new this.webSocketFactoryImplementation(wsUri); + const socket = new this.webSocketFactoryImplementation(wsUri); + this.webSocket = socket; + this.expectResponse(socket); + + this.webSocket.onerror = (): void => { + this.logger.warn("WebSocket transport error"); + this.reconnect(socket); + }; this.webSocket.onopen = (): void => { // Check if we've been stopped while connecting @@ -200,7 +282,18 @@ export class WebSocketManager { return; } this.logger.info("WebSocket connection opened"); - this.onWebSocketStatusChanged.trigger(true); + try { + this.onWebSocketStatusChanged.trigger(true); + // A handshake failure can retire this socket from inside an + // observer. Finish delivery with its actual connection state. + if (this.webSocket !== socket) + this.onWebSocketStatusChanged.trigger(false); + } catch (error) { + this.logger.warn( + `WebSocket handshake failed: ${String(error)}` + ); + this.reconnect(socket); + } }; this.webSocket.onmessage = (event): void => { @@ -210,6 +303,8 @@ export class WebSocketManager { event.data ) as WebSocketServerMessage; + if (["vaultEvents", "cursorPositions"].includes(message.type)) + this.expectResponse(socket); // Track the message handling promise const messageHandlingPromise = this.handleWebSocketMessage( message @@ -238,17 +333,7 @@ export class WebSocketManager { this.logger.warn( `WebSocket closed with code ${event.code} (${event.reason == "" ? "unknown reason" : event.reason})` ); - this.onWebSocketStatusChanged.trigger(false); - - if (this.isStopped) { - this.resolveDisconnectingPromise?.(); - this.resolveDisconnectingPromise = null; - } else { - this.reconnectTimeoutId = setTimeout(() => { - this.reconnectTimeoutId = undefined; - this.initializeWebSocket(); - }, this.settings.getSettings().webSocketRetryIntervalMs); - } + this.reconnect(socket); }; } diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 2a272c86..71dabeaa 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -1,62 +1,246 @@ -import type { PersistenceProvider } from "./persistence/persistence"; -import type { HistoryEntry, HistoryStats } from "./tracing/sync-history"; -import { SyncHistory } from "./tracing/sync-history"; -import { Logger, LogLevel, LogLine } from "./tracing/logger"; -import type { RelativePath, StoredDatabase } from "./persistence/database"; -import { Database } from "./persistence/database"; import * as Sentry from "@sentry/browser"; -import type { SyncSettings } from "./persistence/settings"; -import { DEFAULT_SETTINGS, Settings } from "./persistence/settings"; -import { SyncService } from "./services/sync-service"; -import { Syncer } from "./sync-operations/syncer"; +import { setUpTelemetry } from "./utils/set-up-telemetry"; +import type { PersistenceProvider } from "./persistence/persistence"; +import { Database, type StoredDatabase } from "./persistence/database"; +import { + Settings, + DEFAULT_SETTINGS, + type SyncSettings +} from "./persistence/settings"; import type { FileSystemOperations } from "./file-operations/filesystem-operations"; import { FileOperations } from "./file-operations/file-operations"; +import { Logger, LogLevel } from "./tracing/logger"; +import { SyncHistory } from "./tracing/sync-history"; import { FetchController } from "./services/fetch-controller"; -import { UnrestrictedSyncer } from "./sync-operations/unrestricted-syncer"; -import { rateLimit } from "./utils/rate-limit"; -import type { NetworkConnectionStatus } from "./types/network-connection-status"; -import { DocumentSyncStatus } from "./types/document-sync-status"; -import { WebSocketManager } from "./services/websocket-manager"; -import { createClientId } from "./utils/create-client-id"; -import { CursorTracker } from "./sync-operations/cursor-tracker"; -import type { CursorSpan } from "./services/types/CursorSpan"; -import type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; -import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; -import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; -import { setUpTelemetry } from "./utils/set-up-telemetry"; -import { DIFF_CACHE_SIZE_MB } from "./consts"; +import { SyncService } from "./services/sync-service"; import { ServerConfig } from "./services/server-config"; -import type { EventListeners } from "./utils/data-structures/event-listeners"; +import { WebSocketManager } from "./services/websocket-manager"; +import { Syncer } from "./sync-operations/syncer"; +import { CursorTracker } from "./sync-operations/cursor-tracker"; +import { FileChangeNotifier } from "./sync-operations/file-change-notifier"; +import { createClientId } from "./utils/create-client-id"; +import { DocumentSyncStatus } from "./types/document-sync-status"; +import type { CursorSpan } from "./services/types/CursorSpan"; +import { isInternalPath } from "./utils/portable-path"; +import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; +import { globsToRegexes } from "./utils/globs-to-regexes"; +import { Lock } from "./utils/data-structures/locks"; +import type { LocalChange } from "./sync-operations/local-changes"; + +type StoredClient = Partial<{ + settings: Partial; + database: Partial; + localChanges: LocalChange[]; + localChangesVaultKey: string; + historyCheckpoint: { vaultKey: string; checkpoint?: string }; +}>; + +const vaultKey = (settings: SyncSettings): string => + JSON.stringify([ + settings.remoteUri.replace(/\/$/u, ""), + settings.vaultName + ]); export class SyncClient { - private hasStartedOfflineSync = false; - private hasFinishedOfflineSync = false; - private hasStarted = false; - private hasBeenDestroyed = false; + private readonly lifecycle = new Lock(); + private destroying?: Promise; + private started = false; + private destroyed = false; private unloadTelemetry?: () => void; - private isDestroying = false; - private readonly eventUnsubscribers: (() => void)[] = []; - + private readonly unsubscribeLog: () => void; private constructor( + public readonly logger: Logger, private readonly history: SyncHistory, private readonly settings: Settings, private readonly database: Database, private readonly syncer: Syncer, private readonly webSocketManager: WebSocketManager, - public readonly logger: Logger, private readonly fetchController: FetchController, + private readonly serverConfig: ServerConfig, private readonly cursorTracker: CursorTracker, private readonly fileChangeNotifier: FileChangeNotifier, private readonly contentCache: FixedSizeDocumentCache, - private readonly fileOperations: FileOperations, - private readonly serverConfig: ServerConfig, - private readonly persistence: PersistenceProvider< - Partial<{ - settings: Partial; - database: Partial; - }> - > - ) {} + private readonly persistence: PersistenceProvider + ) { + syncer.onServerHistoryChanged.add(() => { + cursorTracker.reset(); + }); + syncer.onReadyForEvents.add(() => { + if ( + this.started && + !this.destroyed && + settings.getSettings().isSyncEnabled + ) + webSocketManager.start(); + }); + this.unsubscribeLog = logger.onLogEmitted.add((log) => { + if (log.level === LogLevel.ERROR && Sentry.isInitialized()) + Sentry.captureMessage(log.message); + }); + } + + public static async create({ + fs, + persistence, + fetch, + webSocket + }: { + fs: FileSystemOperations; + persistence: PersistenceProvider; + fetch?: typeof globalThis.fetch; + webSocket?: typeof globalThis.WebSocket; + /** V4 preserves bytes and line endings as stored, on every platform. */ + nativeLineEndings?: string; + }): Promise { + const logger = new Logger(); + let stored = (await persistence.load()) ?? {}; + let saving: Promise = Promise.resolve(); + let reloadBeforeSave = false; + // Settings and engine state share a store. Serialize complete durable + // replacements so one cannot overwrite a newer save from the other. + const save = async (update: StoredClient): Promise => { + const operation = saving.then(async () => { + if (reloadBeforeSave) { + stored = (await persistence.load()) ?? {}; + reloadBeforeSave = false; + } + const next = { ...stored, ...structuredClone(update) }; + try { + await persistence.save(next); + stored = next; + } catch (error) { + // A failed save may have committed. Do not let a queued + // settings/notification save roll back that engine state. + reloadBeforeSave = true; + throw error; + } + }); + saving = operation.catch(() => {}); + return operation; + }; + const settings = new Settings(logger, stored.settings, async (data) => + save({ settings: data }) + ); + const database = new Database( + logger, + stored.database, + async (data) => save({ database: data }), + vaultKey(settings.getSettings()), + async () => { + await saving; + stored = (await persistence.load()) ?? {}; + return stored.database as StoredDatabase | undefined; + } + ); + if ( + stored.localChanges?.length && + stored.localChangesVaultKey !== vaultKey(settings.getSettings()) + ) { + throw new Error( + "Offline notifications belong to another vault; use a separate state store" + ); + } + const fetchController = new FetchController( + settings.getSettings().isSyncEnabled, + logger + ); + const deviceId = createClientId(); + const service = new SyncService( + deviceId, + fetchController, + settings, + logger, + fetch, + { + get: () => + stored.historyCheckpoint?.vaultKey === + vaultKey(settings.getSettings()) + ? stored.historyCheckpoint.checkpoint + : undefined, + save: async (checkpoint) => + save({ + historyCheckpoint: { + vaultKey: vaultKey(settings.getSettings()), + checkpoint + } + }) + } + ); + const serverConfig = new ServerConfig(service); + const notifier = new FileChangeNotifier(); + const localChanges = stored.localChanges ?? []; + let syncer: Syncer; + const files = new FileOperations( + fs, + database, + serverConfig, + (paths) => { + for (const path of paths) notifier.notifyOfFileChange(path); + }, + { + entries: () => localChanges, + flush: async () => syncer?.flushLocalChanges() + }, + (path, size) => { + const current = settings.getSettings(); + return ( + size > current.maxFileSizeMB * 1024 * 1024 || + globsToRegexes(current.ignorePatterns, logger).some( + (pattern) => pattern.test(path) + ) + ); + } + ); + const websocket = new WebSocketManager( + deviceId, + logger, + settings, + webSocket + ); + const history = new SyncHistory(logger); + const contentCache = new FixedSizeDocumentCache( + settings.getSettings().diffCacheSizeMB * 1024 * 1024 + ); + syncer = new Syncer( + deviceId, + logger, + database, + settings, + service, + websocket, + files, + serverConfig, + history, + contentCache, + { + entries: localChanges, + save: async (entries) => + save({ + localChanges: entries, + localChangesVaultKey: vaultKey(settings.getSettings()) + }) + } + ); + const cursors = new CursorTracker(database, websocket, files, notifier); + return new SyncClient( + logger, + history, + settings, + database, + syncer, + websocket, + fetchController, + serverConfig, + cursors, + notifier, + contentCache, + persistence + ); + } + + private check(): void { + if (this.destroyed) throw new Error("SyncClient has been destroyed"); + } public get documentCount(): number { return this.database.length; @@ -66,478 +250,229 @@ export class SyncClient { return this.webSocketManager.isWebSocketConnected; } - public get onSyncHistoryUpdated(): EventListeners< - (stats: HistoryStats) => unknown - > { - this.checkIfDestroyed("onSyncHistoryUpdated getter"); + public get onSyncHistoryUpdated() { return this.history.onHistoryUpdated; } - public get onSettingsChanged(): EventListeners< - (newSettings: SyncSettings, oldSettings: SyncSettings) => unknown - > { - this.checkIfDestroyed("onSettingsChanged getter"); + public get onSettingsChanged() { return this.settings.onSettingsChanged; } - public get onRemainingOperationsCountChanged(): EventListeners< - (remainingOperationsCount: number) => unknown - > { - this.checkIfDestroyed("onRemainingOperationsCountChanged getter"); + public get onRemainingOperationsCountChanged() { return this.syncer.onRemainingOperationsCountChanged; } - public get onWebSocketStatusChanged(): EventListeners< - (isConnected: boolean) => unknown - > { - this.checkIfDestroyed("onWebSocketStatusChanged getter"); + public get onWebSocketStatusChanged() { return this.webSocketManager.onWebSocketStatusChanged; } - public get onRemoteCursorsUpdated(): EventListeners< - (cursors: MaybeOutdatedClientCursors[]) => unknown - > { - this.checkIfDestroyed("onRemoteCursorsUpdated getter"); + public get onRemoteCursorsUpdated() { return this.cursorTracker.onRemoteCursorsUpdated; } - public static async create({ - fs, - persistence, - fetch, - webSocket, - nativeLineEndings = "\n" - }: { - fs: FileSystemOperations; - persistence: PersistenceProvider< - Partial<{ - settings: Partial; - database: Partial; - }> - >; - fetch?: typeof globalThis.fetch; - webSocket?: typeof globalThis.WebSocket; - nativeLineEndings?: string; - }): Promise { - const logger = new Logger(); - - const deviceId = createClientId(); - - logger.info(`Creating SyncClient with client id ${deviceId}`); - - const history = new SyncHistory(logger); - - let state = (await persistence.load()) ?? { - settings: undefined, - database: undefined - }; - - const settings = new Settings( - logger, - state.settings, - async (data): Promise => { - state = { ...state, settings: data }; - // we're not rate-limiting settings saves as (1) we need to initialise the settings to know the rate limit - // and (2) settings changes are infrequent enough that rate-limiting is not necessary - await persistence.save(state); - } - ); - - const rateLimitedSave = rateLimit( - persistence.save, - () => settings.getSettings().minimumSaveIntervalMs - ); - - const database = new Database( - logger, - state.database, - async (data): Promise => { - state = { ...state, database: data }; - await rateLimitedSave(state); - } - ); - - const fetchController = new FetchController( - settings.getSettings().isSyncEnabled, - logger - ); - - const syncService = new SyncService( - deviceId, - fetchController, - settings, - logger, - fetch - ); - - const serverConfig = new ServerConfig(syncService); - - const fileOperations = new FileOperations( - logger, - database, - fs, - serverConfig, - nativeLineEndings - ); - - const contentCache = new FixedSizeDocumentCache( - 1024 * 1024 * DIFF_CACHE_SIZE_MB - ); - const unrestrictedSyncer = new UnrestrictedSyncer( - logger, - database, - settings, - syncService, - fileOperations, - history, - contentCache, - serverConfig - ); - - const webSocketManager = new WebSocketManager( - deviceId, - logger, - settings, - webSocket - ); - - const syncer = new Syncer( - deviceId, - logger, - database, - settings, - syncService, - webSocketManager, - fileOperations, - unrestrictedSyncer - ); - - const fileChangeNotifier = new FileChangeNotifier(); - const cursorTracker = new CursorTracker( - database, - webSocketManager, - fileOperations, - fileChangeNotifier - ); - const client = new SyncClient( - history, - settings, - database, - syncer, - webSocketManager, - logger, - fetchController, - cursorTracker, - fileChangeNotifier, - contentCache, - fileOperations, - serverConfig, - persistence - ); - - logger.info("SyncClient created successfully"); - - return client; - } - - public async start(): Promise { - this.checkIfDestroyed("start"); - - if (this.hasStarted) { - throw new Error("SyncClient has already been started"); - } - this.hasStarted = true; - - if ( - !this.unloadTelemetry && - this.settings.getSettings().enableTelemetry - ) { - this.unloadTelemetry = setUpTelemetry(); - } - - this.eventUnsubscribers.push( - this.settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (oldSettings.isSyncEnabled != newSettings.isSyncEnabled) { - this.fetchController.canFetch = newSettings.isSyncEnabled; - } - }) - ); - - this.eventUnsubscribers.push( - this.logger.onLogEmitted.add((log): void => { - if (log.level === LogLevel.ERROR && Sentry.isInitialized()) { - Sentry.captureMessage(log.message); - } - }) - ); - - this.eventUnsubscribers.push( - this.settings.onSettingsChanged.add( - this.onSettingsChange.bind(this) - ) - ); - - if (this.settings.getSettings().isSyncEnabled) { - this.logger.info("Starting SyncClient"); - await this.startSyncing(); - this.logger.info("SyncClient has successfully started"); - } - } - - /** - * Reload settings from disk overriding current in-memory settings. - * Missing values will be filled in from DEFAULT_SETTINGS rather than - * retaining current in-memory settings. - */ - public async reloadSettings(): Promise { - this.checkIfDestroyed("reloadSettings"); - - const state = (await this.persistence.load()) ?? { - settings: undefined - }; - - const settings = { - ...DEFAULT_SETTINGS, - ...(state.settings ?? {}) - }; - - await this.setSettings(settings); - } - - public async checkConnection(): Promise { - this.checkIfDestroyed("checkConnection"); - - const server = await this.serverConfig.checkConnection(true); - return { - isSuccessful: server.isSuccessful, - serverMessage: server.message, - isWebSocketConnected: this.webSocketManager.isWebSocketConnected - }; - } - - public getHistoryEntries(): readonly HistoryEntry[] { + public getHistoryEntries() { return this.history.entries; } - /** - * Wait for the in-flight operations to finish, reset all tracking, - * and the local database but retain the settings. - * The SyncClient can be used again after calling this method. - */ - public async reset(): Promise { - this.checkIfDestroyed("reset"); - - this.logger.info( - "Stopping SyncClient to apply changed connection settings" - ); - await this.pause(); - - // clear all local state - this.logger.info("Resetting SyncClient's local state"); - this.database.reset(); - await this.database.save(); // ensure the new database reads as empty - this.resetInMemoryState(); - this.hasStartedOfflineSync = false; - this.hasFinishedOfflineSync = false; - this.serverConfig.reset(); - - await this.startSyncing(); - } - public getSettings(): SyncSettings { return this.settings.getSettings(); } - public async setSetting( - key: T, - value: SyncSettings[T] - ): Promise { - this.checkIfDestroyed("setSetting"); - - await this.settings.setSetting(key, value); - } - - public async setSettings(value: Partial): Promise { - this.checkIfDestroyed("setSettings"); - - await this.settings.setSettings(value); - } - - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { - this.checkIfDestroyed("syncLocallyCreatedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyCreatedFile(relativePath); - } - - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - this.checkIfDestroyed("syncLocallyDeletedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyDeletedFile(relativePath); - } - - public async syncLocallyUpdatedFile({ - oldPath, - relativePath - }: { - oldPath?: RelativePath; - relativePath: RelativePath; - }): Promise { - this.checkIfDestroyed("syncLocallyUpdatedFile"); - - this.fileChangeNotifier.notifyOfFileChange(relativePath); - return this.syncer.syncLocallyUpdatedFile({ - oldPath, - relativePath - }); - } - - public getDocumentSyncingStatus( - relativePath: RelativePath - ): DocumentSyncStatus { - this.checkIfDestroyed("getDocumentSyncingStatus"); - - if (!this.settings.getSettings().isSyncEnabled) { - return DocumentSyncStatus.SYNCING_IS_DISABLED; - } - - if (!this.syncer.isFirstSyncComplete || !this.hasFinishedOfflineSync) { - return DocumentSyncStatus.SYNCING; - } - - const document = - this.database.getLatestDocumentByRelativePath(relativePath); - if (document === undefined) { - return DocumentSyncStatus.SYNCING; - } - return document.updates.length > 0 - ? DocumentSyncStatus.SYNCING - : DocumentSyncStatus.UP_TO_DATE; - } - - public async updateLocalCursors( - documentToCursors: Record - ): Promise { - this.checkIfDestroyed("updateLocalCursors"); - - await this.cursorTracker.sendLocalCursorsToServer(documentToCursors); - } - - public async waitUntilFinished(): Promise { - this.checkIfDestroyed("waitUntilIdle"); - await this.syncer.waitUntilFinished(); - await this.webSocketManager.waitUntilFinished(); - await this.database.save(); // flush all changes to disk - } - - /** - * Completely destroy the SyncClient, cancelling all in-progress operations. - * After calling this method, the SyncClient cannot be used again. - */ - public async destroy(): Promise { - this.checkIfDestroyed("destroy"); - - // Prevent concurrent destroy calls - if (this.isDestroying) { - this.logger.warn( - "destroy() called while already destroying, ignoring" + public async start(): Promise { + await this.lifecycle.withLock(async () => { + this.check(); + if (this.started) + throw new Error("SyncClient has already been started"); + await this.syncer.recoverLocalState(); + await this.database.bindVault( + vaultKey(this.settings.getSettings()) ); - return; - } - this.isDestroying = true; - - // cancel everything that's in progress - await this.pause(); - - this.hasBeenDestroyed = true; - - this.resetInMemoryState(); - - // Clean up event listeners to prevent memory leaks - this.eventUnsubscribers.forEach((unsubscribe) => { - unsubscribe(); + this.started = true; + this.configureTelemetry(); + if (this.settings.getSettings().isSyncEnabled) await this.resume(); }); - this.eventUnsubscribers.length = 0; - - this.logger.info("SyncClient has been successfully disposed"); - - this.unloadTelemetry?.(); } - private async startSyncing(): Promise { - this.checkIfDestroyed("startSyncing"); + private async resume(): Promise { + if (this.destroyed) return; + if (!this.settings.getSettings().isSyncEnabled) return; + await this.database.bindVault(vaultKey(this.settings.getSettings())); this.fetchController.finishReset(); - - await this.serverConfig.initialize(); - this.webSocketManager.start(); - - if (!this.hasStartedOfflineSync) { - this.hasStartedOfflineSync = true; - await this.syncer.scheduleSyncForOfflineChanges(); - } - - this.hasFinishedOfflineSync = true; + this.fetchController.canFetch = true; + this.syncer.start(); + await this.syncer.waitUntilFinished(); } private async pause(): Promise { + const stopped = this.syncer.stop(); this.fetchController.startReset(); await this.webSocketManager.stop(); - await this.waitUntilFinished(); + await stopped; } - private resetInMemoryState(): void { - this.history.reset(); - this.contentCache.reset(); - // don't reset the logger - this.cursorTracker.reset(); - this.syncer.reset(); - this.fileOperations.reset(); + /** Restart transports; never discard pending requests or filesystem journals. */ + public async reset(): Promise { + await this.lifecycle.withLock(async () => { + this.check(); + await this.pause(); + this.serverConfig.reset(); + this.cursorTracker.reset(); + await this.syncer.retryRejectedRequests(); + if (this.started && this.settings.getSettings().isSyncEnabled) + await this.resume(); + }); } - private async onSettingsChange( - newSettings: SyncSettings, - oldSettings: SyncSettings - ): Promise { - this.checkIfDestroyed("onSettingsChange"); + public async setSettings(change: Partial): Promise { + const update = structuredClone(change); + const notification = await this.lifecycle.withLock(async () => { + this.check(); + const old = this.settings.getSettings(); + const next = { ...old, ...update }; + if ( + vaultKey(next) !== vaultKey(old) && + (this.started || + this.database.state.initialized || + this.database.state.bootstrap || + this.database.state.pending || + this.database.state.application || + this.syncer.hasLocalChanges || + Object.keys(this.database.state.local).length) + ) + throw new Error( + "Changing vaults requires a separate state store and client; pending recovery data must not be reset" + ); - if ( - newSettings.vaultName !== oldSettings.vaultName || - newSettings.remoteUri !== oldSettings.remoteUri - ) { - await this.reset(); - } - - if (newSettings.isSyncEnabled !== oldSettings.isSyncEnabled) { - if (newSettings.isSyncEnabled) { - await this.startSyncing(); - } else { - await this.pause(); + try { + if (this.started) await this.pause(); + await this.database.recoverPersistence(); + await this.settings.setSettings(update, false); + await this.syncer.retryRejectedRequests(); + this.contentCache.resize(next.diffCacheSizeMB * 1024 * 1024); + await this.database.bindVault(vaultKey(next)); + } catch (error) { + this.serverConfig.reset(); + this.contentCache.resize( + this.settings.getSettings().diffCacheSizeMB * 1024 * 1024 + ); + this.configureTelemetry(); + if (this.started && this.settings.getSettings().isSyncEnabled) + await this.resume().catch((resumeError: unknown) => { + this.logger.error( + `Settings recovery could not resume sync: ${String(resumeError)}` + ); + }); + throw error; } - } + this.serverConfig.reset(); + this.configureTelemetry(); + if (this.started && next.isSyncEnabled) await this.resume(); + return [this.settings.getSettings(), old] as const; + }); + await this.settings.onSettingsChanged.triggerAsync(...notification); + } - if (newSettings.diffCacheSizeMB !== oldSettings.diffCacheSizeMB) { - this.contentCache.resize(newSettings.diffCacheSizeMB * 1024 * 1024); - } + public async setSetting( + key: K, + value: SyncSettings[K] + ): Promise { + await this.setSettings({ [key]: value }); + } - if (newSettings.enableTelemetry !== oldSettings.enableTelemetry) { - if (newSettings.enableTelemetry) { - this.unloadTelemetry = setUpTelemetry(); - } else { + public async reloadSettings(): Promise { + const stored = await this.persistence.load(); + await this.setSettings({ ...DEFAULT_SETTINGS, ...stored?.settings }); + } + + public async checkConnection() { + const result = await this.serverConfig.checkConnection(true); + return { + isSuccessful: result.isSuccessful, + serverMessage: result.message, + isWebSocketConnected: this.isWebSocketConnected + }; + } + + /** Notifications enqueue work. Use waitUntilFinished() to await convergence. */ + public async syncLocallyCreatedFile(path: string): Promise { + this.check(); + if (isInternalPath(path)) return; + this.fileChangeNotifier.notifyOfFileChange(path); + await this.syncer.syncLocallyCreatedFile(path); + } + + public async syncLocallyDeletedFile(path: string): Promise { + this.check(); + if (isInternalPath(path)) return; + this.fileChangeNotifier.notifyOfFileChange(path); + await this.syncer.syncLocallyDeletedFile(path); + } + + public async syncLocallyUpdatedFile(change: { + oldPath?: string; + relativePath: string; + }): Promise { + this.check(); + if ( + isInternalPath(change.relativePath) || + (change.oldPath && isInternalPath(change.oldPath)) + ) + return; + this.fileChangeNotifier.notifyOfFileChange(change.relativePath); + await this.syncer.syncLocallyUpdatedFile(change); + } + + public async updateLocalCursors( + cursors: Record + ): Promise { + this.check(); + await this.cursorTracker.sendLocalCursorsToServer(cursors); + } + + public getDocumentSyncingStatus(path: string): DocumentSyncStatus { + this.check(); + if (!this.settings.getSettings().isSyncEnabled) + return DocumentSyncStatus.SYNCING_IS_DISABLED; + return this.syncer.isDocumentUpToDate(path) + ? DocumentSyncStatus.UP_TO_DATE + : DocumentSyncStatus.SYNCING; + } + + public async waitUntilFinished(): Promise { + this.check(); + await this.syncer.waitUntilFinished(); + } + + private configureTelemetry(): void { + if ( + this.settings.getSettings().enableTelemetry && + typeof window !== "undefined" + ) { + this.unloadTelemetry ??= setUpTelemetry(); + } else { + this.unloadTelemetry?.(); + this.unloadTelemetry = undefined; + } + } + + public async destroy(): Promise { + if (this.destroying) return this.destroying; + this.destroyed = true; + // Interrupt a start/reset awaiting network progress before waiting for + // its lifecycle operation to finish. + this.fetchController.startReset(); + const stopped = this.syncer.stop(); + this.destroying = this.lifecycle.withLock(async () => { + try { + await this.webSocketManager.stop(); + await stopped; + await this.syncer.flushLocalChanges(); + } finally { + this.cursorTracker.reset(); + this.unsubscribeLog(); this.unloadTelemetry?.(); } - } - } - - private checkIfDestroyed(origin: string): void { - if (this.hasBeenDestroyed) { - throw new Error( - `SyncClient has been destroyed and can no longer be used; called from ${origin}` - ); - } + }); + return this.destroying; } } diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts index 01eac85a..8828d887 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts @@ -23,6 +23,9 @@ it("retains future cursors and publishes them when content catches up", async () }) } as unknown as Database; const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), onRemoteCursorsUpdateReceived: new EventListeners< (cursors: ClientCursors[]) => Promise >(), @@ -65,3 +68,209 @@ it("retains future cursors and publishes them when content catches up", async () "note.md" ); }); + +it("refreshes unchanged cursors on reconnect and heartbeat, and follows the document through renames", async (context) => { + context.mock.timers.enable({ apis: ["setInterval"] }); + const content = new TextEncoder().encode("hello"); + let path = "note.md"; + const record = () => ({ + documentId: "document", + relativePath: path, + metadata: { parentVersionId: 1, hash: contentHash } + }); + const contentHash = await hash(content); + const database = { + getLatestDocumentByRelativePath: (candidate: string) => + candidate === path ? record() : undefined, + getDocumentByDocumentId: () => record() + } as unknown as Database; + const sent: { + documentsWithCursors: { + relative_path: string; + vault_update_id: number | null; + }[]; + }[] = []; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: (value: (typeof sent)[number]) => { + sent.push(value); + } + } as unknown as WebSocketManager; + const notifier = new FileChangeNotifier(); + const tracker = new CursorTracker( + database, + websocket, + { read: async () => content } as unknown as FileOperations, + notifier + ); + const drain = async () => { + for (let i = 0; i < 20; i++) + await new Promise((resolve) => setImmediate(resolve)); + }; + try { + await tracker.sendLocalCursorsToServer({ + "note.md": [{ start: 1, end: 2 }] + }); + assert.equal(sent.length, 1); + await tracker.sendLocalCursorsToServer({ + "note.md": [{ start: 1, end: 2 }] + }); + assert.equal(sent.length, 1); + websocket.onWebSocketStatusChanged.trigger(true); + await drain(); + assert.equal(sent.length, 2); + context.mock.timers.tick(15_000); + await drain(); + assert.equal(sent.length, 3); + path = "renamed.md"; + await notifier.onFileChanged.triggerAsync(path); + assert.equal(sent.at(-1)!.documentsWithCursors[0].relative_path, path); + websocket.onWebSocketStatusChanged.trigger(false); + const count = sent.length; + context.mock.timers.tick(30_000); + await drain(); + assert.equal(sent.length, count); + websocket.onWebSocketStatusChanged.trigger(true); + await drain(); + assert.equal(sent.length, count + 1); + } finally { + tracker.reset(); + context.mock.timers.reset(); + } +}); + +for (const interruption of ["disconnect", "reset"] as const) { + it(`an in-flight cursor read cannot resurrect presence after ${interruption}`, async () => { + const content = new TextEncoder().encode("current"); + const contentHash = await hash(content); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const database = { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata: { parentVersionId: 1, hash: contentHash } + }) + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const tracker = new CursorTracker( + database, + websocket, + { + read: async () => { + entered.resolve(); + await release.promise; + return content; + } + } as unknown as FileOperations, + new FileChangeNotifier() + ); + let received: ClientCursors[] = []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + received = cursors; + }); + const cursor: ClientCursors = { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 1 }] + } + ] + }; + const first = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + cursor + ]); + await entered.promise; + const queued = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + cursor + ]); + if (interruption === "reset") tracker.reset(); + else websocket.onWebSocketStatusChanged.trigger(false); + assert.deepEqual(received, []); + release.resolve(); + await first; + await queued; + assert.deepEqual( + received, + [], + "a retired connection must not restore its cursors" + ); + tracker.reset(); + }); +} + +it("a remote cursor becomes outdated if the document advances during its snapshot read", async () => { + const content = new TextEncoder().encode("old"); + let metadata = { parentVersionId: 1, hash: await hash(content) }; + const entered = Promise.withResolvers(), + release = Promise.withResolvers(); + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteCursorsUpdateReceived: new EventListeners< + (cursors: ClientCursors[]) => Promise + >(), + updateLocalCursors: () => {} + } as unknown as WebSocketManager; + const tracker = new CursorTracker( + { + getDocumentByDocumentId: () => ({ + documentId: "document", + relativePath: "note.md", + metadata + }) + } as unknown as Database, + websocket, + { + read: async () => { + entered.resolve(); + await release.promise; + return content; + } + } as unknown as FileOperations, + new FileChangeNotifier() + ); + let outdated: boolean | undefined; + tracker.onRemoteCursorsUpdated.add((cursors) => { + outdated = cursors[0]?.isOutdated; + }); + const update = websocket.onRemoteCursorsUpdateReceived.triggerAsync([ + { + userName: "Other", + deviceId: "device", + documentsWithCursors: [ + { + document_id: "document", + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 1 }] + } + ] + } + ]); + await entered.promise; + metadata = { + parentVersionId: 2, + hash: await hash(new TextEncoder().encode("new content")) + }; + release.resolve(); + await update; + assert.equal(outdated, true); + tracker.reset(); +}); diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.ts index e3c63cd4..a6170195 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.ts @@ -23,6 +23,7 @@ export class CursorTracker { >(); private readonly updateLock = new Lock(); + private sessionGeneration = 0; private knownRemoteCursors: (ClientCursors & { upToDateness: DocumentUpToDateness; @@ -31,6 +32,10 @@ export class CursorTracker { private lastLocalCursorState: DocumentWithCursors[] = []; private lastLocalCursorStateWithoutDirtyDocuments: DocumentWithCursors[] = []; + private localCursors: Record = {}; + private localCursorDocuments: Record = {}; + private cursorTimer?: ReturnType; + private readonly sendLock = new Lock(); public constructor( private readonly database: Database, @@ -38,9 +43,31 @@ export class CursorTracker { private readonly fileOperations: FileOperations, private readonly fileChangeNotifier: FileChangeNotifier ) { + this.webSocketManager.onWebSocketStatusChanged.add((connected) => { + this.sessionGeneration++; + clearInterval(this.cursorTimer); + this.lastLocalCursorState = []; + this.lastLocalCursorStateWithoutDirtyDocuments = []; + if (!connected) { + this.knownRemoteCursors = []; + this.onRemoteCursorsUpdated.trigger([]); + return; + } + const refresh = () => { + void this.publishLocalCursors(true).catch(() => { + // A concurrent move/deletion can invalidate an editor view; + // the next notification or heartbeat retries the snapshot. + }); + }; + refresh(); + this.cursorTimer = setInterval(refresh, 15_000); + this.cursorTimer.unref?.(); + }); this.webSocketManager.onRemoteCursorsUpdateReceived.add( async (clientCursors) => { + const generation = this.sessionGeneration; await this.updateLock.withLock(async () => { + if (generation !== this.sessionGeneration) return; // The latest message will contain all active clients, so we can delete the ones // from the local list which are no longer active. const allIds = new Set( @@ -51,11 +78,7 @@ export class CursorTracker { allIds.has(c.deviceId) ); - for (const cursor of clientCursors.filter((client) => - client.documentsWithCursors.every( - (doc) => doc.vault_update_id != null - ) - )) { + for (const cursor of clientCursors) { updatedKnownRemoteCursors.push({ ...cursor, upToDateness: @@ -63,17 +86,20 @@ export class CursorTracker { }); } + if (generation !== this.sessionGeneration) return; this.knownRemoteCursors = updatedKnownRemoteCursors; + this.onRemoteCursorsUpdated.trigger( + this.getRelevantAndPruneKnownClientCursors() + ); }); - - this.onRemoteCursorsUpdated.trigger( - this.getRelevantAndPruneKnownClientCursors() - ); } ); this.fileChangeNotifier.onFileChanged.add(async (relativePath) => { + const generation = this.sessionGeneration; + await this.publishLocalCursors(true); const cursors = await this.updateLock.withLock(async () => { + if (generation !== this.sessionGeneration) return []; for (const clientCursor of this.knownRemoteCursors) { if ( clientCursor.documentsWithCursors.some( @@ -88,10 +114,13 @@ export class CursorTracker { await this.getDocumentsUpToDateness(clientCursor); } } - return this.getRelevantAndPruneKnownClientCursors(); + return generation === this.sessionGeneration + ? this.getRelevantAndPruneKnownClientCursors() + : []; }); - this.onRemoteCursorsUpdated.trigger(cursors); + if (generation === this.sessionGeneration) + this.onRemoteCursorsUpdated.trigger(cursors); }); } @@ -101,71 +130,106 @@ export class CursorTracker { public async sendLocalCursorsToServer( documentToCursors: Record ): Promise { - const documentsWithCursors: DocumentWithCursors[] = []; + this.localCursors = structuredClone(documentToCursors); + this.localCursorDocuments = {}; + await this.publishLocalCursors(false); + } - for (const [relativePath, cursors] of Object.entries( - documentToCursors - )) { - const record = - this.database.getLatestDocumentByRelativePath(relativePath); + private async publishLocalCursors(force: boolean): Promise { + const generation = this.sessionGeneration; + await this.sendLock.withLock(async () => { + if (generation !== this.sessionGeneration) return; + const documentsWithCursors: DocumentWithCursors[] = []; - if (!record) { - continue; // Let's wait for the file to be created before sending cursors + for (const [editorPath, cursors] of Object.entries( + this.localCursors + )) { + const id = this.localCursorDocuments[editorPath]; + const record = id + ? this.database.getDocumentByDocumentId(id) + : this.database.getLatestDocumentByRelativePath(editorPath); + + if (!record) { + continue; // Let's wait for the file to be created before sending cursors + } + this.localCursorDocuments[editorPath] = record.documentId; + const { relativePath } = record; + + if (!record.metadata) { + continue; // this is a new document, no need to sync the cursors + } + + documentsWithCursors.push({ + relative_path: relativePath, + document_id: record.documentId, + vault_update_id: record.metadata.parentVersionId, + cursors: cursors.map(({ start, end }) => ({ + start: Math.min(start, end), + end: Math.max(start, end) + })) // the client might send directional selections + }); } - if (!record.metadata) { - continue; // this is a new document, no need to sync the cursors + if ( + !force && + JSON.stringify(this.lastLocalCursorState) === + JSON.stringify(documentsWithCursors) + ) { + // Caching step to avoid reading the edited files all the time + return; + } + this.lastLocalCursorState = structuredClone(documentsWithCursors); + + for (const doc of documentsWithCursors) { + let readContent: Uint8Array; + try { + readContent = await this.fileOperations.read( + doc.relative_path + ); + } catch { + doc.vault_update_id = null; + continue; + } + const digest = await hash(readContent); + const record = this.database.getLatestDocumentByRelativePath( + doc.relative_path + ); + if ( + record?.documentId !== doc.document_id || + record.relativePath !== doc.relative_path || + record.metadata?.parentVersionId !== doc.vault_update_id || + record.metadata.hash !== digest + ) { + doc.vault_update_id = null; + } } - documentsWithCursors.push({ - relative_path: relativePath, - document_id: record.documentId, - vault_update_id: record.metadata.parentVersionId, - cursors: cursors.map(({ start, end }) => ({ - start: Math.min(start, end), - end: Math.max(start, end) - })) // the client might send directional selections - }); - } - - if ( - JSON.stringify(this.lastLocalCursorState) === - JSON.stringify(documentsWithCursors) - ) { - // Caching step to avoid reading the edited files all the time - return; - } - this.lastLocalCursorState = documentsWithCursors; - - for (const doc of documentsWithCursors) { - const readContent = await this.fileOperations.read( - doc.relative_path - ); - const record = this.database.getLatestDocumentByRelativePath( - doc.relative_path - ); - if (record?.metadata?.hash !== (await hash(readContent))) { - doc.vault_update_id = null; + if (generation !== this.sessionGeneration) return; + if ( + !force && + JSON.stringify( + this.lastLocalCursorStateWithoutDirtyDocuments + ) === JSON.stringify(documentsWithCursors) + ) { + return; } - } - if ( - JSON.stringify(this.lastLocalCursorStateWithoutDirtyDocuments) === - JSON.stringify(documentsWithCursors) - ) { - return; - } + this.lastLocalCursorStateWithoutDirtyDocuments = + documentsWithCursors; - this.lastLocalCursorStateWithoutDirtyDocuments = documentsWithCursors; - - this.webSocketManager.updateLocalCursors({ documentsWithCursors }); + this.webSocketManager.updateLocalCursors({ documentsWithCursors }); + }); } public reset(): void { + this.sessionGeneration++; + clearInterval(this.cursorTimer); this.knownRemoteCursors = []; + this.onRemoteCursorsUpdated.trigger([]); this.lastLocalCursorState = []; this.lastLocalCursorStateWithoutDirtyDocuments = []; - this.updateLock.reset(); + // Keep serialization intact while old reads drain. A generation fence + // discards both active and queued work from the retired connection. } private getRelevantAndPruneKnownClientCursors(): MaybeOutdatedClientCursors[] { @@ -245,6 +309,8 @@ export class CursorTracker { private async getDocumentUpToDateness( document: DocumentWithCursors ): Promise { + if (document.vault_update_id === null) + return DocumentUpToDateness.Prior; const record = this.database.getDocumentByDocumentId( document.document_id ); @@ -255,13 +321,11 @@ export class CursorTracker { } if ( - (record.metadata?.parentVersionId ?? 0) < - (document.vault_update_id ?? 0) + (record.metadata?.parentVersionId ?? 0) < document.vault_update_id ) { return DocumentUpToDateness.Later; } else if ( - (document.vault_update_id ?? 0) < - (record.metadata?.parentVersionId ?? 0) + document.vault_update_id < (record.metadata?.parentVersionId ?? 0) ) { // the document of the cursor must be from the past return DocumentUpToDateness.Prior; @@ -276,7 +340,14 @@ export class CursorTracker { return DocumentUpToDateness.Prior; } - return record.metadata?.hash === (await hash(currentContent)) + const contentHash = await hash(currentContent); + const current = this.database.getDocumentByDocumentId( + document.document_id + ); + return current?.metadata?.parentVersionId === + document.vault_update_id && + current.relativePath === record.relativePath && + current.metadata.hash === contentHash ? DocumentUpToDateness.UpToDate : DocumentUpToDateness.Prior; } diff --git a/frontend/sync-client/src/sync-operations/file-manifest.ts b/frontend/sync-client/src/sync-operations/file-manifest.ts new file mode 100644 index 00000000..ad2d393a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/file-manifest.ts @@ -0,0 +1,69 @@ +import type { FileManifestEntries } from "../types/file-manifest-entries"; +import { + allocatePortablePath, + validatePortablePaths +} from "../utils/portable-path"; + +export function validateFileManifest(entries: FileManifestEntries): void { + validatePortablePaths(Object.values(entries)); +} + +export function sameFileManifest( + a: FileManifestEntries, + b: FileManifestEntries +): boolean { + return ( + Object.keys(a).length === Object.keys(b).length && + Object.keys(a).every((id) => a[id] === b[id]) + ); +} + +export function resolvePaths( + proposed: FileManifestEntries, + remote: FileManifestEntries, + protectedIds: readonly string[] = [], + reservedPaths: readonly string[] = [] +): FileManifestEntries { + const result: FileManifestEntries = {}; + for (const id of protectedIds) + if (proposed[id] !== undefined) result[id] = proposed[id]; + const occupied = { ...result }; + for (const [index, path] of reservedPaths.entries()) + if (!Object.values(result).includes(path)) + occupied[`reserved-${index}`] = path; + const ids = Object.keys(proposed).sort( + (a, b) => + Number(remote[b] === proposed[b]) - + Number(remote[a] === proposed[a]) || + (a < b ? -1 : a > b ? 1 : 0) + ); + for (const id of ids.filter( + (candidate) => result[candidate] === undefined + )) { + result[id] = allocatePortablePath(proposed[id], id, occupied); + occupied[id] = result[id]; + } + return result; +} + +export function mergeFileManifests( + base: FileManifestEntries, + local: FileManifestEntries, + remote: FileManifestEntries +): FileManifestEntries { + const proposed: FileManifestEntries = {}; + for (const id of new Set([ + ...Object.keys(base), + ...Object.keys(local), + ...Object.keys(remote) + ])) { + const path = + local[id] === base[id] + ? remote[id] + : remote[id] === base[id] + ? local[id] + : remote[id]; + if (path !== undefined) proposed[id] = path; + } + return resolvePaths(proposed, remote); +} diff --git a/frontend/sync-client/src/sync-operations/local-changes.ts b/frontend/sync-client/src/sync-operations/local-changes.ts new file mode 100644 index 00000000..6022f992 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/local-changes.ts @@ -0,0 +1,82 @@ +import { v4 as uuid } from "uuid"; +import type { EngineState } from "../persistence/database"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import { arePathAliases } from "../utils/portable-path"; + +export type LocalChange = { + changeId?: string; + /** Identities at notification time, before a remote application can reuse paths. */ + identities?: Record; +} & ( + | { type: "move"; oldPath: string; relativePath: string } + | { type: "create"; path: string; documentId?: string } + | { type: "delete"; path: string } + | { type: "update"; path: string } +); + +export function unappliedChanges( + state: EngineState, + changes: readonly LocalChange[] +): LocalChange[] { + for (const change of changes) change.changeId ??= uuid(); + const applied = changes.findIndex( + (change) => change.changeId === state.lastAppliedLocalChangeId + ); + return changes.slice(applied + 1); +} + +export function applyLocalChange( + next: EngineState, + change: LocalChange, + ignored: (path: string) => boolean, + initial?: VaultSnapshot +): void { + if (change.type === "create") { + if ( + ignored(change.path) || + Object.values(next.local).includes(change.path) + ) + return; + const id = (change.documentId ??= + (initial && + Object.keys(initial.fileManifest.entries).find( + (id) => + !next.local[id] && + arePathAliases( + initial.fileManifest.entries[id], + change.path + ) + )) || + uuid()); + next.local[id] = change.path; + next.documents[id] ??= { + materialized: true, + bootstrap: initial?.fileManifest.entries[id] !== undefined + }; + return; + } + const from = change.type === "move" ? change.oldPath : change.path; + change.identities ??= Object.fromEntries( + Object.entries(next.local).filter( + ([id, path]) => + next.documents[id]?.materialized && + (path === from || path.startsWith(from + "/")) + ) + ); + if (change.type === "update") return; + for (const [id, path] of Object.entries(change.identities)) { + if (change.type === "delete") { + if (ignored(path)) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = next.fileManifest.entries[id] ?? null; + } + delete next.local[id]; + } else { + const to = change.relativePath + path.slice(from.length); + for (const [other, otherPath] of Object.entries(next.local)) + if (other !== id && otherPath === to) delete next.local[other]; + next.local[id] = to; + } + } +} diff --git a/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts new file mode 100644 index 00000000..b50d2b8a --- /dev/null +++ b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { CursorTracker } from "./cursor-tracker"; +import { FileChangeNotifier } from "./file-change-notifier"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { hash } from "../utils/hash"; +import type { Database, DocumentRecord } from "../persistence/database"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { DocumentWithCursors } from "../services/types/DocumentWithCursors"; + +for (const change of ["version", "identity", "path", "delete"] as const) + test(`outgoing cursor is dirty when ${change} changes during its snapshot read`, async () => { + // Equal bytes deliberately defeat a freshness check that compares only hashes. + const content = new TextEncoder().encode("same bytes"); + const digest = await hash(content); + let current: DocumentRecord | undefined = { + documentId: "a", + relativePath: "a.md", + isDeleted: false, + metadata: { parentVersionId: 1, hash: digest } + }; + const sent: { documentsWithCursors: DocumentWithCursors[] }[] = []; + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: (data: { + documentsWithCursors: DocumentWithCursors[]; + }) => sent.push(data) + }; + const files = { + read: async () => { + if (change === "delete") current = undefined; + else if (change === "version") + current!.metadata!.parentVersionId = 2; + else if (change === "identity") + current!.documentId = "replacement"; + else current!.relativePath = "moved.md"; + return content; + } + }; + const tracker = new CursorTracker( + { + getDocumentByDocumentId: (id: string) => + current?.documentId === id ? current : undefined, + getLatestDocumentByRelativePath: (path: string) => + current?.relativePath === path ? current : undefined + } as unknown as Database, + websocket as unknown as WebSocketManager, + files as unknown as FileOperations, + new FileChangeNotifier() + ); + await tracker.sendLocalCursorsToServer({ + "a.md": [{ start: 4, end: 4 }] + }); + assert.equal(sent.length, 1); + assert.equal(sent[0].documentsWithCursors[0].vault_update_id, null); + tracker.reset(); + }); diff --git a/frontend/sync-client/src/sync-operations/scan.ts b/frontend/sync-client/src/sync-operations/scan.ts new file mode 100644 index 00000000..3e743231 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/scan.ts @@ -0,0 +1,152 @@ +import { v4 as uuid } from "uuid"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { EngineState } from "../persistence/database"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import { EMPTY_HASH, hash } from "../utils/hash"; +import { + arePathAliases, + findPathWithSameSpelling +} from "../utils/portable-path"; + +import { + applyLocalChange, + unappliedChanges, + type LocalChange +} from "./local-changes"; +export type { LocalChange } from "./local-changes"; + +interface ScanOptions { + next: EngineState; + guard: () => void; + changes: LocalChange[]; + files: FileOperations; + ignored: (path: string) => boolean; + oversized: (size: number) => boolean; + commit: (next: EngineState) => Promise; +} + +/** Scans are complete or fail. Missing unmaterialized/ignored entries are not deletions. */ +export async function scanLocalFiles( + { + next, + guard, + changes: queuedChanges, + files, + ignored, + oversized, + commit + }: ScanOptions, + initial?: VaultSnapshot +): Promise> { + // Leave the prefix queued until its resulting state is durably committed. + // A persisted change ID makes replay safe after an uncertain save. + const changes = [...queuedChanges]; + const unsyncablePaths = new Set(); + for (const change of unappliedChanges(next, changes)) { + applyLocalChange(next, change, ignored, initial); + next.lastAppliedLocalChangeId = change.changeId; + } + const paths: string[] = []; + const reportedPaths = await files.listFilesRecursively(); + for (const path of reportedPaths) { + // Some filesystems may report a different Unicode spelling after a + // rename. Keep an already-accessible spelling when possible. + const alias = findPathWithSameSpelling(Object.values(next.local), path); + paths.push( + alias && + !reportedPaths.includes(alias) && + (await files.fs.exists(alias)) + ? alias + : path + ); + } + const present = new Set(paths); + const snapshots = new Map(); + for (const path of paths) { + if (ignored(path)) { + unsyncablePaths.add(path); + continue; + } + const info = await files.fs.stat(path); + if (!info || info.kind !== "file") + throw new Error(`File changed during scan: ${path}`); + if (oversized(info.size)) { + unsyncablePaths.add(path); + continue; + } + const snapshot = await files.fs.readSnapshot(path); + if (!snapshot) throw new Error(`File changed during scan: ${path}`); + if (!oversized(snapshot.content.length)) { + snapshots.set(path, { hash: await hash(snapshot.content) }); + } else { + unsyncablePaths.add(path); + } + } + const missing = new Set( + Object.keys(next.local).filter( + (id) => + next.documents[id]?.materialized && + !ignored(next.local[id]) && + !present.has(next.local[id]) + ) + ); + const knownPaths = new Set(Object.values(next.local)); + const unknown = [...snapshots.keys()].filter( + (path) => !knownPaths.has(path) + ); + for (const path of unknown) { + const snapshot = snapshots.get(path)!; + const matches = [...missing].filter( + (id) => next.documents[id]?.observedHash === snapshot.hash + ); + const sameHash = unknown.filter( + (other) => snapshots.get(other)?.hash === snapshot.hash + ); + let id: string | undefined; + if ( + snapshot.hash !== EMPTY_HASH && + matches.length === 1 && + sameHash.length === 1 + ) { + id = matches[0]; + missing.delete(id); + } else if (initial) { + id = Object.keys(initial.fileManifest.entries).find( + (candidate) => + !next.local[candidate] && + arePathAliases( + initial.fileManifest.entries[candidate], + path + ) + ); + } + id ??= uuid(); + next.local[id] = path; + next.documents[id] ??= { + materialized: true, + bootstrap: initial?.fileManifest.entries[id] !== undefined + }; + } + for (const id of missing) delete next.local[id]; + for (const [id, path] of Object.entries(next.local)) { + if (unsyncablePaths.has(path) || ignored(path)) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = next.fileManifest.entries[id] ?? null; + } + const snapshot = snapshots.get(path); + if (snapshot) { + next.documents[id] ??= { materialized: true }; + if (!next.documents[id].materialized && !next.documents[id].base) + next.documents[id].bootstrap = true; + next.documents[id].materialized = true; + next.documents[id].observedHash = snapshot.hash; + } + } + next.protectedPaths = [...unsyncablePaths]; + guard(); + await commit(next); + queuedChanges.splice(0, changes.length); + guard(); + return unsyncablePaths; +} diff --git a/frontend/sync-client/src/sync-operations/syncer.test.ts b/frontend/sync-client/src/sync-operations/syncer.test.ts new file mode 100644 index 00000000..e3c11360 --- /dev/null +++ b/frontend/sync-client/src/sync-operations/syncer.test.ts @@ -0,0 +1,351 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import assert from "node:assert"; +import { afterEach, beforeEach, describe, it, mock } from "node:test"; +import type { Database } from "../persistence/database"; +import { + DEFAULT_SETTINGS, + type Settings, + type SyncSettings +} from "../persistence/settings"; +import type { FileOperations } from "../file-operations/file-operations"; +import type { SyncService } from "../services/sync-service"; +import type { ServerConfig } from "../services/server-config"; +import type { WebSocketManager } from "../services/websocket-manager"; +import type { Logger } from "../tracing/logger"; +import type { SyncHistory } from "../tracing/sync-history"; +import type { EventBatch } from "../services/types/EventBatch"; +import { EventListeners } from "../utils/data-structures/event-listeners"; +import { Syncer } from "./syncer"; +import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { toStoredSnapshot } from "./content"; +import { undiff } from "reconcile-text"; + +function createSyncer(overrides: Partial = {}): { + syncer: Syncer; + runCount: () => number; +} { + const settings = { + getSettings: () => ({ + ...DEFAULT_SETTINGS, + ...overrides + }) + } as unknown as Settings; + const database = { + state: {} + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners<() => unknown>() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + {} as Logger, + database, + settings, + {} as SyncService, + websocket, + {} as FileOperations, + {} as ServerConfig, + {} as SyncHistory, + new FixedSizeDocumentCache(1024) + ); + let runs = 0; + const internals = syncer as unknown as { + dirty: boolean; + run: () => Promise; + }; + internals.run = async (): Promise => { + runs++; + internals.dirty = false; + }; + + return { syncer, runCount: () => runs }; +} + +describe("Syncer interval", () => { + beforeEach(() => { + mock.timers.enable({ apis: ["setTimeout"] }); + }); + + afterEach(() => { + mock.timers.reset(); + }); + + it("does not schedule a timer when syncIntervalMs is unset", async () => { + const { syncer, runCount } = createSyncer({ + syncIntervalMs: undefined + }); + + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("keeps periodic reconciliation opt-in by default", async () => { + assert.strictEqual(DEFAULT_SETTINGS.syncIntervalMs, undefined); + const { syncer, runCount } = createSyncer(); + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("does not poll when explicitly disabled with zero", async () => { + const { syncer, runCount } = createSyncer({ syncIntervalMs: 0 }); + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(60_000); + assert.strictEqual(runCount(), 1); + await syncer.stop(); + }); + + it("wakes after the configured sync interval", async () => { + const { syncer, runCount } = createSyncer({ syncIntervalMs: 25 }); + + syncer.start(); + await syncer.waitUntilFinished(); + mock.timers.tick(24); + assert.strictEqual(runCount(), 1); + + mock.timers.tick(1); + assert.strictEqual(runCount(), 2); + await syncer.waitUntilFinished(); + await syncer.stop(); + }); +}); + +describe("Syncer per-file state and event delivery", () => { + const head = { + vaultUpdateId: 1, + documentId: "document", + updatedDate: "2026-01-01T00:00:00Z", + userId: "user", + deviceId: "device", + contentSize: 4 + }; + + function fixture() { + const database = { + state: { + initialized: true, + local: { document: "note.md" }, + documents: { + document: { + materialized: true, + observedHash: "hash", + base: { ...head, hash: "hash" } + } + }, + fileManifest: { + fileManifestId: 1, + entries: { document: "note.md" } + }, + remoteHeads: { document: head }, + lastSeenUpdateId: 0 + }, + getLatestDocumentByRelativePath: (path: string) => + path === "note.md" + ? { + documentId: "document", + relativePath: path, + isDeleted: false + } + : undefined + } as unknown as Database; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => Promise + >() + } as unknown as WebSocketManager; + let httpEventsCalls = 0; + const service = { + events: async () => { + httpEventsCalls++; + return { headEventId: 0, events: [] }; + } + } as unknown as SyncService; + const syncer = new Syncer( + "device", + {} as Logger, + database, + { + getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) + } as Settings, + service, + websocket, + {} as FileOperations, + {} as ServerConfig, + {} as SyncHistory, + new FixedSizeDocumentCache(1024) + ); + (syncer as unknown as { hasScanned: boolean }).hasScanned = true; + return { + database, + websocket, + syncer, + httpEventsCalls: () => httpEventsCalls + }; + } + + it("reports only a genuinely current path as up to date", () => { + const { syncer } = fixture(); + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true); + assert.strictEqual(syncer.isDocumentUpToDate("missing.md"), false); + + const internals = syncer as unknown as { + dirty: boolean; + unsyncablePaths: Set; + }; + internals.dirty = true; + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), true); + internals.unsyncablePaths.add("note.md"); + assert.strictEqual(syncer.isDocumentUpToDate("note.md"), false); + }); + + it("consumes a complete WebSocket event batch without fetching it again", async () => { + const { websocket, syncer, httpEventsCalls } = fixture(); + const batch: EventBatch = { + headEventId: 1, + events: [ + { + eventId: 1, + requestId: "00000000-0000-0000-0000-000000000001", + type: "fileManifest", + fileManifest: { fileManifestId: 1, entries: {} } + } + ] + }; + await websocket.onRemoteVaultUpdateReceived.triggerAsync(batch); + + const selected = await ( + syncer as unknown as { nextEventBatch: () => Promise } + ).nextEventBatch(); + assert.deepEqual(selected, batch); + assert.strictEqual(httpEventsCalls(), 0); + }); + + it("projects content responses to metadata before persistence", () => { + const { syncer } = fixture(); + const projected = ( + syncer as unknown as { + documentHead: (value: typeof head) => Record; + } + ).documentHead({ ...head, contentBase64: "secret" } as typeof head); + assert.deepEqual(projected, head); + assert.strictEqual("contentBase64" in projected, false); + }); +}); + +describe("Syncer diff uploads", () => { + const encoder = new TextEncoder(); + + function fixture(cacheSize = 1024): { + syncer: Syncer; + parent: Uint8Array; + } { + const cache = new FixedSizeDocumentCache(cacheSize); + const parent = encoder.encode("one two three\n"); + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners<() => unknown>() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + {} as Logger, + { state: {} } as Database, + { + getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) + } as Settings, + { + getDocumentVersionContent: async () => parent + } as unknown as SyncService, + websocket, + {} as FileOperations, + { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig, + {} as SyncHistory, + cache + ); + return { syncer, parent }; + } + + it("sends a diff when the fetched parent is still cached", async () => { + const { syncer, parent } = fixture(); + const internals = syncer as unknown as { + remoteSnapshot: ( + head: { + vaultUpdateId: number; + documentId: string; + }, + path: string + ) => Promise; + pushContent: ( + path: string, + parentVersionId: number, + snapshot: Awaited> + ) => Promise< + | { type: "Diff"; value: (number | string)[] } + | { type: "Snapshot"; value: string } + >; + }; + await internals.remoteSnapshot( + { vaultUpdateId: 7, documentId: "document" }, + "note.md" + ); + const changed = "one changed three\n"; + const payload = await internals.pushContent( + "note.md", + 7, + await toStoredSnapshot({ content: encoder.encode(changed) }) + ); + + assert.equal(payload.type, "Diff"); + if (payload.type === "Diff") { + assert.equal( + undiff(new TextDecoder().decode(parent), payload.value), + changed + ); + } + }); + + it("falls back to a snapshot when the parent cannot fit in the cache", async () => { + const { syncer } = fixture(0); + const internals = syncer as unknown as { + remoteSnapshot: ( + head: { + vaultUpdateId: number; + documentId: string; + }, + path: string + ) => Promise; + pushContent: ( + path: string, + parentVersionId: number, + snapshot: Awaited> + ) => Promise<{ type: string; value: unknown }>; + }; + await internals.remoteSnapshot( + { vaultUpdateId: 7, documentId: "document" }, + "note.md" + ); + const payload = await internals.pushContent( + "note.md", + 7, + await toStoredSnapshot({ content: encoder.encode("changed") }) + ); + + assert.equal(payload.type, "Snapshot"); + }); +}); diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 71dedd85..3a9bc83a 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -1,522 +1,1396 @@ +import type { SyncHistory } from "../tracing/sync-history"; +import { SyncStatus, SyncType } from "../tracing/sync-history"; +import { v4 as uuid } from "uuid"; +import { base64ToBytes } from "byte-base64"; +import { diff } from "reconcile-text"; import type { Database, - DocumentId, - DocumentRecord, - RelativePath + EngineState, + StoredSnapshot } from "../persistence/database"; -import type { SyncService } from "../services/sync-service"; -import type { Logger } from "../tracing/logger"; -import PQueue from "p-queue"; -import { hash } from "../utils/hash"; -import { v4 as uuidv4 } from "uuid"; +import { emptyState } from "../persistence/database"; +import type { + FileOperations, + FileWrite +} from "../file-operations/file-operations"; import type { Settings } from "../persistence/settings"; -import type { FileOperations } from "../file-operations/file-operations"; -import { findMatchingFile } from "../utils/find-matching-file"; -import type { UnrestrictedSyncer } from "./unrestricted-syncer"; -import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "../services/sync-reset-error"; -import { Locks } from "../utils/data-structures/locks"; -import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; -import type { WebSocketVaultUpdate } from "../services/types/WebSocketVaultUpdate"; +import type { SyncService } from "../services/sync-service"; +import type { ServerConfig } from "../services/server-config"; import type { WebSocketManager } from "../services/websocket-manager"; -import type { WebSocketClientMessage } from "../services/types/WebSocketClientMessage"; -import { awaitAll } from "../utils/await-all"; +import type { DocumentVersion } from "../services/types/DocumentVersion"; +import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; +import type { EventBatch } from "../services/types/EventBatch"; +import type { FileManifest } from "../services/types/FileManifest"; +import type { VaultSnapshot } from "../services/types/VaultSnapshot"; +import type { PushContent } from "../services/types/PushContent"; +import type { Logger } from "../tracing/logger"; import { EventListeners } from "../utils/data-structures/event-listeners"; +import { globsToRegexes } from "../utils/globs-to-regexes"; +import { mergeContent, toStoredSnapshot } from "./content"; +import { scanLocalFiles, type LocalChange } from "./scan"; +import { + mergeFileManifests, + resolvePaths, + sameFileManifest, + validateFileManifest +} from "./file-manifest"; +import { removeFromArray } from "../utils/remove-from-array"; +import { isInternalPath } from "../utils/portable-path"; +import { isBinary } from "../utils/is-binary"; +import { isFileTypeMergable } from "../utils/is-file-type-mergable"; +import type { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; +import { + AuthenticationError, + LocalChangesDuringReconciliation, + PermanentSyncError, + ServerVersionMismatchError, + SyncResetError, + ServerHistoryChangedError +} from "../errors/errors"; export class Syncer { + public readonly onReadyForEvents = new EventListeners<() => void>(); + public readonly onServerHistoryChanged = new EventListeners<() => void>(); public readonly onRemainingOperationsCountChanged = new EventListeners< - (remainingOperations: number) => unknown + (count: number) => unknown >(); - private readonly remoteDocumentsLock: Locks; - - // FIFO to limit the number of concurrent sync operations - private readonly syncQueue: PQueue; - - private _isFirstSyncComplete = false; - private runningScheduleSyncForOfflineChanges: Promise | undefined; - private previousRemainingOperationsCount = 0; + private running?: Promise; + private timer?: ReturnType; + private stopped = true; + private dirty = false; + private localGeneration = 0; + private failure?: unknown; + private readonly changes: LocalChange[]; + private notificationSaving: Promise = Promise.resolve(); + private notificationSaveFailed = false; + private readonly pendingPaths = new Set(); + private unsyncablePaths = new Set(); + private eventBatches: EventBatch[] = []; + private hasScanned = false; public constructor( - private readonly deviceId: string, + deviceId: string, private readonly logger: Logger, private readonly database: Database, private readonly settings: Settings, - private readonly syncService: SyncService, - private readonly webSocketManager: WebSocketManager, - private readonly operations: FileOperations, - private readonly internalSyncer: UnrestrictedSyncer + private readonly service: SyncService, + websocket: WebSocketManager, + private readonly files: FileOperations, + private readonly serverConfig: ServerConfig, + private readonly history: SyncHistory, + private readonly contentCache: FixedSizeDocumentCache, + private readonly localChangeStorage: { + entries: LocalChange[]; + save: (entries: LocalChange[]) => Promise; + } = { entries: [], save: async () => {} } ) { - this.syncQueue = new PQueue({ - concurrency: settings.getSettings().syncConcurrency - }); - - this.remoteDocumentsLock = new Locks(this.logger); - - settings.onSettingsChanged.add((newSettings, oldSettings) => { - if (newSettings.syncConcurrency !== oldSettings.syncConcurrency) { - this.syncQueue.concurrency = newSettings.syncConcurrency; + this.changes = localChangeStorage.entries; + websocket.onWebSocketStatusChanged.add((connected) => { + if (connected) { + websocket.sendHandshakeMessage({ + type: "handshake", + token: settings.getSettings().token, + deviceId, + lastSeenVaultUpdateId: database.state.lastSeenUpdateId + }); } + // Failed upgrades and lost transports still need HTTP catchup, + // including when the optional polling interval is disabled. + this.wake(); }); - - this.syncQueue.on("active", () => { - if (this.previousRemainingOperationsCount !== this.syncQueue.size) { - this.previousRemainingOperationsCount = this.syncQueue.size; - this.onRemainingOperationsCountChanged.trigger( - this.syncQueue.size - ); - } + websocket.onRemoteVaultUpdateReceived.add(async (batch) => { + this.eventBatches.push(batch); + // Notifications are only hints; HTTP replay recovers discarded pages. + if (this.eventBatches.length > 4) this.eventBatches.shift(); + this.wake(); }); - - this.webSocketManager.onWebSocketStatusChanged.add((isConnected) => { - if (isConnected) { - // The JS WebSocket API doesn't support setting headers, so we have to send the token as a message - this.sendHandshakeMessage(); - } - }); - this.webSocketManager.onRemoteVaultUpdateReceived.add( - this.syncRemotelyUpdatedFile.bind(this) - ); } public get isFirstSyncComplete(): boolean { - return this._isFirstSyncComplete; + return this.database.state.initialized; } - public async syncLocallyCreatedFile( - relativePath: RelativePath - ): Promise { + public get hasLocalChanges(): boolean { + return this.changes.length > 0; + } + + public get isBusy(): boolean { + return ( + !!this.running || + this.dirty || + !!this.database.state.pending || + !!this.database.state.application || + !!this.failure || + !!this.rejectionMessage + ); + } + + public isDocumentUpToDate(path: string): boolean { if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === false + !this.database.state.initialized || + !this.hasScanned || + this.pendingPaths.has(path) || + this.unsyncablePaths.has(path) || + this.ignored(path) ) { - this.logger.debug( - `Document ${relativePath} already exists in the database, skipping` - ); + return false; + } + + const record = this.database.getLatestDocumentByRelativePath(path); + if (!record) return false; + const document = this.database.state.documents[record.documentId]; + if ( + !document?.materialized || + !document.base || + document.bootstrap || + document.observedHash !== document.base.hash || + this.database.state.fileManifest.entries[record.documentId] !== path + ) { + return false; + } + + const remote = this.database.state.remoteHeads[record.documentId]; + if (remote && remote.vaultUpdateId > document.base.vaultUpdateId) + return false; + + const { pending } = this.database.state; + if ( + pending?.type === "content" && + pending.documentId === record.documentId + ) { + return false; + } + if ( + pending?.type === "fileManifest" && + pending.request.entries[record.documentId] !== + this.database.state.fileManifest.entries[record.documentId] + ) { + return false; + } + + return !this.database.state.application?.steps.some( + (step) => + step.documentId === record.documentId && + step.phase !== "installed" + ); + } + + public start(): void { + this.stopped = false; + this.failure = undefined; + this.hasScanned = false; + this.wake(); + } + + public async stop(): Promise { + this.stopped = true; + clearTimeout(this.timer); + await this.running; + } + + public wake(): void { + if (this.stopped) { return; } - const [promise, resolve, reject] = createPromise(); + this.dirty = true; - const id = uuidv4(); - const document = this.database.createNewPendingDocument( - id, - relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyCreatedFile(document) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - } - - public async syncLocallyDeletedFile( - relativePath: RelativePath - ): Promise { - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { - // This is must be a consequence of us deleting a file because of a remote update - // which triggered a local delete, so we don't need to do anything here. - this.logger.debug( - `Document ${relativePath} has already been markes as deleted, skipping` - ); + if (this.running) { return; } - // We have to have a record of the delete in case there's an in-flight update for the same - // document which finishes after the delete has succeeded and would introduce a phantom metadata record. - this.database.delete(relativePath); + clearTimeout(this.timer); - const [promise, resolve, reject] = createPromise(); - - const document = await this.database.getResolvedDocumentByRelativePath( - relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyDeletedFile(document) + this.running = this.run().finally(() => { + this.running = undefined; + this.onRemainingOperationsCountChanged.trigger( + this.failure || + this.database.state.pending || + this.rejectionMessage + ? 1 + : 0 ); - resolve(); + if (this.stopped) { + return; + } - this.database.removeDocument(document); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); + if (this.dirty && !this.failure) { + this.wake(); + return; + } + + if ( + this.failure instanceof PermanentSyncError || + this.failure instanceof AuthenticationError || + this.failure instanceof ServerVersionMismatchError + ) { + return; + } + + const intervalMs = + this.failure === undefined + ? this.settings.getSettings().syncIntervalMs + : this.settings.getSettings().networkRetryIntervalMs; + if (intervalMs === undefined || intervalMs === 0) { + return; + } + + this.timer = setTimeout(() => { + this.wake(); + }, intervalMs); + }); + } + + public async waitUntilFinished(): Promise { + while (this.running) { + await this.running; } + if (this.failure) { + throw this.failure; + } + if (this.rejectionMessage) + throw new PermanentSyncError(this.rejectionMessage); + } + + public async syncLocallyCreatedFile(_path: string): Promise { + this.localGeneration++; + this.pendingPaths.add(_path); + await this.recordLocalChange({ type: "create", path: _path }); + } + + public async syncLocallyDeletedFile(_path: string): Promise { + this.localGeneration++; + this.pendingPaths.add(_path); + await this.recordLocalChange({ type: "delete", path: _path }); } public async syncLocallyUpdatedFile({ oldPath, relativePath }: { - oldPath?: RelativePath; - relativePath: RelativePath; + oldPath?: string; + relativePath: string; }): Promise { - if (oldPath !== undefined) { - // We might have moved the document in the database before calling this method, - // in that case, we mustn't move it again. - if ( - this.database.getLatestDocumentByRelativePath(relativePath) === - undefined || - this.database.getLatestDocumentByRelativePath(relativePath) - ?.isDeleted === true - ) { - if (oldPath === relativePath) { - throw new Error( - `Old path and new path are the same: ${oldPath}` - ); - } - - this.database.move(oldPath, relativePath); - } - } - - let document = - this.database.getLatestDocumentByRelativePath(relativePath); - + // Content snapshots are re-read during journal staging. Only namespace + // notifications invalidate an identity plan; keystrokes must not starve + // a complete scan of unrelated documents. + if (oldPath !== undefined && oldPath !== "") this.localGeneration++; + this.pendingPaths.add(relativePath); + if (oldPath) this.pendingPaths.add(oldPath); if ( - oldPath !== undefined && - document?.metadata?.remoteRelativePath === relativePath + oldPath && + !isInternalPath(oldPath) && + !isInternalPath(relativePath) ) { - this.logger.debug( - `Document ${relativePath} has been moved as a result of a remote update, skipping sync` - ); - return; + await this.recordLocalChange({ + type: "move", + oldPath, + relativePath + }); + } else if (!isInternalPath(relativePath)) { + await this.recordLocalChange({ + type: "update", + path: relativePath + }); } + this.wake(); + } - if (document === undefined) { - this.logger.debug( - `Cannot find document ${relativePath} in the database, skipping` - ); - return; - } - - if (document.isDeleted) { - this.logger.debug( - `Document ${relativePath} has been deleted locally, skipping` - ); - return; - } - - const [promise, resolve, reject] = createPromise(); - - document = await this.database.getResolvedDocumentByRelativePath( - relativePath, - promise - ); + /** Local recovery must run even when network sync is disabled. Caller stops + * the engine first and holds the client's lifecycle lock. */ + public async recoverLocalState(): Promise { + await this.flushLocalChanges(); + await this.database.recoverPersistence(); + const { application } = this.database.state; + await this.files.recover(); + if (application) this.recordRecoveredApplication(application.steps); + } + private async recordLocalChange(change: LocalChange): Promise { + change.changeId = uuid(); + const previous = [...this.changes]; + this.changes.push(change); + const saving = this.notificationSaving.then(async () => { + await this.files.captureChange(change, previous); + await this.localChangeStorage.save(structuredClone(this.changes)); + this.notificationSaveFailed = false; + }); + this.notificationSaving = saving.catch(() => { + this.notificationSaveFailed = true; + }); try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncLocallyUpdatedFile({ - oldPath, - document - }) - ); - - resolve(); - } catch (e) { - reject(e); + await saving; } finally { - this.database.removeDocumentPromise(promise); + this.wake(); } } - public async scheduleSyncForOfflineChanges(): Promise { - if (this.runningScheduleSyncForOfflineChanges !== undefined) { - this.logger.debug("Uploading local changes is already in progress"); - return this.runningScheduleSyncForOfflineChanges; + public async flushLocalChanges(): Promise { + await this.notificationSaving; + if (this.notificationSaveFailed) { + await this.localChangeStorage.save(structuredClone(this.changes)); + this.notificationSaveFailed = false; } + } + private async run(): Promise { + this.onRemainingOperationsCountChanged.trigger(1); + this.failure = undefined; try { - this.runningScheduleSyncForOfflineChanges = - this.internalScheduleSyncForOfflineChanges(); - await this.runningScheduleSyncForOfflineChanges; - this.logger.info(`All local changes have been applied remotely`); - } catch (e) { - if (e instanceof SyncResetError) { - this.logger.info( - "Failed to apply local changes remotely due to a reset" - ); + await this.recoverLocalState(); + if ( + this.database.state.historyRecovery || + (this.database.state.initialized && + this.service.verifiesHistory && + !this.service.hasHistoryCheckpoint) + ) + await this.recoverServerHistory(); + await this.serverConfig.initialize(); + if (!this.database.state.initialized) { + await this.bootstrap(); + } + if (!this.stopped) this.onReadyForEvents.trigger(); + do { + this.dirty = false; + // An unknown accepted request must be resolved before observing a + // newer base, otherwise we can merge our own edit a second time. + if (this.database.state.pending) { + await this.finishPending(); + } + await this.scan(); + const batch = await this.nextEventBatch(); + await this.incorporateEventBatch(batch); + if (this.database.state.eventReplay) { + this.dirty = true; + continue; + } + await this.scan(); + // Previously excluded content becomes eligible after a setting + // change. Its durable obligation survives advancing the cursor. + if (Object.keys(this.database.state.excluded ?? {}).length) + await this.incorporateFileManifest( + this.database.state.fileManifest + ); + for (const head of Object.values( + this.database.state.remoteHeads + )) { + const doc = this.database.state.documents[head.documentId]; + if ( + this.database.state.local[head.documentId] !== + undefined && + (!doc?.materialized || + !doc.base || + doc.bootstrap || + doc.base.vaultUpdateId < head.vaultUpdateId) + ) { + await this.incorporateContent(head); + } + } + if (await this.preparePush()) { + await this.finishPending(); + this.dirty = true; + } + if ( + this.eventBatches.some( + (batch) => + batch.headEventId > + this.database.state.lastSeenUpdateId + ) + ) + this.dirty = true; + } while (this.dirty && !this.stopped); + if (!this.dirty && !this.database.state.pending) + this.pendingPaths.clear(); + } catch (error) { + if (error instanceof ServerHistoryChangedError && !this.stopped) { + try { + await this.recoverServerHistory(); + this.dirty = true; + return; + } catch (recoveryError) { + error = recoveryError; + } + } + if (error instanceof LocalChangesDuringReconciliation) { + this.dirty = true; return; } - this.logger.error( - `Not all local changes have been applied remotely: ${e}` - ); - throw e; - } finally { - this.runningScheduleSyncForOfflineChanges = undefined; - } - } - - public async waitUntilFinished(): Promise { - await this.runningScheduleSyncForOfflineChanges; - await this.syncQueue.onIdle(); // Wait for queue to be empty and running tasks to finish - } - - public async syncRemotelyUpdatedFile( - message: WebSocketVaultUpdate - ): Promise { - try { - const handlerPromise = awaitAll( - message.documents.map(async (document) => - this.internalSyncRemotelyUpdatedFile(document) - ) - ); - - await handlerPromise; - - if (message.isInitialSync && message.documents.length > 0) { - this.database.setLastSeenUpdateId( - message.documents - .map((document) => document.vaultUpdateId) - .reduce((a, b) => Math.max(a, b)) - ); + if (!(error instanceof SyncResetError && this.stopped)) { + this.failure = error; + this.serverConfig.reset(); + this.logger.error(`Sync paused for retry: ${String(error)}`); } - - this._isFirstSyncComplete = true; - } catch (e) { - this.logger.error(`Failed to sync remotely updated file: ${e}`); } } - public reset(): void { - this._isFirstSyncComplete = false; - this.syncQueue.clear(); - this.remoteDocumentsLock.reset(); - this.runningScheduleSyncForOfflineChanges = undefined; + private async recoverServerHistory(): Promise { + this.onServerHistoryChanged.trigger(); + this.contentCache.reset(); + this.eventBatches = []; + this.hasScanned = false; + if (!this.database.state.historyRecovery) { + await this.scan(); + const guard = this.localGuard(); + const previous = this.database.state; + for (const pending of [ + previous.pending, + ...(previous.unconfirmed ?? []) + ]) { + if (pending?.type === "content") + await this.files.retainSnapshot( + pending.request.requestId, + pending.documentId, + previous.local[pending.documentId], + pending.snapshot + ); + } + const next = emptyState(previous.vaultKey); + next.historyRecovery = true; + next.lastAppliedLocalChangeId = previous.lastAppliedLocalChangeId; + next.protectedPaths = previous.protectedPaths; + for (const [id, path] of Object.entries(previous.local)) { + const info = await this.files.fs.stat(path); + if (info?.kind !== "file") continue; + const doc = previous.documents[id]; + if (this.ignored(path) || this.oversized(info.size)) { + next.local[id] = path; + next.documents[id] = { + materialized: true, + bootstrap: true + }; + (next.excluded ??= {})[id] = + previous.fileManifest.entries[id] ?? null; + continue; + } + const snapshot = await this.files.snapshot(path); + if (!snapshot) throw new LocalChangesDuringReconciliation(); + // An unsent edit from the abandoned history is a separate local + // document. It must never be diffed against a reused version ID. + const clean = doc?.base?.hash === snapshot.hash; + const localId = doc?.base && !clean ? uuid() : id; + next.local[localId] = path; + next.documents[localId] = { + materialized: true, + observedHash: snapshot.hash, + recoveryBase: clean ? snapshot : doc?.recoveryBase + }; + } + guard(); + // Persist the reset BEFORE releasing the old checkpoint. After a + // crash no old request can be retried against the replacement log. + await this.database.commit(next); + } + await this.service.resetHistory(); + const next = this.next(); + delete next.historyRecovery; + await this.database.commit(next); } - private sendHandshakeMessage(): void { - const message: WebSocketClientMessage = { - type: "handshake", - deviceId: this.deviceId, - token: this.settings.getSettings().token, - lastSeenVaultUpdateId: this.database.getLastSeenUpdateId() + private localGuard(): () => void { + const generation = this.localGeneration; + return () => { + if (generation !== this.localGeneration) { + throw new LocalChangesDuringReconciliation(); + } }; - this.webSocketManager.sendHandshakeMessage(message); } - private async internalSyncRemotelyUpdatedFile( - remoteVersion: DocumentVersionWithoutContent + private ignored(path: string): boolean { + return ( + isInternalPath(path) || + globsToRegexes( + this.settings.getSettings().ignorePatterns, + this.logger + ).some((pattern) => pattern.test(path)) + ); + } + + private oversized(size: number): boolean { + return size > this.settings.getSettings().maxFileSizeMB * 1024 * 1024; + } + + private async cacheContent( + updateId: number, + path: string | undefined, + content: Uint8Array ): Promise { - let document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - if (document === undefined) { - // Let's avoid the same documents getting created in parallel multiple times. - // There might be multiple tasks waiting for the lock - return this.remoteDocumentsLock.withLock( - remoteVersion.documentId, - async () => { - document = this.database.getDocumentByDocumentId( - remoteVersion.documentId - ); - - // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile` - if (document === undefined) { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion - ) - ); - } else { - const [promise, resolve, reject] = createPromise(); - - document = - await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - } - - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); - } - ); + if ( + path !== undefined && + !isBinary(content) && + isFileTypeMergable( + path, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) + ) { + this.contentCache.put(updateId, content); } - - // We're either the first one to get the lock, so we have to create the document in `unrestrictedSyncRemotelyUpdatedFile` - const [promise, resolve, reject] = createPromise(); - - document = await this.database.getResolvedDocumentByRelativePath( - document.relativePath, - promise - ); - - try { - await this.syncQueue.add(async () => - this.internalSyncer.unrestrictedSyncRemotelyUpdatedFile( - remoteVersion, - document - ) - ); - - resolve(); - } catch (e) { - reject(e); - } finally { - this.database.removeDocumentPromise(promise); - } - - this.database.addSeenUpdateId(remoteVersion.vaultUpdateId); } - private async internalScheduleSyncForOfflineChanges(): Promise { - await this.createFakeDocumentsFromRemoteState(); + private async pushContent( + path: string, + parentVersionId: number | undefined, + snapshot: StoredSnapshot + ): Promise { + const content = base64ToBytes(snapshot.contentBase64); + const parent = + parentVersionId === undefined + ? undefined + : this.contentCache.get(parentVersionId); + if ( + parent !== undefined && + !isBinary(content) && + !isBinary(parent) && + isFileTypeMergable( + path, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ) + ) { + const decode = (bytes: Uint8Array): string => + new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true + }).decode(bytes); + const value = diff(decode(parent), decode(content)); + // Match the server's work bound; large edit sets remain syncable. + if (value.length <= 10_000) return { type: "Diff", value }; + } - const allLocalFiles = await this.operations.listFilesRecursively(); - this.logger.info( - `Scheduling sync for ${allLocalFiles.length} local files` + return { type: "Snapshot", value: snapshot.contentBase64 }; + } + + private next(): EngineState { + const { application: _, ...next } = structuredClone( + this.database.state ); + return next; + } - let locallyPossiblyDeletedFiles: DocumentRecord[] = []; + private async bootstrap(): Promise { + let vaultSnapshot = this.database.state.bootstrap; + if (!vaultSnapshot) { + vaultSnapshot = await this.service.vaultSnapshot(); + validateFileManifest(vaultSnapshot.fileManifest.entries); + await this.database.commit({ + ...this.next(), + bootstrap: vaultSnapshot + }); + } + await this.scan(vaultSnapshot); + await this.incorporateFileManifest( + vaultSnapshot.fileManifest, + vaultSnapshot.headEventId, + vaultSnapshot + ); + } - for (const document of this.database.resolvedDocuments) { + private async scan(initial?: VaultSnapshot): Promise { + await this.flushLocalChanges(); + this.unsyncablePaths = await scanLocalFiles( + { + next: this.next(), + guard: this.localGuard(), + changes: this.changes, + files: this.files, + ignored: (path) => this.ignored(path), + oversized: (size) => this.oversized(size), + commit: async (next) => this.database.commit(next) + }, + initial + ); + this.hasScanned = true; + await this.localChangeStorage.save(structuredClone(this.changes)); + } + + private async incorporateEventBatch(batch: EventBatch): Promise { + const replay = this.database.state.eventReplay; + const after = replay?.after ?? this.database.state.lastSeenUpdateId; + let cursor = after; + let manifest = replay?.manifest ?? this.database.state.fileManifest; + const heads = { ...(replay?.heads ?? this.database.state.remoteHeads) }; + const receipts = [...(replay?.receipts ?? [])]; + for (const event of batch.events) { + if (event.eventId <= cursor) continue; + if (event.eventId !== cursor + 1) + throw new PermanentSyncError( + "Non-contiguous event history; refusing to skip changes" + ); + cursor = event.eventId; if ( - !document.isDeleted && - !(await this.operations.exists(document.relativePath)) + this.database.state.unconfirmed?.some( + (pending) => pending.request.requestId === event.requestId + ) === true + ) + receipts.push(event); + if (event.type === "fileManifest") { + if (event.fileManifest.fileManifestId > manifest.fileManifestId) + manifest = event.fileManifest; + } else if ( + !heads[event.document.documentId] || + heads[event.document.documentId].vaultUpdateId < + event.document.vaultUpdateId ) { - locallyPossiblyDeletedFiles.push(document); + heads[event.document.documentId] = this.documentHead( + event.document + ); } } - - await awaitAll( - allLocalFiles.map(async (relativePath) => { - if ( - this.database.getLatestDocumentByRelativePath(relativePath) - ?.metadata !== undefined - ) { - this.logger.debug( - `Document ${relativePath} might have been updated locally, scheduling sync to validate and update it` + const end = batch.endEventId ?? batch.headEventId; + if ( + end !== cursor || + end > batch.headEventId || + (end === after && end < batch.headEventId) + ) + throw new PermanentSyncError("Incomplete event batch"); + if (end < batch.headEventId) { + await this.database.commit({ + ...this.next(), + eventReplay: { + after: end, + manifest, + heads, + receipts + } + }); + return; + } + if (cursor === this.database.state.lastSeenUpdateId) return; + // Recover our submitted bases before folding later remote edits. A + // permanent retry failure cannot cancel an earlier in-flight attempt. + const next = this.next(); + let acknowledged = false; + for (const event of receipts) { + const pending = next.unconfirmed?.find( + (candidate) => candidate.request.requestId === event.requestId + ); + if (!pending) continue; + if (event.type === "content" && pending.type === "content") { + const doc = next.documents[pending.documentId]; + if (!doc.base || doc.base.vaultUpdateId < event.eventId) { + doc.base = { + ...event.document, + hash: pending.snapshot.hash + }; + doc.bootstrap = false; + delete doc.rejected; + await this.cacheContent( + event.eventId, + next.local[pending.documentId], + base64ToBytes(pending.snapshot.contentBase64) ); - - return this.syncLocallyUpdatedFile({ - relativePath - }); } - - // Perhaps the file has been moved; let's check by looking at the deleted files - const contentHash = await this.syncQueue.add(async () => { - const contentBytes = - await this.operations.read(relativePath); // this can throw FileNotFoundError - return hash(contentBytes); - }); - - if (contentHash == undefined) { - // The file was deleted before we had a chance to read it, no need to sync it here - return; + } else if ( + event.type === "fileManifest" && + pending.type === "fileManifest" + ) { + if (next.fileManifest.fileManifestId < event.eventId) { + next.fileManifest = event.fileManifest; + delete next.rejectedManifest; } - - const originalFile = findMatchingFile( - contentHash, - locallyPossiblyDeletedFiles + } else + throw new PermanentSyncError( + "Request receipt type does not match submission" ); - if (originalFile !== undefined) { - // `originalFile` hasn't been deleted but it got moved instead - /* eslint-disable no-restricted-syntax -- Comparing by property, not direct equality */ - locallyPossiblyDeletedFiles = - locallyPossiblyDeletedFiles.filter( - (item) => - item.relativePath !== originalFile.relativePath - ); - /* eslint-enable no-restricted-syntax */ - - this.logger.debug( - `Document '${originalFile.relativePath}' was not found under its current path in the database but was found under a different path (${relativePath}), scheduling sync to move it` - ); - - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyUpdatedFile({ - oldPath: originalFile.relativePath, - relativePath - }); - } - - this.logger.debug( - `Document ${relativePath} not found in database, scheduling sync to create it` - ); - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyCreatedFile(relativePath); - }) - ); - - // this has to happen strictly after the previous awaitAll, as that one - // might have removed some of the documents from the list - await awaitAll( - locallyPossiblyDeletedFiles.map(async ({ relativePath }) => { - this.logger.debug( - `Document ${relativePath} has been deleted locally, scheduling sync to delete it` - ); - - // We're outside of the pqueue, so we need to call the public wrapper - return this.syncLocallyDeletedFile(relativePath); - }) + if (next.unconfirmed) removeFromArray(next.unconfirmed, pending); + acknowledged = true; + } + if (acknowledged) await this.database.commit(next); + // Three-way merging compares snapshots. Replaying intermediate remote + // edits can discard a local edit even when the remote ends at its base. + await this.incorporateFileManifest( + manifest, + cursor, + undefined, + false, + heads ); } - /** - * Create fake documents in the database for all files that are present locally - * and also exist remotely. This will stop the subequent syncs from duplicating - * the documents by creating the same documents from multiple clients. - */ - private async createFakeDocumentsFromRemoteState(): Promise { - if (this.database.getHasInitialSyncCompleted()) { + private async nextEventBatch(): Promise { + const after = + this.database.state.eventReplay?.after ?? + this.database.state.lastSeenUpdateId; + if (this.service.verifiesHistory) { + this.eventBatches = []; + return this.service.events(after); + } + this.eventBatches = this.eventBatches.filter( + (batch) => (batch.endEventId ?? batch.headEventId) > after + ); + const queued = this.eventBatches.findIndex((batch) => + batch.events.some((event) => event.eventId === after + 1) + ); + if (queued !== -1) return this.eventBatches.splice(queued, 1)[0]; + return this.service.events(after); + } + + private documentHead( + version: DocumentVersionWithoutContent + ): DocumentVersionWithoutContent { + return { + vaultUpdateId: version.vaultUpdateId, + documentId: version.documentId, + updatedDate: version.updatedDate, + userId: version.userId, + deviceId: version.deviceId, + contentSize: version.contentSize + }; + } + + private recordRecoveredApplication( + steps: readonly { + documentId: string; + from?: string; + to?: string; + replacement?: unknown; + }[] + ): void { + for (const step of steps) { + const to = this.database.state.local[step.documentId]; + if (!step.from && !to) continue; + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Filesystem application recovered", + details: + step.from && to && step.from !== to + ? { + type: SyncType.MOVE, + relativePath: to, + movedFrom: step.from + } + : !step.from && to + ? { type: SyncType.CREATE, relativePath: to } + : step.from && !to + ? { + type: SyncType.DELETE, + relativePath: step.from + } + : { + type: SyncType.UPDATE, + relativePath: to ?? step.from! + } + }); + } + } + + private recordManifestApplication( + before: Readonly>, + writes: Readonly> + ): void { + const after = this.database.state.local; + for (const id of new Set([ + ...Object.keys(before), + ...Object.keys(after) + ])) { + const from = before[id]; + const to = after[id]; + if (from === to && !writes[id]) continue; + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Remote change applied", + details: + from && to && from !== to + ? { + type: SyncType.MOVE, + relativePath: to, + movedFrom: from + } + : !from && to + ? { type: SyncType.CREATE, relativePath: to } + : from && !to + ? { type: SyncType.DELETE, relativePath: from } + : { + type: SyncType.UPDATE, + relativePath: to ?? from + } + }); + } + } + + private async remoteSnapshot( + head: DocumentVersionWithoutContent, + path: string, + supplied?: DocumentVersion + ): Promise { + const content = + supplied !== undefined + ? base64ToBytes(supplied.contentBase64) + : await this.service.getDocumentVersionContent({ + documentId: head.documentId, + vaultUpdateId: head.vaultUpdateId + }); + await this.cacheContent(head.vaultUpdateId, path, content); + return toStoredSnapshot({ content }); + } + + private async contentWrite( + next: EngineState, + head: DocumentVersionWithoutContent, + path: string, + supplied?: DocumentVersion + ): Promise { + const document = (next.documents[head.documentId] ??= { + materialized: false + }); + if ( + next.excluded?.[head.documentId] !== undefined || + this.ignored(path) || + this.oversized(head.contentSize) + ) { + return undefined; + } + const currentPath = this.database.state.local[head.documentId]; + const local = + currentPath && + (await this.files.fs.stat(currentPath))?.kind === "file" + ? await this.files.snapshot(currentPath) + : undefined; + if ( + local !== undefined && + this.oversized(base64ToBytes(local.contentBase64).length) + ) { + return undefined; + } + const base = + document.recoveryBase ?? + (local !== undefined && + document.base !== undefined && + document.bootstrap !== true + ? await this.remoteSnapshot(document.base, path) + : undefined); + // Cache the new head after the older merge base so it remains the most + // recently used version and can back the follow-up local diff. + const remote = await this.remoteSnapshot(head, path, supplied); + let result = remote; + if (local !== undefined) { + result = await mergeContent( + path, + base, + local, + remote, + (await this.serverConfig.getConfig()).mergeableFileExtensions + ); + } + document.base = { ...this.documentHead(head), hash: remote.hash }; + delete document.recoveryBase; + document.bootstrap = false; + document.materialized = true; + document.observedHash = result.hash; + return local?.contentBase64 === result.contentBase64 + ? undefined + : { expected: local, replacement: result }; + } + + private async incorporateContent( + head: DocumentVersionWithoutContent, + cursor?: number, + supplied?: DocumentVersion, + clearPending = false + ): Promise { + await this.scan(); + const guard = this.localGuard(); + const next = this.next(); + if ( + !next.remoteHeads[head.documentId] || + next.remoteHeads[head.documentId].vaultUpdateId < head.vaultUpdateId + ) { + next.remoteHeads[head.documentId] = this.documentHead(head); + } + if (cursor !== undefined) { + next.lastSeenUpdateId = cursor; + } + if (clearPending) { + delete next.pending; + } + const path = next.local[head.documentId]; + const document = next.documents[head.documentId]; + const wasMaterialized = document?.materialized; + const writes: Record = {}; + if ( + path !== undefined && + (!document?.base || + !document.materialized || + document.bootstrap || + document.base.vaultUpdateId < head.vaultUpdateId) + ) { + const write = await this.contentWrite(next, head, path, supplied); + if (write) { + writes[head.documentId] = write; + } + } + await this.files.apply(next, writes, guard); + if (path !== undefined && writes[head.documentId]) { + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Remote content applied", + details: { + type: wasMaterialized ? SyncType.UPDATE : SyncType.CREATE, + relativePath: path + }, + author: head.userId, + timestamp: new Date(head.updatedDate) + }); + } + } + + private async incorporateFileManifest( + remote: FileManifest, + cursor?: number, + initial?: VaultSnapshot, + clearPending = false, + heads?: EngineState["remoteHeads"] + ): Promise { + validateFileManifest(remote.entries); + await this.scan(initial); + const before = { ...this.database.state.local }; + const guard = this.localGuard(); + const next = this.next(); + if (cursor !== undefined) { + next.lastSeenUpdateId = cursor; + delete next.eventReplay; + } + if (clearPending) { + delete next.pending; + } + if (remote.fileManifestId < next.fileManifest.fileManifestId) { + await this.database.commit(next); return; } - const [allLocalFiles, remote] = await awaitAll([ - this.operations.listFilesRecursively(), - this.syncQueue.add(async () => this.syncService.getAll()) - ]); - - if (remote !== undefined) { - remote.latestDocuments - .filter( - (remoteDocument) => - allLocalFiles.includes(remoteDocument.relativePath) && - !remoteDocument.isDeleted && - this.database.getDocumentByDocumentId( - remoteDocument.documentId - ) === undefined - ) - .forEach((remoteDocument) => { - this.database.createNewEmptyDocument( - remoteDocument.documentId, - remoteDocument.vaultUpdateId, - remoteDocument.relativePath + const base = { ...next.fileManifest.entries }; + for (const [id, path] of Object.entries(next.excluded ?? {})) { + if (path === null) delete base[id]; + else base[id] = path; + } + const previousManifest = next.fileManifest.entries; + next.local = mergeFileManifests(base, next.local, remote.entries); + next.fileManifest = remote; + if (heads) next.remoteHeads = heads; + for (const [id, oldPath] of Object.entries(before)) { + const path = next.local[id]; + if ( + this.ignored(oldPath) || + this.unsyncablePaths.has(oldPath) || + (path !== undefined && this.ignored(path)) + ) { + next.excluded ??= {}; + if (next.excluded[id] === undefined) + next.excluded[id] = previousManifest[id] ?? null; + next.local[id] = oldPath; + } else if (next.excluded) delete next.excluded[id]; + } + for (const [id, path] of Object.entries(next.excluded ?? {})) { + if ( + before[id] === undefined && + (path === null || !this.ignored(path)) + ) + delete next.excluded![id]; + } + next.local = resolvePaths( + next.local, + remote.entries, + Object.keys(next.excluded ?? {}), + next.protectedPaths + ); + if (initial) { + next.initialized = true; + delete next.bootstrap; + for (const head of initial.documents) { + next.remoteHeads[head.documentId] = this.documentHead(head); + } + } + const writes: Record = {}; + for (const [id, path] of Object.entries(next.local)) { + next.documents[id] ??= { materialized: false }; + if (before[id] === undefined) + next.documents[id].materialized = false; + if ( + next.excluded?.[id] === undefined && + !this.ignored(path) && + remote.entries[id] !== undefined && + (!this.database.state.local[id] || + !next.documents[id].base || + next.documents[id].bootstrap || + !next.documents[id].materialized || + (next.remoteHeads[id]?.vaultUpdateId ?? 0) > + (next.documents[id].base?.vaultUpdateId ?? 0)) + ) { + const knownHead = + next.remoteHeads[id] ?? + initial?.documents.find((doc) => doc.documentId === id); + const head = knownHead ?? (await this.service.metadata(id)); + if (!head) { + throw new PermanentSyncError( + `File manifest references missing content: ${id}` ); - }); + } + next.remoteHeads[id] = this.documentHead(head); + const write = await this.contentWrite(next, head, path); + if (write) { + writes[id] = write; + } + } + } + await this.files.apply(next, writes, guard); + this.recordManifestApplication(before, writes); + } + + private async preparePush(): Promise { + if (this.stopped || this.database.state.eventReplay) { + return false; } - this.database.setHasInitialSyncCompleted(true); + let guard = this.localGuard(); + const canonical = resolvePaths( + this.database.state.local, + this.database.state.fileManifest.entries, + Object.keys(this.database.state.excluded ?? {}), + this.database.state.protectedPaths + ); + if (!sameFileManifest(canonical, this.database.state.local)) { + await this.files.apply( + { ...this.next(), local: canonical }, + {}, + guard + ); + await this.scan(); + guard = this.localGuard(); + } + + // Resolve the map after restoring excluded server paths, then make + // those allocations visible locally before submitting a manifest. + const submitted = this.manifestToSubmit(); + const local = { ...this.database.state.local }; + for (const [id, path] of Object.entries(submitted)) + if (this.database.state.excluded?.[id] === undefined) + local[id] = path; + if (!sameFileManifest(local, this.database.state.local)) { + await this.files.apply({ ...this.next(), local }, {}, guard); + await this.scan(); + guard = this.localGuard(); + } + for (const [id, path] of Object.entries(this.database.state.local)) { + if ( + this.ignored(path) || + this.database.state.excluded?.[id] !== undefined || + !this.database.state.documents[id]?.materialized || + this.unsyncablePaths.has(path) || + this.oversized( + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime. + this.database.state.remoteHeads[id]?.contentSize ?? 0 + ) + ) { + continue; + } + const snapshot = await this.files.snapshot(path); + if ( + !snapshot || + this.oversized(base64ToBytes(snapshot.contentBase64).length) + ) { + continue; + } + const { base, rejected } = this.database.state.documents[id]; + if ( + base?.hash === snapshot.hash || + rejected?.hash === snapshot.hash + ) { + continue; + } + guard(); + await this.database.commit({ + ...this.next(), + pending: { + type: "content", + documentId: id, + snapshot, + request: { + requestId: uuid(), + parentVersionId: base?.vaultUpdateId ?? null, + content: await this.pushContent( + path, + base?.vaultUpdateId, + snapshot + ) + } + } + }); + return true; + } + + const entries = this.manifestToSubmit(); + validateFileManifest(entries); + if ( + !sameFileManifest( + entries, + this.database.state.fileManifest.entries + ) && + (!this.database.state.rejectedManifest || + !sameFileManifest( + entries, + this.database.state.rejectedManifest.entries + )) + ) { + guard(); + await this.database.commit({ + ...this.next(), + pending: { + type: "fileManifest", + request: { + requestId: uuid(), + parentFileManifestId: + this.database.state.fileManifest.fileManifestId, + entries + } + } + }); + return true; + } + return false; + } + + private manifestToSubmit(): Record { + // Do not publish a new document until its initial content is acknowledged. + const entries = { ...this.database.state.local }; + for (const id of Object.keys(this.database.state.excluded ?? {})) { + const remotePath = this.database.state.fileManifest.entries[id]; + if (remotePath === undefined) delete entries[id]; + else entries[id] = remotePath; + } + for (const id of Object.keys(entries)) { + if ( + !this.database.state.documents[id]?.base && + this.database.state.fileManifest.entries[id] === undefined + ) { + delete entries[id]; + } + } + return resolvePaths( + entries, + this.database.state.fileManifest.entries, + Object.keys(this.database.state.excluded ?? {}), + [ + ...(this.database.state.protectedPaths ?? []), + ...Object.keys(this.database.state.excluded ?? {}).flatMap( + (id) => + this.database.state.local[id] + ? [this.database.state.local[id]] + : [] + ) + ] + ); + } + + private get rejectionMessage(): string | undefined { + const { state } = this.database; + for (const [id, path] of Object.entries(state.local ?? {})) { + const doc = state.documents[id]; + if ( + doc?.rejected && + doc.observedHash === doc.rejected.hash && + doc.base?.hash !== doc.rejected.hash && + !this.ignored(path) && + state.excluded?.[id] === undefined + ) + return doc.rejected.message; + } + if ( + state.rejectedManifest && + sameFileManifest( + this.manifestToSubmit(), + state.rejectedManifest.entries + ) && + !sameFileManifest( + state.fileManifest.entries, + state.rejectedManifest.entries + ) + ) + return state.rejectedManifest.message; + return undefined; + } + + /** An explicit reset or settings change permits fresh attempts after rejection. */ + public async retryRejectedRequests(): Promise { + await this.database.recoverPersistence(); + const next = structuredClone(this.database.state); + let changed = false; + for (const state of [next, next.application?.next]) { + if (!state) continue; + if (state.rejectedManifest) { + delete state.rejectedManifest; + changed = true; + } + for (const doc of Object.values(state.documents)) { + if (doc.rejected) { + delete doc.rejected; + changed = true; + } + } + } + if (changed) await this.database.commit(next); + } + + private async finishPending(): Promise { + let { pending } = this.database.state; + if (!pending) { + return; + } + + if (!pending.response && !pending.rejection) { + const copy = structuredClone(pending); + try { + if (copy.type === "content") { + copy.response = await this.service.putFileContent( + copy.documentId, + copy.request + ); + } else { + copy.response = await this.service.pushFileManifest( + copy.request + ); + } + } catch (error) { + if (!(error instanceof PermanentSyncError)) throw error; + copy.rejection = error.message; + } + await this.database.commit({ ...this.next(), pending: copy }); + pending = copy; + } + + if (pending.rejection && !pending.response) { + const next = this.next(); + if (pending.type === "content") { + await this.files.retainSnapshot( + pending.request.requestId, + pending.documentId, + next.local[pending.documentId], + pending.snapshot + ); + next.documents[pending.documentId].rejected = { + hash: pending.snapshot.hash, + message: pending.rejection + }; + } else + next.rejectedManifest = { + entries: pending.request.entries, + message: pending.rejection + }; + (next.unconfirmed ??= []).push(pending); + delete next.pending; + await this.database.commit(next); + this.history.addHistoryEntry({ + status: SyncStatus.ERROR, + message: pending.rejection, + details: { + type: SyncType.SKIPPED, + relativePath: + pending.type === "content" + ? (next.local[pending.documentId] ?? + pending.documentId) + : "File manifest" + } + }); + return; + } + + if (pending.type === "fileManifest") { + const response = pending.response!; + if (response.type === "StaleBase") { + await this.incorporateFileManifest( + response, + undefined, + undefined, + true + ); + } else { + const next = this.next(); + delete next.rejectedManifest; + next.fileManifest = { + fileManifestId: response.fileManifestId, + entries: pending.request.entries + }; + delete next.pending; + const previous = this.database.state.fileManifest.entries; + await this.database.commit(next); + for (const id of new Set([ + ...Object.keys(previous), + ...Object.keys(next.fileManifest.entries) + ])) { + const before = previous[id], + after = next.fileManifest.entries[id]; + if (before === after) { + continue; + } + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "File manifest change committed", + details: + after === undefined + ? { + type: SyncType.DELETE, + relativePath: before + } + : before === undefined + ? { + type: SyncType.CREATE, + relativePath: after + } + : { + type: SyncType.MOVE, + relativePath: after, + movedFrom: before + } + }); + } + } + } else { + const response = pending.response!; + if (response.type === "StaleBase") { + await this.files.retainSnapshot( + pending.request.requestId, + pending.documentId, + this.database.state.local[pending.documentId], + pending.snapshot + ); + await this.incorporateContent( + response, + undefined, + undefined, + true + ); + } else { + const next = this.next(); + const doc = (next.documents[pending.documentId] ??= { + materialized: false + }); + doc.base = { + ...this.documentHead(response), + hash: pending.snapshot.hash + }; + doc.bootstrap = false; + delete doc.rejected; + next.remoteHeads[pending.documentId] = + this.documentHead(response); + delete next.pending; + await this.database.commit(next); + const path = next.local[pending.documentId]; + await this.cacheContent( + response.vaultUpdateId, + path, + base64ToBytes(pending.snapshot.contentBase64) + ); + this.history.addHistoryEntry({ + status: SyncStatus.SUCCESS, + message: "Content committed", + details: { + type: SyncType.UPDATE, + relativePath: + next.local[pending.documentId] ?? pending.documentId + }, + author: response.userId, + timestamp: new Date(response.updatedDate) + }); + } + } } } diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts index a5f0cc7c..47c850f9 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.test.ts @@ -3,6 +3,30 @@ import assert from "node:assert"; import { EventListeners } from "./event-listeners"; describe("EventListeners", () => { + it("isolates throwing and rejecting observers and still notifies later listeners", async () => { + const failures: unknown[] = []; + const listeners = new EventListeners<() => unknown>((error) => + failures.push(error) + ); + const syncError = new Error("sync observer"); + const asyncError = new Error("async observer"); + let called = 0; + listeners.add(() => { + throw syncError; + }); + listeners.add(async () => { + throw asyncError; + }); + listeners.add(() => { + called++; + }); + assert.doesNotThrow(() => { + listeners.trigger(); + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(called, 1); + assert.deepEqual(failures, [syncError, asyncError]); + }); it("should add & remove listeners", () => { const listeners = new EventListeners<() => void>(); // eslint-disable-next-line @typescript-eslint/no-empty-function diff --git a/frontend/sync-client/src/utils/data-structures/event-listeners.ts b/frontend/sync-client/src/utils/data-structures/event-listeners.ts index e08ca65e..0c7b0e0b 100644 --- a/frontend/sync-client/src/utils/data-structures/event-listeners.ts +++ b/frontend/sync-client/src/utils/data-structures/event-listeners.ts @@ -8,50 +8,65 @@ import { awaitAll } from "../await-all"; export class EventListeners any> { private readonly listeners: TListener[] = []; + public constructor( + private readonly onError: (error: unknown) => void = (error) => { + console.error("Event observer failed", error); + } + ) {} + public get count(): number { return this.listeners.length; } /** - * Adds a new listener to the collection. - * - * @param listener The listener callback to add - * @returns An unsubscribe function that removes this listener when called - */ + * Adds a new listener to the collection. + * + * @param listener The listener callback to add + * @returns An unsubscribe function that removes this listener when called + */ public add(listener: TListener): () => void { this.listeners.push(listener); return () => this.remove(listener); } /** - * Removes a listener from the collection. - * - * @param listener The listener callback to remove - * @returns true if the listener was found and removed, false otherwise - */ + * Removes a listener from the collection. + * + * @param listener The listener callback to remove + * @returns true if the listener was found and removed, false otherwise + */ public remove(listener: TListener): boolean { return removeFromArray(this.listeners, listener); } /** - * Triggers all listeners synchronously with the provided arguments. - * Any returned promises are ignored. Use triggerAsync() to await them. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners synchronously with the provided arguments. + * Observer failures are reported without interrupting delivery or background + * work. Returned promises are observed; use triggerAsync() to await them. + * + * @param args The arguments to pass to each listener + */ public trigger(...args: Parameters): void { - this.listeners.forEach((listener) => { - listener(...args); - }); + for (const listener of [...this.listeners]) { + try { + void Promise.resolve(listener(...args)).catch( + (error: unknown) => { + this.reportFailure(error); + } + ); + } catch (error) { + this.reportFailure(error); + } + } } /** - * Triggers all listeners and awaits any promises they return. - * Synchronous listeners are called immediately, and any async listeners - * are awaited in parallel. - * - * @param args The arguments to pass to each listener - */ + * Triggers all listeners and awaits any promises they return. + * Synchronous listeners are called immediately, and any async listeners + * are awaited in parallel. + * + * @param args The arguments to pass to each listener + */ public async triggerAsync(...args: Parameters): Promise { await awaitAll( this.listeners @@ -68,4 +83,11 @@ export class EventListeners any> { public clear(): void { this.listeners.length = 0; } + private reportFailure(error: unknown): void { + try { + this.onError(error); + } catch { + // A diagnostic callback must not break background work either. + } + } } diff --git a/frontend/sync-client/src/utils/portable-path.test.ts b/frontend/sync-client/src/utils/portable-path.test.ts new file mode 100644 index 00000000..1ada24d5 --- /dev/null +++ b/frontend/sync-client/src/utils/portable-path.test.ts @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { test } from "node:test"; +import { validatePortablePaths } from "./portable-path"; + +test("a small deeply nested manifest fits within a bounded validation heap", () => { + const source = pathToFileURL(resolve(__dirname, "portable-path.ts")).href; + const result = spawnSync( + process.execPath, + [ + "--max-old-space-size=64", + "--import", + require.resolve("tsx"), + "--input-type=module", + "--eval", + ` + import { validatePortablePaths } from ${JSON.stringify(source)}; + const directory = Array(20000).fill("a").join("/"); + validatePortablePaths([directory + "/one.md", directory + "/two.md"]); + ` + ], + { encoding: "utf8", timeout: 10000 } + ); + assert.equal(result.status, 0, result.stderr || String(result.error)); +}); + +test("shared directories remain distinct from aliases and file ancestors", () => { + validatePortablePaths(["Notes/one.md", "Notes/two.md", "Other/one.md"]); + for (const paths of [ + ["Notes/one.md", "notes/two.md"], + ["Straße/one.md", "STRASSE/two.md"], + ["Notes", "Notes/one.md"], + ["Notes/one.md", "Notes"], + ["Notes/one.md", "Notes/one.md"] + ]) + assert.throws(() => validatePortablePaths(paths), /Conflicting path/); +}); diff --git a/frontend/sync-client/src/utils/portable-path.ts b/frontend/sync-client/src/utils/portable-path.ts new file mode 100644 index 00000000..f794cae7 --- /dev/null +++ b/frontend/sync-client/src/utils/portable-path.ts @@ -0,0 +1,190 @@ +export const INTERNAL_DIRECTORY = ".vault-link-sync"; +const MAX_COMPONENT_BYTES = 255; +const byteLength = (value: string): number => + new TextEncoder().encode(value).length; +function truncate(value: string, limit: number): string { + let result = ""; + for (const character of value) { + if (byteLength(result + character) > limit) break; + result += character; + } + return result; +} + +const reservedDeviceName = + /^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$/u; + +// NFC -> uppercase -> NFC matches the server's locale-independent alias rules. +const pathKey = (path: string): string => + path.normalize("NFC").toUpperCase().normalize("NFC"); + +export const arePathAliases = (left: string, right: string): boolean => + pathKey(left) === pathKey(right); + +export function findPathWithSameSpelling( + paths: Iterable, + wanted: string +): string | undefined { + const normalized = wanted.normalize("NFC"); + for (const path of paths) { + if (path !== wanted && path.normalize("NFC") === normalized) + return path; + } + return undefined; +} + +export const isInternalPath = (path: string): boolean => + arePathAliases(path.split("/")[0], INTERNAL_DIRECTORY); + +function validatePath(path: string): void { + if (!path || path !== path.normalize("NFC") || isInternalPath(path)) + throw new Error(`Invalid portable path: ${path}`); + + for (const part of path.split("/")) { + if ( + !part || + part === "." || + part === ".." || + /[<>:"\\|?*\p{Cc}]|[. ]$/u.test(part) || + reservedDeviceName.test( + part.split(".")[0].replace(/ +$/u, "").toUpperCase() + ) + ) + throw new Error(`Invalid portable path: ${path}`); + } +} + +export function validatePortablePaths(paths: Iterable): void { + // Component keys keep storage proportional to the input, even when a + // manifest contains thousands of directory levels. + const nodes = new Map< + string, + { id: number; spelling: string; file: boolean } + >(); + for (const path of paths) { + validatePath(path); + const parts = path.split("/"); + let parent = 0; + parts.forEach((spelling, index) => { + const file = index === parts.length - 1; + const key = `${parent}/${pathKey(spelling)}`; + const previous = nodes.get(key); + if ( + previous && + (previous.spelling !== spelling || previous.file || file) + ) + throw new Error(`Conflicting path: ${path}`); + const node = previous ?? { id: nodes.size + 1, spelling, file }; + nodes.set(key, node); + parent = node.id; + }); + } +} + +function portableName(part: string): string { + let result = part + .normalize("NFC") + .replace(/[<>:"\\|?*\p{Cc}]/gu, "_") + .replace(/[. ]+$/u, "_"); + if (!result || result === "." || result === "..") result = "_"; + if ( + reservedDeviceName.test( + result.split(".")[0].replace(/ +$/u, "").toUpperCase() + ) + ) + result = "_" + result; + return result; +} + +function fitName(name: string, suffix: string, file: boolean): string { + const dot = file ? name.lastIndexOf(".") : -1; + const extension = dot > 0 ? truncate(name.slice(dot), 64) : ""; + const stem = dot > 0 ? name.slice(0, dot) : name; + return portableName( + truncate(stem, MAX_COMPONENT_BYTES - byteLength(suffix + extension)) + + suffix + + extension + ); +} + +function pathsConflict(left: string, right: string): boolean { + const a = left.split("/"), + b = right.split("/"); + for (let i = 0; i < Math.min(a.length, b.length); i++) { + if (!arePathAliases(a[i], b[i])) return false; + if (a[i] !== b[i]) return true; + } + return true; // Equal names, or a file occupies an ancestor directory. +} + +function isAvailablePath( + candidate: string, + id: string, + occupied: Readonly> +): boolean { + validatePath(candidate); + return Object.entries(occupied).every( + ([otherId, path]) => otherId === id || !pathsConflict(candidate, path) + ); +} + +/** Deterministic allocation also handles a file blocking an ancestor directory. */ +export function allocatePortablePath( + wanted: string, + id: string, + occupied: Readonly> +): string { + id = portableName(truncate(id, 64)); + const originals = wanted.split("/"); + const parts = originals.map((part, index) => { + const portable = portableName(part); + return byteLength(portable) > MAX_COMPONENT_BYTES + ? fitName( + portable, + ` (conflict ${id})`, + index === originals.length - 1 + ) + : portable; + }); + if (isInternalPath(parts.join("/"))) parts[0] = "_" + parts[0]; + let candidate = parts.join("/"); + const suffix = ` (conflict ${id})`; + const limit = (Object.keys(occupied).length + 1) * (parts.length + 1); + for (let attempt = 0; attempt <= limit; attempt++) { + try { + if (isAvailablePath(candidate, id, occupied)) return candidate; + } catch { + /* allocate below */ + } + const current = candidate.split("/"); + let conflictAt = current.length - 1; + for (let i = 0; i < current.length; i++) { + const prefix = current.slice(0, i + 1).join("/"); + if ( + Object.values(occupied).some((path) => { + const other = path + .split("/") + .slice(0, i + 1) + .join("/"); + return ( + arePathAliases(prefix, other) && + (prefix !== other || + path.split("/").length === i + 1 || + i === current.length - 1) + ); + }) + ) { + conflictAt = i; + break; + } + } + const original = parts[conflictAt] ?? current[conflictAt]; + current[conflictAt] = fitName( + original, + suffix + (attempt ? ` (${attempt})` : ""), + conflictAt === current.length - 1 + ); + candidate = current.join("/"); + } + throw new Error(`Unable to allocate a portable path: ${wanted}`); +} diff --git a/frontend/test-client/README.md b/frontend/test-client/README.md new file mode 100644 index 00000000..230f42d7 --- /dev/null +++ b/frontend/test-client/README.md @@ -0,0 +1,36 @@ +# Seeded sync workloads + +The old random mock filesystem/agent have been replaced by the same v4 adapter, +client wrapper, and canonical-state assertions as the scripted suite. +Periodic polling is explicitly disabled so it cannot repair missed notifications +and hide failures in these workloads. Polling has separate fake-clock tests. + +From `frontend`: + +```sh +npm run build --workspace sync-client --workspace test-client +node test-client/dist/cli.js --seed 42 --iterations 25 +node test-client/dist/cli.js --replay /path/to/trace-42.json +``` + +The server binary must already be built. Each invocation starts and cleans up its +own server/database, and prints the replay command before executing any actions. +`--artifacts` chooses the trace/result directory. Failures include database, +visible/durable disk images, and CAS request records. The default 25 iterations +are scenarios selected in shuffled cycles of nine operations, using two or three +clients. Existing documents are repeatedly edited, moved, swapped, deleted and +recreated. Concurrent edits share an established base; delayed editor events stay +queued through reconnection and a subsequent remote edit. Create, content-update +and manifest requests receive before/after network faults. + +Each checkpoint serializes a complete expected file set with exact bytes, paths +and persistent identity keys. Independent creates must remain distinct, and a +recreated file must obtain a new UUID. Later checkpoints reassert earlier edits +and moves, rather than retaining only their markers. Old traces without a client +count still replay with two clients. + +All preservation assertions run even with delayed editor events. Explicit deletes +and overwrites exempt only their specific markers; unrelated content still must +appear exactly once. UUID continuity and canonical server state are checked too. +Seeds reproduce choices, not OS scheduling. See the +[full harness documentation](../deterministic-tests/README.md). diff --git a/frontend/test-client/src/agent/mock-agent.ts b/frontend/test-client/src/agent/mock-agent.ts deleted file mode 100644 index 1640c2ec..00000000 --- a/frontend/test-client/src/agent/mock-agent.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { choose } from "../utils/choose"; -import { v4 as uuidv4 } from "uuid"; -import { assert } from "../utils/assert"; -import type { RelativePath, SyncSettings } from "sync-client"; -import { debugging, Logger, LogLevel, utils } from "sync-client"; -import { MockClient } from "./mock-client"; -import { sleep } from "../utils/sleep"; -import type { LogLine } from "sync-client"; -import { withTimeout } from "../utils/with-timeout"; - -const TIMEOUT_MS = 10 * 60 * 1000; - -export class MockAgent extends MockClient { - private readonly writtenContents: string[] = []; - private readonly pendingActions: Promise[] = []; - - // The renamed file finding algorithm isn't too smart so we can't both update and rename the same file - private readonly doNotTouchWhileOffline: string[] = []; - - public constructor( - initialSettings: Partial, - public readonly name: string, - private readonly doDeletes: boolean, - private readonly doResets: boolean, - useSlowFileEvents: boolean, - private readonly jitterScaleInSeconds: number - ) { - super(initialSettings, useSlowFileEvents); - } - - public async init(): Promise { - await super.init( - debugging.slowFetchFactory(this.jitterScaleInSeconds), - debugging.slowWebSocketFactory( - this.jitterScaleInSeconds, - new Logger() // this logger isn't wired anywhere, so messages to it will be ignored - ) - ); - - assert( - (await this.client.checkConnection()).isSuccessful, - "Connection check failed" - ); - - this.client.logger.onLogEmitted.add((logLine: LogLine) => { - const state = this.client.getSettings().isSyncEnabled - ? "(online) " - : "(offline)"; - const formatted = `[${this.name} ${state}] ${logLine.timestamp.toISOString()} ${logLine.level} ${logLine.message}`; - - // HACK: we have to ensure the file has been synced if we want to change it offline without data loss - const historyEntry = /.*History entry: (.*.md).*/.exec( - logLine.message - ); - - if (historyEntry) { - utils.removeFromArray( - this.doNotTouchWhileOffline, - historyEntry[1] - ); - } - switch (logLine.level) { - case LogLevel.ERROR: - console.error(formatted); - - if (!this.useSlowFileEvents) { - // Let's wait for the error to be caught if there was one - // eslint-disable-next-line @typescript-eslint/no-floating-promises - sleep(100).then(() => process.exit(1)); - } - - break; - case LogLevel.WARNING: - console.warn(formatted); - break; - case LogLevel.INFO: - console.info(formatted); - break; - case LogLevel.DEBUG: - console.debug(formatted); - break; - } - }); - - this.client.logger.info("Agent initialized"); - } - - public async act(): Promise { - const options: (() => Promise)[] = [ - this.createFileAction.bind(this) - ]; - - if ( - this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.length === 0 - ) { - options.push(this.disableSyncAction.bind(this)); - } else { - options.push(this.enableSyncAction.bind(this)); - } - - const files = await this.listFilesRecursively(); - - if (files.length > 0) { - options.push( - this.renameFileAction.bind(this, files), - this.updateFileAction.bind(this, files) - ); - - if (this.doDeletes) { - options.push(this.deleteFileAction.bind(this, files)); - } - } - - if (Math.random() < 0.015 && this.doResets) { - // we can't just queue this up as once it's destroyed, no more method calls can go to SyncClient - await this.resetClient(); - } else { - this.pendingActions.push( - (async (): Promise => { - try { - return await choose(options)(); - } catch (error) { - this.client.logger.error( - `Failed to perform an action: ${error}` - ); - this.client.logger.info( - JSON.stringify(this.data, null, 2) - ); - this.client.logger.info( - JSON.stringify(this.localFiles, null, 2) - ); - throw error; - } - })() - ); - } - } - - public async finish(): Promise { - await withTimeout( - (async (): Promise => { - await this.client.setSetting("isSyncEnabled", true); - await utils.awaitAll(this.pendingActions); - await this.client.waitUntilFinished(); - })(), - TIMEOUT_MS, - "finish()" - ); - } - - public async destroy(): Promise { - await withTimeout( - (async (): Promise => { - await this.client.waitUntilFinished(); - await this.client.destroy(); - })(), - TIMEOUT_MS, - "destroy()" - ); - } - - public assertFileSystemsAreConsistent(otherAgent: MockAgent): void { - const globalFiles = Array.from(otherAgent.localFiles.keys()); - const localFiles = Array.from(this.localFiles.keys()); - - const missingInOther = localFiles.filter( - (file) => !otherAgent.localFiles.has(file) - ); - const missingInLocal = globalFiles.filter( - (file) => !this.localFiles.has(file) - ); - - try { - assert( - missingInOther.length === 0, - `Files from ${this.name} missing in ${otherAgent.name}: ${missingInOther.join(", ")}` - ); - assert( - missingInLocal.length === 0, - `Files from ${otherAgent.name} missing in ${this.name}: ${missingInLocal.join(", ")}` - ); - - for (const file of globalFiles) { - const localContent = new TextDecoder().decode( - this.localFiles.get(file) - ); - const otherContent = new TextDecoder().decode( - otherAgent.localFiles.get(file) - ); - assert( - localContent === otherContent, - `Content mismatch for file ${file}:\n${localContent}\n${otherContent}` - ); - } - } catch (e) { - this.client.logger.info( - "Local data: " + JSON.stringify(this.data, null, 2) - ); - this.client.logger.info( - "Local files: " + - Array.from(otherAgent.localFiles.keys()).join(", ") - ); - otherAgent.client.logger.info( - "Local data: " + JSON.stringify(otherAgent.data, null, 2) - ); - otherAgent.client.logger.info( - "Local files: " + - Array.from(otherAgent.localFiles.keys()).join(", ") - ); - - throw e; - } - } - - public assertAllContentIsPresentOnce(): void { - if (this.useSlowFileEvents) { - this.client.logger.info( - // We can't ensure that we have seen every single update - `Skipping content check for ${this.name} because slow file events are enabled` - ); - return; - } - - for (const content of this.writtenContents) { - const found = Array.from(this.localFiles.keys()).filter((key) => { - return new TextDecoder() - .decode(this.localFiles.get(key)) - .includes(content); - }); - - if (this.doDeletes) { - assert( - found.length <= 1, - `[${this.name}] Content ${content} found in ${found.join(", ")}` - ); - } else { - assert( - found.length >= 1, - `[${this.name}] Content ${content} not found in any files` - ); - - assert( - found.length <= 1, - `[${this.name}] Content ${content} found in multiple files: ${found.join(", ")}` - ); - - const [file] = found; - const fileContent = new TextDecoder().decode( - this.localFiles.get(file) - ); - assert( - fileContent.split(content).length == 2, - `Content ${content} (of ${this.name}) found more than once in '${file}'. File content:\n${fileContent}` - ); - } - } - } - - private async resetClient(): Promise { - this.client.logger.info(`Resetting client ${this.name}`); - await this.client.destroy(); - await this.init(); - } - - private async createFileAction(): Promise { - const file = this.getFileName(); - - if ( - (!this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file)) || - (await this.exists(file)) - ) { - return; - } - - const content = this.getContent(); - this.client.logger.info( - `Decided to create file ${file} with content ${content}` - ); - - return this.create(file, new TextEncoder().encode(` ${content} `)); - } - - private async disableSyncAction(): Promise { - this.client.logger.info(`Decided to disable sync`); - await this.client.setSetting("isSyncEnabled", false); - } - - private async enableSyncAction(): Promise { - this.client.logger.info(`Decided to enable sync`); - await this.client.setSetting("isSyncEnabled", true); - } - - private async renameFileAction(files: RelativePath[]): Promise { - const file = choose(files); - - // We can't edit files offline that have been updated while offline. - // Otherwise, the resolution logic couldn't handle it. - if ( - !this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file) - ) { - this.client.logger.info( - `Skipping file ${file} because it has been updated while offline` - ); - return; - } - - const newName = this.getFileName(); - - if ( - (!this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(newName)) || - (await this.exists(newName)) - ) { - return; - } - - this.client.logger.info(`Decided to rename file ${file} to ${newName}`); - this.doNotTouchWhileOffline.push(file, newName); - - return this.rename(file, newName); - } - - private async updateFileAction(files: RelativePath[]): Promise { - const file = choose(files); - - // We can't edit files offline that have been updated while offline. - // Otherwise, the resolution logic couldn't handle it. - if ( - !this.client.getSettings().isSyncEnabled && - this.doNotTouchWhileOffline.includes(file) - ) { - this.client.logger.info( - `Skipping file ${file} because it has been updated while offline` - ); - return; - } - - const content = this.getContent(); - this.client.logger.info( - `Decided to update file ${file} with ${content}` - ); - this.doNotTouchWhileOffline.push(file); - await this.atomicUpdateText(file, (old) => ({ - text: old.text + ` ${content} `, - cursors: [] - })); - } - - private async deleteFileAction(files: RelativePath[]): Promise { - const file = choose(files); - this.client.logger.info(`Decided to delete file ${file}`); - return this.delete(file); - } - - private getContent(): string { - const uuid = uuidv4(); - this.writtenContents.push(uuid); - return uuid; - } - - private getFileName(): string { - // Simulate name collisions between the clients - return `file-${Math.floor(Math.random() * 64)}.md`; - } -} diff --git a/frontend/test-client/src/agent/mock-client.ts b/frontend/test-client/src/agent/mock-client.ts deleted file mode 100644 index c814879a..00000000 --- a/frontend/test-client/src/agent/mock-client.ts +++ /dev/null @@ -1,197 +0,0 @@ -import type { StoredDatabase, TextWithCursors } from "sync-client"; -import { assert } from "../utils/assert"; -import { - type RelativePath, - type FileSystemOperations, - type SyncSettings, - SyncClient -} from "sync-client"; - -export class MockClient implements FileSystemOperations { - protected readonly localFiles = new Map(); - protected client!: SyncClient; - - protected data: Partial<{ - settings: Partial; - database: Partial; - }> = { - database: { - // Assume all clients start at the same time so there's no need to fetch - // any shared state. - hasInitialSyncCompleted: true - } - }; - - public constructor( - initialSettings: Partial, - protected readonly useSlowFileEvents: boolean - ) { - this.data.settings = initialSettings; - } - - public async init( - fetchImplementation: typeof globalThis.fetch, - webSocketImplementation: typeof globalThis.WebSocket - ): Promise { - this.client = await SyncClient.create({ - fs: this, - persistence: { - load: async () => this.data, - save: async (data) => void (this.data = data) - }, - fetch: fetchImplementation, - webSocket: webSocketImplementation - }); - - await this.client.start(); - } - - public async listFilesRecursively( - _root: RelativePath | undefined = undefined // we don't use multi-level paths during tests - ): Promise { - return Array.from(this.localFiles.keys()); - } - - public async read(path: RelativePath): Promise { - const file = this.localFiles.get(path); - if (!file) { - throw new Error(`File ${path} does not exist`); - } - return file; - } - - public async getFileSize(path: RelativePath): Promise { - return (await this.read(path)).length; - } - - public async exists(path: RelativePath): Promise { - return this.localFiles.has(path); - } - - public async create( - path: RelativePath, - newContent: Uint8Array - ): Promise { - if (this.localFiles.has(path)) { - throw new Error(`File ${path} already exists`); - } - this.client.logger.info( - `Creating file ${path} with content ${new TextDecoder().decode(newContent)}` - ); - this.localFiles.set(path, newContent); - - this.executeFileOperation(async () => - this.client.syncLocallyCreatedFile(path) - ); - } - - public async createDirectory(_path: RelativePath): Promise { - // This doesn't mean anything in our virtual FS representation - } - - public async atomicUpdateText( - path: RelativePath, - updater: (currentContent: TextWithCursors) => TextWithCursors - ): Promise { - const file = this.localFiles.get(path); - if (!file) { - throw new Error(`File ${path} does not exist`); - } - const currentContent = new TextDecoder().decode(file); - const newContent = updater({ text: currentContent, cursors: [] }).text; - const newContentUint8Array = new TextEncoder().encode(newContent); - this.localFiles.set(path, newContentUint8Array); - - if (!this.useSlowFileEvents) { - const existingParts = currentContent - .split(" ") - .map((part) => part.trim()); - const newParts = newContent.split(" ").map((part) => part.trim()); - existingParts.forEach((part) => - // all changes should be additive - { - assert( - newParts.includes(part), - `Part ${part} not found in new content: ${newContent}` - ); - } - ); - } - - this.client.logger.info( - `Updated file ${path} with:\n current content: ${currentContent}\n new content: ${newContent}` - ); - - this.executeFileOperation(async () => - this.client.syncLocallyUpdatedFile({ - relativePath: path - }) - ); - - return newContent; - } - - public async write(path: RelativePath, content: Uint8Array): Promise { - const hasExisted = this.localFiles.has(path); - this.localFiles.set(path, content); - - this.client.logger.info( - `Updated file ${path} with:\n new content: ${new TextDecoder().decode(content)}` - ); - - this.executeFileOperation(async () => { - if (hasExisted) { - return this.client.syncLocallyUpdatedFile({ - relativePath: path - }); - } else { - return this.client.syncLocallyCreatedFile(path); - } - }); - } - - public async delete(path: RelativePath): Promise { - this.client.logger.info( - `Deleting file: ${path} with:\n content ${new TextDecoder().decode(this.localFiles.get(path))}` - ); - this.localFiles.delete(path); - - this.executeFileOperation(async () => - this.client.syncLocallyDeletedFile(path) - ); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - const file = this.localFiles.get(oldPath); - if (!file) { - throw new Error(`File ${oldPath} does not exist`); - } - this.localFiles.set(newPath, file); - if (oldPath !== newPath) { - this.localFiles.delete(oldPath); - } - - this.client.logger.info( - `Renamed file: ${oldPath} -> ${newPath} with:\n content ${new TextDecoder().decode(file)}` - ); - - this.executeFileOperation(async () => - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }) - ); - } - - private executeFileOperation(callback: () => unknown): void { - if (this.useSlowFileEvents) { - // we aren't the best client and it takes some time to notice changes - setTimeout(callback, Math.random() * 100); - } else { - callback(); - } - } -} diff --git a/frontend/test-client/src/cli.ts b/frontend/test-client/src/cli.ts index 3af547e7..eb407615 100644 --- a/frontend/test-client/src/cli.ts +++ b/frontend/test-client/src/cli.ts @@ -1,205 +1,90 @@ -import type { SyncSettings } from "sync-client"; -import { utils } from "sync-client"; -import { MockAgent } from "./agent/mock-agent"; -import { sleep } from "./utils/sleep"; -import { v4 as uuidv4 } from "uuid"; -import { randomCasing } from "./utils/random-casing"; +import { parseArgs } from "node:util"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as os from "node:os"; +import { Logger } from "sync-client"; +import { ServerControl } from "../../deterministic-tests/src/server-control"; +import { ServerManager } from "../../deterministic-tests/src/server-manager"; +import { TestRunner } from "../../deterministic-tests/src/test-runner"; +import { generateTrace, traceTest, type Trace } from "./workload"; -const TEST_ITERATIONS = 5; - -// Simulate async file access by injecting waiting time before returning from file operations. -let slowFileEvents = false; - -// Whether to do resets in the test runs -let doResets = false; - -async function runTest({ - agentCount, - concurrency, - iterations, - doDeletes, - useResets, - useSlowFileEvents, - jitterScaleInSeconds -}: { - agentCount: number; - concurrency: number; - iterations: number; - doDeletes: boolean; - useResets: boolean; - useSlowFileEvents: boolean; - jitterScaleInSeconds: number; -}): Promise { - slowFileEvents = useSlowFileEvents; - doResets = useResets; - - const settings = `with ${agentCount} agents, concurrency ${concurrency}, iterations ${iterations}, doDeletes ${doDeletes}, doResets ${useResets}, jitterScaleInSeconds ${jitterScaleInSeconds}, useSlowFileEvents ${useSlowFileEvents}`; - console.info(`Running test ${settings}`); - - const vaultName = uuidv4(); - console.info(`Using vault name: ${vaultName}`); - const initialSettings: Partial = { - isSyncEnabled: true, - token: " test-token-change-me ", // same as in sync-server/config-e2e.yml with spaces - vaultName: randomCasing(vaultName) + (Math.random() > 0.5 ? " " : ""), // extra spaces shouldn't matter - syncConcurrency: concurrency, - remoteUri: "http://localhost:3000" - }; - - const clients: MockAgent[] = []; - for (let i = 0; i < agentCount; i++) { - clients.push( - new MockAgent( - initialSettings, - `agent-${i}`, - doDeletes, - useResets, - useSlowFileEvents, - jitterScaleInSeconds - ) - ); - } - - try { - await utils.awaitAll(clients.map(async (client) => client.init())); - - for (let i = 0; i < iterations; i++) { - console.info(`Iteration ${i + 1}/${iterations}`); - await utils.awaitAll(clients.map(async (client) => client.act())); - await sleep(Math.random() * 200); - } - - console.info("Stopping agents"); - - // Each agent can have unpushed changes which might conflict with eachother so each has to resolve the conflicts & push, and - for (const client of clients) { - try { - console.info(`Finishing up ${client.name}`); - await client.finish(); - } catch (err) { - if (!slowFileEvents) { - throw err; - } - } - } - - // then we need a second pass to ensure that all agents pull the same state. - for (const client of clients) { - try { - console.info(`Destroying ${client.name}`); - await client.destroy(); - } catch (err) { - if (!slowFileEvents) { - throw err; - } - } - } - - console.info("Agents finished successfully"); - - clients.slice(0, -1).forEach((client, i) => { - console.info( - `Checking consistency between ${client.name} and ${clients[i + 1].name}` - ); - client.assertFileSystemsAreConsistent(clients[i]); - console.info(`Consistency check for ${client.name} passed`); - }); - - console.info("File systems found to be consistent"); - - clients.forEach((client) => { - console.info(`Checking content for ${client.name}`); - client.assertAllContentIsPresentOnce(); - console.info(`Content check for ${client.name} passed`); - }); - - console.info(`Test passed ${settings}`); - } catch (err) { - console.error(`Test failed ${settings}`); - throw err; - } +function integer(value: string, zero = false): number { + const number = Number(value); + if ( + !/^\d+$/.test(value) || + !Number.isSafeInteger(number) || + number < (zero ? 0 : 1) + ) + throw new Error(`Invalid integer: ${value}`); + return number; } -async function runTests(): Promise { - for (let i = 0; i < TEST_ITERATIONS; i++) { - await runTest({ - agentCount: 2, - concurrency: 16, - iterations: 100, - doDeletes: true, - useResets: true, - useSlowFileEvents: true, - jitterScaleInSeconds: 0.75 - }); - - for (const useSlowFileEvents of [true, false]) { - for (const concurrency of [ - 16, - 1 // test with concurrency 1 to check for deadlocks - ]) { - for (const doDeletes of [false, true]) { - await runTest({ - agentCount: 2, - concurrency, - iterations: 100, - doDeletes, - useResets: false, - useSlowFileEvents, - jitterScaleInSeconds: 0.75 - }); - } - } +async function main(): Promise { + const { values } = parseArgs({ + options: { + seed: { type: "string", default: "1" }, + iterations: { type: "string", default: "25" }, + replay: { type: "string" }, + artifacts: { type: "string" }, + "server-binary": { type: "string" }, + config: { type: "string" } } - } -} - -process.on("uncaughtException", (error) => { - if ( - error instanceof Error && - error.message.includes( - "WebSocket was closed before the connection was established" - ) - ) { - return; - } - - console.error("Uncaught exception:", error); - process.exit(1); -}); - -process.on("unhandledRejection", (error, _promise) => { - if (error instanceof Error && error.message === "Sync was reset") { - return; - } - - if ( - slowFileEvents && - error instanceof Error && - (error.message.includes("Document not found") || - error.message.includes("Document already exists at new location")) - ) { - return; - } - - if ( - doResets && - error instanceof Error && - error.message.includes( - "SyncClient has been destroyed and can no longer be used" - ) - ) { - return; - } - - console.error("Unhandled rejection:", error); - process.exit(1); -}); - -runTests() - .then(() => { - process.exit(0); - }) - .catch((err: unknown) => { - console.error(err); - process.exit(1); }); + const seed = integer(values.seed!, true), + iterations = integer(values.iterations!); + const trace: Trace = values.replay + ? (JSON.parse(fs.readFileSync(values.replay, "utf8")) as Trace) + : generateTrace(seed, iterations); + const artifacts = values.artifacts + ? path.resolve(values.artifacts) + : fs.mkdtempSync(path.join(os.tmpdir(), "vault-link-fuzz-")); + fs.mkdirSync(artifacts, { recursive: true }); + const tracePath = path.join(artifacts, `trace-${trace.seed}.json`); + fs.writeFileSync(tracePath, JSON.stringify(trace, null, 2)); + console.log(`Replay: node ${__filename} --replay ${tracePath}`); + const logger = new Logger(); + logger.onLogEmitted.add((line) => console.log(line.message)); + const root = path.resolve(__dirname, "../../.."); + const server = new ServerControl( + values["server-binary"] ?? + path.join(root, "sync-server/target/release/sync_server"), + values.config ?? path.join(root, "sync-server/config-e2e.yml"), + logger + ); + const manager = new ServerManager(logger); + manager.track(server); + manager.installSignalHandlers(); + try { + await server.start(); + const result = await new TestRunner( + server, + logger, + "test-token-change-me", + server.remoteUri + ).runTest(`seed-${trace.seed}`, traceTest(trace)); + fs.writeFileSync( + path.join(artifacts, `result-${trace.seed}.json`), + JSON.stringify(result, null, 2) + ); + if (!result.success) throw new Error(result.error); + } finally { + await server.stop(); + manager.untrack(server); + } +} + +process.on("unhandledRejection", (error) => { + console.error(error); + process.exit(1); +}); +process.on("uncaughtException", (error) => { + console.error(error); + process.exit(1); +}); +main().then( + () => process.exit(0), + (error: unknown) => { + console.error(error); + process.exit(1); + } +); diff --git a/frontend/test-client/src/utils/random-casing.test.ts b/frontend/test-client/src/utils/random-casing.test.ts index 33217525..34d36a37 100644 --- a/frontend/test-client/src/utils/random-casing.test.ts +++ b/frontend/test-client/src/utils/random-casing.test.ts @@ -6,7 +6,7 @@ describe("randomCasing", () => { it("simple test", () => { const input = "hello, this is a really long string with a lot of characters"; - const result = randomCasing(input); + const result = randomCasing(input, () => 0); assert.strictEqual(result.toLowerCase(), input.toLowerCase()); assert.notStrictEqual(result, input); }); diff --git a/frontend/test-client/src/utils/random-casing.ts b/frontend/test-client/src/utils/random-casing.ts index ba09dace..96d7d25a 100644 --- a/frontend/test-client/src/utils/random-casing.ts +++ b/frontend/test-client/src/utils/random-casing.ts @@ -1,7 +1,7 @@ -export function randomCasing(str: string): string { +export function randomCasing(str: string, random: () => number = Math.random): string { const chars = str.split(""); const randomCasedChars = chars.map((char) => { - if (Math.random() < 0.5) { + if (random() < 0.5) { return char.toUpperCase(); } return char.toLowerCase(); diff --git a/frontend/test-client/src/workload.test.ts b/frontend/test-client/src/workload.test.ts new file mode 100644 index 00000000..8d9814f7 --- /dev/null +++ b/frontend/test-client/src/workload.test.ts @@ -0,0 +1,71 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { generateTrace, traceTest } from "./workload"; + +test("concrete traces replay all choices and retain preservation assertions with delayed events", () => { + const trace = generateTrace(42, 12); + assert.deepEqual(trace, generateTrace(42, 12)); + assert.notDeepEqual(trace, generateTrace(43, 12)); + assert.deepEqual( + traceTest(trace), + traceTest(JSON.parse(JSON.stringify(trace))) + ); + assert.equal( + trace.steps.filter((step) => step.type === "assert-markers").length, + 12 + ); + assert(trace.steps.some((step) => step.type === "delay-notifications")); + assert(trace.steps.some((step) => step.type === "delete")); + assert(trace.steps.some((step) => step.type === "assert-documents")); + const assertions = trace.steps.filter( + (step) => step.type === "assert-documents" + ); + assert( + assertions.at(-1)!.expected.some((doc) => doc.conflict), + "Independent creates need two expected documents" + ); + assert( + assertions + .at(-1)! + .expected.some((doc) => + assertions[0].expected.some( + (initial) => initial.key === doc.key + ) + ), + "Later assertions must retain earlier identities" + ); + for (const kind of ["create", "manifest", "content"]) { + assert( + Array.from({ length: 10 }, (_, seed) => + generateTrace(seed, 9) + ).some((generated) => + generated.steps.some( + (step) => + step.type === "drop-response" && step.kind === kind + ) + ) + ); + } + const delayed = trace.steps.findIndex( + (step) => step.type === "delay-notifications" + ); + const flushed = trace.steps.findIndex( + (step, i) => i > delayed && step.type === "flush-notifications" + ); + assert( + trace.steps + .slice(delayed, flushed) + .some((step) => step.type === "enable-sync") + ); + assert( + trace.steps + .slice(delayed, flushed) + .some((step) => step.type === "assert-documents") + ); + assert.equal(traceTest({ ...trace, clients: undefined }).clients, 2); + assert( + Array.from({ length: 10 }, (_, seed) => generateTrace(seed, 1)).some( + (generated) => generated.clients === 3 + ) + ); +}); diff --git a/frontend/test-client/src/workload.ts b/frontend/test-client/src/workload.ts new file mode 100644 index 00000000..4f199f46 --- /dev/null +++ b/frontend/test-client/src/workload.ts @@ -0,0 +1,310 @@ +import type { + ExpectedDocument, + TestDefinition, + TestStep +} from "../../deterministic-tests/src/test-definition"; +import { Random } from "../../test-support/random"; + +export interface Trace { + version: 1; + seed: number; + iterations: number; + /** Old traces omit this and retain their original two-client behavior. */ + clients?: number; + steps: TestStep[]; +} + +type Document = ExpectedDocument & { content: string }; + +/** Concrete actions and independent expected states are serialized together. + * Every round checks the whole history, including earlier bytes and UUIDs. + * A shuffled cycle covers nine actions; most revisit existing documents. + */ +export function generateTrace(seed: number, iterations: number): Trace { + const random = new Random(seed); + const clients = 2 + random.int(2); + const steps: TestStep[] = []; + const documents: Document[] = []; + const markers: string[] = []; + const removed: string[] = []; + let serial = 0; + const newDocument = (path: string): Document => { + const key = `seed-${seed}-document-${serial++}`; + const marker = `[${key}]`; + markers.push(marker); + return { + key, + path, + content: `${marker}\n\nfirst paragraph\n\nlast paragraph\n` + }; + }; + const all = (type: "enable-sync" | "disable-sync") => { + for (let client = 0; client < clients; client++) + steps.push({ type, client }); + }; + const expected = () => + steps.push({ + type: "assert-documents", + expected: structuredClone(documents) + }); + const barrier = () => { + steps.push({ type: "barrier" }); + expected(); + }; + const forget = (doc: Document) => { + for (const marker of markers) + if (doc.content.includes(marker) && !removed.includes(marker)) + removed.push(marker); + documents.splice(documents.indexOf(doc), 1); + }; + all("enable-sync"); + steps.push({ type: "barrier" }); + // A lasting pool makes later actions interact with earlier versions/moves. + for (const path of ["shared/a.md", "日本語/b.md", "café/c.md"]) { + const doc = newDocument(path); + documents.push(doc); + steps.push({ type: "create", client: 0, path, content: doc.content }); + } + barrier(); + let choices: number[] = []; + for (let round = 0; round < iterations; round++) { + if (!choices.length) { + choices = Array.from({ length: 9 }, (_, i) => i); + for (let i = choices.length - 1; i > 0; i--) { + const j = random.int(i + 1); + [choices[i], choices[j]] = [choices[j], choices[i]]; + } + } + const kind = choices.pop()!; + const writer = random.int(clients); + const other = (writer + 1 + random.int(clients - 1)) % clients; + const doc = random.pick(documents.filter((entry) => !entry.conflict)); + const directory = `round-${round}/${random.pick(["nested", "日本語", "café", "space folder"])}`; + if (kind === 0) { + all("disable-sync"); + const first = newDocument(`${directory}/same.md`); + const second = { ...newDocument(first.path), conflict: true }; + steps.push( + { + type: "create", + client: writer, + path: first.path, + content: first.content + }, + { + type: "create", + client: other, + path: second.path, + content: second.content + }, + { + type: "drop-response", + client: writer, + kind: random.pick(["create", "manifest"] as const), + point: random.pick(["before", "after"] as const) + }, + { type: "enable-sync", client: writer }, + { type: "sync", client: writer }, + { type: "wait-for-response-drop", client: writer } + ); + all("enable-sync"); + documents.push(first, second); + } else if (kind === 1) { + // Both clients edit the same established version, at opposite ends. + all("disable-sync"); + const prefix = `remote prefix ${round}\n\n`; + const suffix = `\n\nlocal suffix ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + suffix + }, + { + type: "update", + client: other, + path: doc.path, + content: prefix + doc.content + }, + { type: "enable-sync", client: writer } + ); + all("enable-sync"); + doc.content = prefix + doc.content + suffix; + } else if (kind === 2) { + all("disable-sync"); + const target = `${directory}/moved.md`; + doc.content += `\nremote addition ${round}\n`; + steps.push( + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + }, + { + type: "update", + client: other, + path: doc.path, + content: doc.content + }, + { type: "enable-sync", client: writer } + ); + all("enable-sync"); + doc.path = target; + } else if (kind === 3) { + const second = random.pick( + documents.filter((entry) => !entry.conflict && entry !== doc) + ); + steps.push( + { type: "disable-sync", client: writer }, + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: `${directory}/temporary.md` + }, + { + type: "rename", + client: writer, + oldPath: second.path, + newPath: doc.path + }, + { + type: "rename", + client: writer, + oldPath: `${directory}/temporary.md`, + newPath: second.path + }, + { type: "enable-sync", client: writer } + ); + [doc.path, second.path] = [second.path, doc.path]; + } else if (kind === 4) { + // Reuse the same path but require a new UUID, even in later rounds. + steps.push({ type: "delete", client: writer, path: doc.path }); + forget(doc); + barrier(); + const replacement = newDocument(doc.path); + steps.push({ + type: "create", + client: other, + path: replacement.path, + content: replacement.content + }); + documents.push(replacement); + } else if (kind === 5) { + const target = `${directory}/rapid.md`; + steps.push( + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + }, + { + type: "rename", + client: writer, + oldPath: target, + newPath: doc.path + }, + { + type: "rename", + client: writer, + oldPath: doc.path, + newPath: target + } + ); + doc.path = target; + doc.content += `\nrapid edit ${round}\n`; + steps.push({ + type: "update", + client: writer, + path: doc.path, + content: doc.content + }); + } else if (kind === 6) { + steps.push({ + type: "drop-response", + client: writer, + kind: "content", + point: random.pick(["before", "after"] as const) + }); + doc.content += `\nretry edit ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + }, + { type: "wait-for-response-drop", client: writer } + ); + } else if (kind === 7) { + // Keep an editor event delayed across reconnection and another edit. + steps.push( + { type: "disable-sync", client: writer }, + { type: "delay-notifications", client: writer } + ); + doc.content += `\ndelayed local edit ${round}\n`; + steps.push( + { + type: "update", + client: writer, + path: doc.path, + content: doc.content + }, + { type: "enable-sync", client: writer } + ); + barrier(); + doc.content += `\nlater remote edit ${round}\n`; + steps.push({ + type: "update", + client: other, + path: doc.path, + content: doc.content + }); + barrier(); + steps.push({ type: "flush-notifications", client: writer }); + } else { + // Explicit replacement retains the UUID. Only markers in the + // overwritten bytes are exempted from preservation. + for (const marker of markers) + if (doc.content.includes(marker) && !removed.includes(marker)) + removed.push(marker); + doc.content = newDocument(doc.path).content; + steps.push({ + type: "update", + client: writer, + path: doc.path, + content: doc.content + }); + } + barrier(); + steps.push({ + type: "assert-markers", + markers: [...markers], + removed: [...removed] + }); + if (random.int(4) === 0) { + steps.push({ type: "reset", client: random.int(clients) }); + barrier(); + } + } + return { version: 1, seed, iterations, clients, steps }; +} + +export function traceTest(trace: Trace): TestDefinition { + if ( + trace.version !== 1 || + !Array.isArray(trace.steps) || + !trace.steps.length || + (trace.clients !== undefined && + (!Number.isInteger(trace.clients) || trace.clients < 2)) + ) + throw new Error("Invalid/empty replay trace"); + return { + clients: trace.clients ?? 2, + description: `Seed ${trace.seed}; ${trace.iterations} generated conflict scenarios`, + steps: trace.steps + }; +} diff --git a/frontend/test-client/tsconfig.json b/frontend/test-client/tsconfig.json index e86df89d..7c59fb92 100644 --- a/frontend/test-client/tsconfig.json +++ b/frontend/test-client/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "baseUrl": ".", "strict": true, + "skipLibCheck": true, "target": "ES2022", "module": "CommonJS", "esModuleInterop": true, diff --git a/frontend/test-client/webpack.config.js b/frontend/test-client/webpack.config.js index 6aee1547..11c3e6dd 100644 --- a/frontend/test-client/webpack.config.js +++ b/frontend/test-client/webpack.config.js @@ -4,6 +4,7 @@ const webpack = require("webpack"); module.exports = { entry: "./src/cli.ts", target: "node", + externals: { "sync-client": "commonjs sync-client" }, mode: "production", optimization: { minimize: false diff --git a/frontend/test-support/README.md b/frontend/test-support/README.md new file mode 100644 index 00000000..e3601ba5 --- /dev/null +++ b/frontend/test-support/README.md @@ -0,0 +1,163 @@ +# Shared sync harness support + +This is test-only code, shared by `deterministic-tests` and `test-client` rather +than exported as debugging machinery by the production sync client. + +The journal suites also cover repeated saves replacing earlier unsynced text, +tracked file/directory moves into ignored paths, and saves crossing the size +limit. They check exact bytes and identities through recovery, including real +server reads. Client and Rust unit suites bound deep-path validation resources. + +| Layer | What it establishes | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `harness.test.ts` | Divergence/self-comparison detection, binary-safe comparisons, marker loss/duplication, cleanup failure propagation, deep persistence ownership, exclusive operations, exercised faults, seed reproducibility | +| `oracle-regressions.test.ts` | Negative checks for regenerated retries, wrong fault kinds, duplicate clients, corrupt hashes, BOMs, path byte limits, idempotent unlink, directory durability, and complete identity/content expectations | +| `recovery.test.ts` | Real engine journal recovery at every enumerated apply and replay mutation/save boundary; two interruptions followed by another power loss; identity, retained bytes and merge-policy regressions | +| `engine-regressions.test.ts` | Recovery archives, offline notifications, rejection/retry, exclusions, reverted remote updates, lifecycle races and editor changes during staging/install | +| `event-delivery.test.ts` | Real sync loop with checkpoint-controlled WS bursts before an iteration and during event application; no polling or extra wakeups to hide queued events | +| `polling.test.ts` | Real sync loop with fake clocks: missed local/remote notifications, disabled controls, repeated polls, shutdown and network retries independent of polling | +| `protocol.test.ts` | Terminated worker runtimes after accepted creates, existing-document Snapshot/Diff updates, and manifest requests, exact durable retries after newer remote heads and real server SIGKILL; concurrent CAS and ordered WS catchup | +| `real-disk.test.ts` | Host filesystem exclusive create/rename, symlink/hardlink rejection and a real engine rename-cycle application | +| Scripted suite | Existing multi-client regression scenarios plus v4 binary, identity, path-conflict, retry, and server-restart cases | +| Seeded workloads | Concrete replay traces, offline and rapid online moves/edits, concurrent creates, exact deletion/overwrite exemptions, nested Unicode paths, delayed notifications and before/after network failures | + +`scripts/e2e-runner.test.mjs` also guards clean-install workspace/lockfile +consistency, CI dependency installation and artifact upload wiring, subprocess +exit codes, log failures, spawn failures, and timeouts. + +`MemoryDisk` distinguishes visible namespace/data from durable entries. Each +mutation has `before:`, `visible:`, and `durable:` boundaries. `crash(false)` +models process death with dirty kernel state retained; `crash(true)` discards +unflushed state. Operations that return successfully have flushed their affected +entries. Idempotent unlink flushes absent entries too, and durable directory +removal makes all former descendants unreachable. This is a deliberately small atomic-operation model, not an emulation +of APFS/ext4 writeback. A crashed session is fenced against subsequent writes. + +`MemoryPersistence` atomically saves complete, cloned objects. Failure before a +save leaves the old object; failure after durable replacement leaves the new one. +The journal sweep recreates the production database/file-operations objects after +each fault. From each interrupted state it traces recovery and injects a second +process death or power loss at every replay mutation/save boundary. Final recovery +is replayed again and followed by another power loss. Cases cover swaps with new +output, cross-directory moves with a surviving sibling, concurrent text merges, +binary replacement, deletion and recovery archive cleanup. Exact visible bytes, +UUID/path mappings, retained bytes and absence of transaction files are checked. An intent is resubmitted only if nothing was durably saved; +resubmitting a partially recovered operation would hide journal defects. + +`crash-worker.ts` runs a complete SyncClient in a separate worker. The parent +terminates it at an explicit accepted-response checkpoint without `destroy()`; +no old timers, websocket handlers, or promises survive. Update cases bootstrap an existing UUID and assert the non-null parent and exact +Snapshot/Diff request type before termination. Its saved memory-disk image and persisted database seed a fresh runtime. The separate real-disk fixture +uses a test-only native exclusive-rename primitive. It is **not** a +production adapter or an adversarial `openat` sandbox: concurrent hostile changes +to ancestor directories and actual machine power cuts are outside its coverage. + +## Regression coverage + +The previously exposed WS queue, notified-rename UUID, uncertain-save replay, +and cross-directory durability failures are fixed and remain asserted by the +default harness. Scripted, seeded, event-delivery and engine regression clients +explicitly disable polling so it cannot hide a missing wakeup. Crash workers also +assert that this setting survives restart. Polling is enabled only in separate +tests that intentionally omit notifications and advance a fake clock. Identity +assertions distinguish documents even when bytes +are equal. The engine regressions additionally exercise retained recovery bytes, +durable offline notifications, permanently rejected requests, ignored namespace +conflicts, reverted remote updates, and concurrent lifecycle changes. + +Tests assert intended invariants, including bytes and UUID mappings, rather than +accepting the observed output as an oracle. The memory fault model and real-disk +checks complement one another; neither proves behavior under every real power cut. + +## September 2026 audit regressions + +`scripts/e2e.sh` now runs the Rust and client unit suites as well as the harness, +real-server protocol checks, scripted scenarios, and seeded workloads. A failing +unit suite makes the complete run fail. The audit tests are separate, individually +quoted arguments in the default scripts, with a runner regression guarding that wiring. + +The audit cases were first run with assertions for the required behavior and +observed failing before the corresponding fixes. Coverage is organized by the +original finding numbers: + +| Findings | Permanent coverage and resulting guarantee | +| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Engine and real-server lost-reply/rejected-retry tests preserve later content and rename undos; interrupted receipt saves and paged replay are tested after process replacement. | +| 2, 6 | Ignored/oversized unmanaged occupants keep their paths, including arrivals after scanning; submitted manifests reserve excluded server paths and remain valid. | +| 3, 12 | Oversized remote heads defer uploads; oversized local files are rejected before snapshot reads; scans retain hashes rather than encoded vault contents; content notifications cannot starve namespace scans. | +| 4 | A subprocess deadline detects synchronous conflict-allocation hangs caused by long extensions. | +| 5 | Persistent destination errors select a durable root fallback. The recovery sweep keeps that error active through every first and second crash boundary, checking exact bytes, identities and retained originals. | +| 7–9 | Bootstrap notifications adopt remote identities; settings updates resume enabled engines, disabling sync persists across reset, and initial vault bindings survive restart; disabled startup recovers local journals. | +| 10–11 | Fake-clock silent-socket recovery and real-server connection admission, slot release and handshake expiry. | +| 13 | Rust tests bound event count and aggregate bytes per page; client tests fold remote reversions before application and preserve the replay position across restart. | +| 14–15 | Metadata-only manifest reference validation and active-pool cleanup tests ensure unrelated payload decoding and checked-out transactions cannot block these operations. | +| 16 | Dirty cursor updates invalidate precise positions; disconnect/reset clear presented cursors. | +| 17 | Read-only configuration preservation and concurrent initialization verify that existing credentials are never rewritten or replaced. | + +The follow-up audit added these independently reproduced failures. Each behavior +was asserted before its fix; the default test commands include every regression. + +| Additional failure | Regression coverage | +| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Delayed content receipts crash after local deletion | `sync-audit.test.ts` verifies receipt consumption, preserved deletion, and cursor advancement without a local path. | +| Cursor reads restore presence after disconnect/reset | Cursor unit tests hold an active read and queue another update, retire the connection, then release both. | +| Failed WebSocket upgrades strand clients with polling disabled | `polling.test.ts` requires exact remote content and rename catchup over HTTP after a disconnect signal. | +| WebSocket construction errors escape startup/retry callbacks | Transport unit tests fail the first or second construction and require a successful subsequent connection. | +| Opening a locked vault blocks unrelated vaults | Rust tests hold a real SQLite writer lock during lazy opening; another vault must remain readable. Concurrent openings must share the configured pool, including after cancellation. | +| Failure clearing rejected requests leaves an enabled client stopped | Settings tests fail the second save, before and after commit, and require later notifications to wake synchronization. | +| Caller mutations bypass the checked settings snapshot | A settings test changes the caller's vault name and exclusion array while transports pause; persisted settings must retain the original input. | +| Cursor freshness uses metadata captured before an asynchronous read | A cursor test advances the document while the read is suspended and requires the old cursor to be marked outdated. | +| Throwing/silent socket closes leave live handlers after shutdown | Transport tests require disconnected status and no message delivery after a failed or timed-out close. | +| Handshake send errors escape the socket-open callback | A transport test drops the send during opening and requires safe retirement and retry. | +| Old connection cleanup deletes its replacement's cursors | A real-server overlapping reconnect test preserves the replacement; Rust tests also reject delayed updates from the old session. | + +These cases exercise transitions the earlier tests did not combine: a deletion +between submission and acknowledgement, disconnects during reads, failures after +the settings save, and two connections sharing a device identity. Happy-path +reconnect and single-save fault tests do not establish those guarantees. + +Replay pages contain at most 64 events and approximately 1 MiB of aggregate event +JSON. One atomic manifest can exceed that budget; it is sent alone so replay still +advances. `endEventId` identifies the page boundary while `headEventId` identifies +the vault head. Deploy the updated client and server together: older clients +require each batch to reach the vault head. Local partial replay folds retain only +latest document heads, the latest manifest, and matching outstanding receipts; +local files are reconciled only when catchup reaches the head. + +Permanent destination failures use `Recovered ` in the +vault root, resolving any occupied name. The journal retains the original bytes +when content is replaced. Permission or storage failures that also prevent that +fallback remain explicit recovery failures with the journal intact. + +A rejected retry retains an unconfirmed receipt until its request appears in the +event log. Absence from one catchup is not proof that an earlier in-flight attempt +cannot still commit. Receipt/artifact pruning remains an explicit operational concern; these tests do not claim +formal completeness or simulate arbitrary physical-media corruption. + +Reproduce the complete validation run from the repository root: + +```sh +E2E_ITERATIONS=100 E2E_SEED=8201 E2E_WORKERS=2 scripts/e2e.sh +``` + +## Sync regression coverage added after the September audit + +The normal test commands include these regressions: + +| Failure | Maintained coverage | +| ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Ignored occupant renamed into scope; incoming download discarded by unrelated notifications | `journal-protocol.test.ts` checks visible files, original IDs and actual server content. | +| Source path reused or moved during staging; editor save becomes a duplicate | `journal-interleavings.test.ts` injects moves, delete/recreate, move/recreate and content saves at every enumerated read, mutation and state-save boundary. Assertions specify bytes and document identities independently. | +| Journal/editor interaction followed by a crash | `journal-protocol.test.ts` crosses three editor operations with failures before/after state persistence, process replacement and simulated power loss, then checks another reset. | +| Restored server history falsely appears current | `server-restore.test.ts` restores actual SQLite backups and restarts Rust, covering equal/lower counters, dirty files, lost acknowledgements, interrupted recovery saves and old client state without checkpoints. | +| Oversized remote fallback and stale responses | `content-limits.test.ts` measures payload downloads, checks metadata-only responses and verifies highly fragmented local edits fall back to snapshots. | +| Adversarial diff work | Rust endpoint tests reject excessive operations without appending events and exercise Unicode reconstruction. | +| Observer exception kills sync | `observer.test.ts` runs throwing/rejecting subscribers in a child process and verifies a second sync; listener unit tests verify continued delivery. | +| Outgoing cursor version race | `outgoing-cursor.test.ts` changes version, identity, path and existence during a read, deliberately including identical bytes. | +| Unpersisted credentials | Rust config tests reject both missing users and missing user lists while preserving existing file bytes; initialization tests verify stable saved defaults. | +| Extensionless filenames enable merging | Extension unit tests include root/nested `md` and `txt` and dots in ancestor directories. | +| Corrupt vault blocks startup | Rust database regression opens a healthy vault alongside an invalid SQLite file and verifies the invalid file remains untouched. | + +The filesystem sweep reports the number of boundaries and injected interleavings. +A passing sweep covers this adapter model and these operations; arbitrary physical +media corruption and every possible editor/OS behavior remain outside its scope. diff --git a/frontend/test-support/canonical.ts b/frontend/test-support/canonical.ts new file mode 100644 index 00000000..0f30ffa4 --- /dev/null +++ b/frontend/test-support/canonical.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import type { StoredDatabase } from "sync-client"; +import { assertQuiescent, assertSameFiles } from "./oracles"; + +export interface CanonicalSnapshot { + headEventId: number; + fileManifest: { fileManifestId: number; entries: Record }; + documents: { documentId: string; vaultUpdateId: number }[]; +} +export interface InspectableClient { + files(): Map; + database(): Partial | undefined; +} + +export async function getJson(url: string, token: string): Promise { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + "Device-Id": "harness-observer" + }, + signal: AbortSignal.timeout(5_000) + }); + assert(response.ok, `Observer GET ${url}: HTTP ${response.status}`); + return (await response.json()) as T; +} + +/** Equality alone can pass when every replica loses the same bytes. Check the + * canonical UUIDs, content heads, receipts, journals and event cursor as well. + * Scenario-specific content/identity oracles still run separately. + */ +export async function assertCanonical( + clients: InspectableClient[], + vaultUrl: string, + token: string +): Promise { + assert.equal( + new Set(clients).size, + clients.length, + "Cannot compare a client to itself" + ); + const files = clients.map((client) => client.files()); + const states = clients.map((client) => client.database()); + assertSameFiles(files); + const snapshot = await getJson( + `${vaultUrl}/vault-snapshot`, + token + ); + const entries = snapshot.fileManifest.entries; + for (const id of Object.keys(entries)) + assert( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + id + ), + `Non-UUID document identity: ${id}` + ); + const heads = Object.fromEntries( + snapshot.documents.map((doc) => [doc.documentId, doc]) + ); + assert.equal( + Object.keys(heads).length, + snapshot.documents.length, + "Duplicate document heads" + ); + assert.deepEqual( + Object.keys(heads).sort(), + Object.keys(entries).sort(), + "Snapshot heads differ from membership" + ); + for (const [i, state] of states.entries()) { + assert(state?.initialized, `Client ${i} never initialized`); + assertQuiescent(state, entries, snapshot.headEventId); + assert.equal( + state.fileManifest?.fileManifestId, + snapshot.fileManifest.fileManifestId, + "Wrong manifest CAS parent" + ); + assert.deepEqual( + [...files[i].keys()].sort(), + Object.values(entries).sort(), + "Untracked or missing disk files" + ); + for (const id of Object.keys(entries)) { + assert( + state.documents?.[id]?.materialized, + `Unmaterialized document: ${id}` + ); + assert.equal( + state.documents?.[id]?.base?.vaultUpdateId, + heads[id]?.vaultUpdateId, + `Wrong content CAS parent: ${id}` + ); + } + } + for (const [id, path] of Object.entries(entries)) { + const doc = await getJson<{ contentBase64: string }>( + `${vaultUrl}/documents/${id}`, + token + ); + const serverBytes = new Uint8Array( + Buffer.from(doc.contentBase64, "base64") + ); + assert.deepEqual( + files[0].get(path), + serverBytes, + `Server bytes differ for ${id} at ${path}` + ); + const hash = createHash("sha256").update(serverBytes).digest("hex"); + for (const [i, state] of states.entries()) { + assert.equal( + state!.documents?.[id]?.base?.hash, + hash, + `Wrong base hash: client ${i}, ${id}` + ); + assert.equal( + state!.documents?.[id]?.observedHash, + hash, + `Wrong observed hash: client ${i}, ${id}` + ); + } + } + const events: { eventId: number; requestId: string }[] = []; + let after = 0; + do { + const batch = await getJson<{ + headEventId: number; + endEventId?: number; + events: { eventId: number; requestId: string }[]; + }>(`${vaultUrl}/events-since?after=${after}`, token); + assert.equal( + batch.headEventId, + snapshot.headEventId, + "Server changed while checking convergence" + ); + const end = batch.endEventId ?? batch.headEventId; + assert.equal( + end, + batch.events.at(-1)?.eventId ?? after, + "Event page cursor differs from its contents" + ); + if (after < snapshot.headEventId) + assert(end > after, "Event page made no progress"); + events.push(...batch.events); + after = end; + } while (after < snapshot.headEventId); + assert.deepEqual( + events.map((e) => e.eventId), + Array.from({ length: snapshot.headEventId }, (_, i) => i + 1), + "Server event log is not contiguous and totally ordered" + ); + assert.equal( + new Set(events.map((e) => e.requestId)).size, + events.length, + "An idempotent request emitted multiple events" + ); + return snapshot; +} diff --git a/frontend/test-support/content-limits.test.ts b/frontend/test-support/content-limits.test.ts new file mode 100644 index 00000000..baf510a4 --- /dev/null +++ b/frontend/test-support/content-limits.test.ts @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +test("stale manifest reconciliation must not fetch excluded remote content", async () => { + const f = await fixture(); + try { + const id = randomUUID(), + big = randomUUID(); + await f.put("/documents/" + id, f.content("A")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.disk.userRename("a.md", "renamed.md"); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "renamed.md" + }); + let injected = false, + downloaded = 0; + f.setFetchHook(async (input, init) => { + const path = String(input); + if ( + !injected && + init?.method === "PUT" && + path.endsWith("/file-manifest") + ) { + injected = true; + await f.put("/documents/" + big, f.content("B".repeat(4096))); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "a.md", [big]: "big.bin" } + }); + } + const response = await fetch(input, init); + if ( + path.includes("/documents/" + big) && + !path.endsWith("/metadata") && + (!init?.method || init.method === "GET") + ) + downloaded += (await response.clone().arrayBuffer()).byteLength; + return response; + }); + await f.client.setSetting("isSyncEnabled", true); + + assert(injected); + assert(!f.disk.userFiles().has("big.bin")); + assert.equal(downloaded, 0); + } finally { + await f.dispose(); + } +}); + +test("stale content acknowledgements contain metadata without the excluded payload", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + await f.put("/documents/" + id, f.content("B".repeat(4096))); + const stale = await f.put("/documents/" + id, f.content("LOCAL")); + assert.equal(stale.type, "StaleBase"); + assert.equal(stale.contentSize, 4096); + assert.equal(stale.contentBase64, undefined); + const metadata = await f.get("/documents/" + id + "/metadata"); + assert.equal(metadata.contentSize, 4096); + assert.equal(metadata.contentBase64, undefined); + } finally { + await f.dispose(); + } +}); + +test("highly fragmented local edits use a snapshot within the server operation budget", async () => { + const { fixture } = await import("./sync-fixture"); + const { toStoredSnapshot } = await import( + "../sync-client/src/sync-operations/content" + ); + const f = await fixture({}); + f.contentCache.resize(1024 * 1024); + const original = Array.from({ length: 12_000 }, (_, i) => `word${i}`).join( + " " + ); + const edited = Array.from({ length: 12_000 }, (_, i) => + i % 2 ? `changed${i}` : `word${i}` + ).join(" "); + f.contentCache.put(1, bytes(original)); + const syncer = f.syncer as unknown as { + pushContent( + path: string, + version: number, + snapshot: unknown + ): Promise<{ type: string }>; + }; + const request = await syncer.pushContent( + "a.md", + 1, + await toStoredSnapshot({ content: bytes(edited) }) + ); + assert.equal(request.type, "Snapshot"); +}); diff --git a/frontend/test-support/engine-regressions.test.ts b/frontend/test-support/engine-regressions.test.ts new file mode 100644 index 00000000..517d1360 --- /dev/null +++ b/frontend/test-support/engine-regressions.test.ts @@ -0,0 +1,910 @@ +import assert from "node:assert/strict"; +import { PermanentSyncError } from "../sync-client/src/errors/errors"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { allocatePortablePath } from "../sync-client/src/utils/portable-path"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import type { LocalChange } from "../sync-client/src/sync-operations/local-changes"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const head = (id: string, version: number, content: string) => ({ + documentId: id, + vaultUpdateId: version, + contentSize: bytes(content).length, + updatedDate: "2026-09-19T00:00:00Z", + userId: "user", + deviceId: "device" +}); + +async function fixture( + contents: Record = { "a.md": "local" }, + service: Partial = {} +) { + const disk = new MemoryDisk(); + const initial = emptyState("test"); + initial.initialized = true; + for (const [path, content] of Object.entries(contents)) { + await disk.userWrite(path, bytes(content)); + const id = path.split(".")[0]; + initial.local[id] = path; + initial.documents[id] = { + materialized: true, + observedHash: (await toStoredSnapshot({ content: bytes(content) })) + .hash + }; + } + initial.fileManifest = { fileManifestId: 1, entries: { ...initial.local } }; + const persistence = new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + "test", + async () => (await persistence.load()).database as StoredDatabase + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + const changes: LocalChange[] = []; + const files = new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} } + ); + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteVaultUpdateReceived: new EventListeners() + } as unknown as WebSocketManager; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket, + files, + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(1024), + { entries: changes, save: async () => {} } + ); + const internals = syncer as unknown as { + finishPending(): Promise; + preparePush(): Promise; + scan(): Promise; + incorporateEventBatch( + batch: import("../sync-client/src/services/types/EventBatch").EventBatch + ): Promise; + stopped: boolean; + incorporateFileManifest(remote: { + fileManifestId: number; + entries: Record; + }): Promise; + incorporateContent(remote: ReturnType): Promise; + }; + return { + disk, + database, + files, + settings, + syncer, + internals, + persistence, + changes + }; +} + +test("remote binary replacement retains displaced bytes outside the visible namespace", async () => { + const f = await fixture({ "a.bin": "private edit" }); + const next = structuredClone(f.database.state); + await f.files.apply(next, { + a: { + expected: await f.files.snapshot("a.bin"), + replacement: await toStoredSnapshot({ content: bytes("remote") }) + } + }); + assert.deepEqual(f.disk.userFiles(), new Map([["a.bin", bytes("remote")]])); + const retained = (await f.disk.listFilesRecursively()).filter((path) => + path.startsWith(".vault-link-sync/recovery/") + ); + assert( + ( + await Promise.all(retained.map((path) => f.disk.readSnapshot(path))) + ).some( + (value) => + value && + Buffer.from(value.content).toString() === "private edit" + ) + ); + await f.files.recover(); + assert.deepEqual( + await f.disk.listFilesRecursively(), + ["a.bin", ...retained].sort() + ); +}); + +test("remote deletion retains unsent bytes for recovery", async () => { + const f = await fixture(); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + assert.equal(f.disk.userFiles().size, 0); + const retained = await f.disk.listFilesRecursively(); + assert( + ( + await Promise.all(retained.map((path) => f.disk.readSnapshot(path))) + ).some( + (value) => + value && Buffer.from(value.content).toString() === "local" + ) + ); +}); + +test("rejected content recovery remains idempotent across a rename and network failure", async () => { + let offline = true; + const f = await fixture(undefined, { + getDocumentVersionContent: async () => { + if (offline) throw new Error("offline"); + return bytes("base"); + } + }); + const snapshot = (await f.files.snapshot("a.md"))!; + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.pending = { + type: "content", + documentId: "a", + snapshot, + request: { + requestId: "request", + parentVersionId: 1, + content: { type: "Snapshot", value: snapshot.contentBase64 } + }, + response: { + type: "StaleBase", + ...head("a", 2, "remote") + } + }; + await f.database.save(); + await f.disk.userRename("a.md", "renamed.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "renamed.md" + }); + await assert.rejects(f.internals.finishPending(), /offline/); + offline = false; + await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.equal(f.database.state.local.a, "renamed.md"); +}); + +test("conflict names respect component limits without imposing a total path limit", () => { + const wanted = "a".repeat(235) + ".md"; + const allocated = allocatePortablePath( + wanted, + "00000000-0000-4000-8000-000000000001", + { other: wanted } + ); + assert(Buffer.byteLength(allocated) <= 255); + assert(allocated.endsWith(".md")); + assert.notEqual(allocated, wanted); + const longPath = "folder/".repeat(100) + "note.md"; + assert.equal(allocatePortablePath(longPath, "id", {}), longPath); +}); + +test("an unmaterialized file with a directory at its path is deconflicted", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("remote") } + ); + await f.disk.userWrite("folder/note.md", bytes("local")); + f.database.state.local.a = "folder"; + f.database.state.documents.a = { materialized: false }; + f.database.state.fileManifest.entries.a = "folder"; + await f.database.save(); + await f.internals.incorporateContent(head("a", 2, "remote")); + assert( + [...f.disk.userFiles().values()].some( + (value) => Buffer.from(value).toString() === "remote" + ) + ); + assert( + [...f.disk.userFiles().values()].some( + (value) => Buffer.from(value).toString() === "local" + ) + ); +}); + +test("ignored materialized content survives remote deletion", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); + assert.deepEqual(f.database.state.fileManifest.entries, {}); +}); + +test("unfinished bootstrap cannot be rebound to another vault", async () => { + const database = emptyState(JSON.stringify(["http://example.test", "old"])); + database.bootstrap = { + headEventId: 1, + fileManifest: { fileManifestId: 1, entries: {} }, + documents: [] + }; + const persistence = new MemoryPersistence({ + settings: { remoteUri: "http://example.test", vaultName: "old" }, + database + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence + }); + try { + await assert.rejects( + client.setSettings({ vaultName: "new" }), + /separate state store/ + ); + assert.equal(persistence.snapshot().settings!.vaultName, "old"); + } finally { + await client.destroy(); + } +}); + +test("local swap arriving after a remote journal is saved preserves both identities and contents", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let fired = false; + f.persistence.boundary = async (label) => { + if ( + fired || + label !== "durable:save" || + !f.persistence.snapshot().database?.application + ) + return; + fired = true; + for (const [oldPath, relativePath] of [ + ["a.md", "tmp.md"], + ["b.md", "a.md"], + ["tmp.md", "b.md"] + ]) { + await f.disk.userRename(oldPath, relativePath); + await f.syncer.syncLocallyUpdatedFile({ oldPath, relativePath }); + } + }; + const next = structuredClone(f.database.state); + next.local = { a: "remote-a.md", b: "remote-b.md" }; + await f.files.apply(next); + assert(fired); + assert.equal( + Buffer.from( + f.disk.userFiles().get(f.database.state.local.a)! + ).toString(), + "A" + ); + assert.equal( + Buffer.from( + f.disk.userFiles().get(f.database.state.local.b)! + ).toString(), + "B" + ); + assert.deepEqual(f.database.state.local, { a: "b.md", b: "a.md" }); +}); + +test("a rejected upload is retained locally and does not block another file or a corrected snapshot", async () => { + let reject = true; + const sent: string[] = []; + const f = await fixture( + { "a.md": "too large", "b.md": "other" }, + { + putFileContent: async (id) => { + sent.push(id); + if (id === "a" && reject) + throw new PermanentSyncError("HTTP 413"); + return { + type: "Accepted", + ...head(id, sent.length + 2, "small") + }; + } + } + ); + f.internals.stopped = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.match(f.database.state.documents.a.rejected!.message, /413/); + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(sent, ["a", "b"]); + assert.equal(await f.internals.preparePush(), false); + await f.disk.userWrite("a.md", bytes("small")); + reject = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(sent, ["a", "b", "a"]); + assert.equal(f.database.state.documents.a.rejected, undefined); +}); + +test("a transient upload failure retries the exact request and recovers its acknowledgement", async () => { + const requests: unknown[] = []; + const f = await fixture(undefined, { + putFileContent: async (_id, request) => { + requests.push(structuredClone(request)); + if (requests.length === 1) throw new Error("lost upload reply"); + return { type: "Accepted", ...head("a", 2, "local") }; + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + await assert.rejects(f.internals.finishPending(), /lost upload reply/); + assert(f.database.state.pending); + await f.internals.finishPending(); + assert.equal(requests.length, 2); + assert.deepEqual(requests[1], requests[0]); + assert.equal(f.database.state.documents.a.base!.vaultUpdateId, 2); + assert.equal(f.database.state.documents.a.rejected, undefined); + assert.equal(f.database.state.pending, undefined); +}); + +test("a persisted upload rejection finishes locally after a recovery write fails", async () => { + let uploads = 0; + const f = await fixture(undefined, { + putFileContent: async () => { + uploads++; + throw new PermanentSyncError("HTTP 413"); + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert(pending); + const { requestId } = pending.request; + const retained = `.vault-link-sync/requests/${requestId}.content`; + const { boundary } = f.disk; + f.disk.boundary = (label): void => { + if (label === `before:write:${retained}`) + throw new Error("recovery write failed"); + }; + await assert.rejects(f.internals.finishPending(), /recovery write failed/); + assert.equal( + f.persistence.snapshot().database?.pending?.rejection, + "HTTP 413" + ); + f.disk.boundary = boundary; + await f.internals.finishPending(); + assert.equal(uploads, 1); + assert.equal(f.database.state.pending, undefined); + assert.equal(f.database.state.documents.a.rejected?.message, "HTTP 413"); + assert.deepEqual( + (await f.disk.readSnapshot(retained))?.content, + bytes("local") + ); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); +}); + +test("event replay incorporates an earlier accepted upload after a permanent retry rejection", async () => { + const f = await fixture(undefined, { + putFileContent: async () => { + throw new PermanentSyncError( + "HTTP 413 after proxy reconfiguration" + ); + }, + getDocumentVersionContent: async () => bytes("local") + }); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert(pending); + const { requestId } = pending.request; + await f.internals.finishPending(); + await assert.rejects(f.syncer.waitUntilFinished(), /413/); + f.database.state.lastSeenUpdateId = 1; + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "local") + } + ] + }); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + assert.equal(f.database.state.pending, undefined); + assert.equal(await f.internals.preparePush(), false); + await f.syncer.waitUntilFinished(); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); +}); + +test("rejected manifests can be replaced after editing the local namespace", async () => { + let reject = true; + const f = await fixture( + {}, + { + pushFileManifest: async () => { + if (reject) throw new PermanentSyncError("HTTP 413"); + return { type: "Accepted", fileManifestId: 3 }; + } + } + ); + f.database.state.fileManifest.entries = { a: "old.md" }; + f.database.state.pending = { + type: "fileManifest", + request: { + requestId: "request", + parentFileManifestId: 1, + entries: { a: "new.md" } + } + }; + f.database.state.local = { a: "new.md" }; + f.database.state.documents.a = { + materialized: false, + base: { ...head("a", 1, "x"), hash: "hash" } + }; + await f.database.save(); + await f.internals.finishPending(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + f.database.state.local = {}; + reject = false; + assert(await f.internals.preparePush()); + await f.internals.finishPending(); + assert.deepEqual(f.database.state.fileManifest.entries, {}); +}); + +test("offline delete/recreate identities survive restarting before any network connection", async () => { + const key = JSON.stringify(["http://offline.test", "test"]); + const initial = emptyState(key); + initial.initialized = true; + initial.local.a = "a.md"; + initial.documents.a = { materialized: true }; + initial.fileManifest = { fileManifestId: 1, entries: { a: "a.md" } }; + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("same")); + const persistence = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + const first = await SyncClient.create({ fs: disk, persistence }); + await disk.userDelete("a.md"); + await first.syncLocallyDeletedFile("a.md"); + await disk.userWrite("a.md", bytes("same")); + await first.syncLocallyCreatedFile("a.md"); + await first.destroy(); + const second = await SyncClient.create({ fs: disk, persistence }); + try { + const internals = second as unknown as { + syncer: { scan(): Promise }; + database: Database; + }; + await internals.syncer.scan(); + assert.equal(internals.database.state.local.a, undefined); + assert.equal( + Object.values(internals.database.state.local).filter( + (path) => path === "a.md" + ).length, + 1 + ); + } finally { + await second.destroy(); + } +}); + +test("offline notifications bind an otherwise empty client to its vault", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("local")); + const persistence = new MemoryPersistence({ + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + const client = await SyncClient.create({ fs: disk, persistence }); + await client.syncLocallyCreatedFile("a.md"); + await assert.rejects( + client.setSettings({ vaultName: "other" }), + /separate state store/ + ); + await client.destroy(); + const stored = persistence.snapshot(); + stored.settings!.vaultName = "other"; + await persistence.save(stored); + await assert.rejects( + SyncClient.create({ fs: disk, persistence }), + /another vault/ + ); +}); + +test("unignoring a deleted file resumes the deferred remote deletion without resurrection", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: {} + }); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert.equal(f.disk.userFiles().size, 0); + assert.deepEqual(f.database.state.local, {}); +}); + +test("an ignored file does not move when a remote rename targets its path", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "c.md", b: "a.md" } + }); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes("A")], + ["b.md", bytes("B")] + ]) + ); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes("A")], + ["b.md", bytes("B")] + ]) + ); +}); + +for (const kind of ["path", "content"] as const) { + test(`a remote ${kind} change that was reverted before observation preserves the local edit`, async () => { + const f = await fixture( + { "a.md": "local" }, + { + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes(vaultUpdateId === 2 ? "temporary" : "base") + } + ); + f.database.state.lastSeenUpdateId = 1; + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + if (kind === "path") { + await f.disk.userRename("a.md", "c.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "c.md" + }); + } + await f.internals.incorporateEventBatch({ + headEventId: 3, + events: [2, 3].map((id) => + kind === "path" + ? { + eventId: id, + requestId: String(id), + type: "fileManifest", + fileManifest: { + fileManifestId: id, + entries: { a: id === 2 ? "b.md" : "a.md" } + } + } + : { + eventId: id, + requestId: String(id), + type: "content", + document: head( + "a", + id, + id === 2 ? "temporary" : "base" + ) + } + ) + }); + assert.equal( + f.database.state.local.a, + kind === "path" ? "c.md" : "a.md" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))!.content + ).toString(), + "local" + ); + assert.equal(f.database.state.lastSeenUpdateId, 3); + }); +} + +test("concurrent settings updates cannot bind the engine to a different vault than its settings", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence(); + const client = await SyncClient.create({ fs: disk, persistence }); + try { + await Promise.all([ + client.setSettings({ vaultName: "changed" }), + client.setSettings({ maxFileSizeMB: 20 }) + ]); + const stored = persistence.snapshot(); + assert.equal( + stored.database!.vaultKey, + JSON.stringify([ + stored.settings!.remoteUri, + stored.settings!.vaultName + ]) + ); + assert.equal(stored.settings!.maxFileSizeMB, 20); + } finally { + await client.destroy(); + } +}); + +test("destroy waits for an in-progress offline notification to become durable", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence(); + const client = await SyncClient.create({ fs: disk, persistence }); + await disk.userWrite("a.md", bytes("local")); + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + persistence.boundary = async (label) => { + if (label === "before:save") { + reached.resolve(); + await release.promise; + } + }; + const notification = client.syncLocallyCreatedFile("a.md"); + await reached.promise; + let destroyed = false; + const destroying = client.destroy().then(() => { + destroyed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(destroyed, false); + release.resolve(); + await Promise.all([notification, destroying]); +}); + +test("ignoring a locally renamed file defers its move without forgetting the local rename", async () => { + const f = await fixture(); + f.database.state.documents.a.base = { + ...head("a", 1, "local"), + hash: f.database.state.documents.a.observedHash! + }; + await f.disk.userRename("a.md", "ignored.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "ignored.md" + }); + await f.settings.setSettings({ ignorePatterns: ["ignored.md"] }); + await f.internals.scan(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "a.md" } + }); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert.equal(f.database.state.local.a, "ignored.md"); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("ignored.md"))!.content + ).toString(), + "local" + ); +}); + +test("a notified deletion of an ignored file does not delete its server membership", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md"] }); + await f.disk.userDelete("a.md"); + await f.syncer.syncLocallyDeletedFile("a.md"); + await f.internals.scan(); + f.internals.stopped = false; + assert.equal(await f.internals.preparePush(), false); + assert.equal(f.database.state.local.a, undefined); + await f.settings.setSettings({ ignorePatterns: [] }); + await f.internals.incorporateFileManifest(f.database.state.fileManifest); + assert(await f.internals.preparePush()); + assert.equal(f.database.state.pending?.type, "fileManifest"); +}); + +test("an incoming ignored document cannot force an existing ignored file to move", async () => { + const f = await fixture(); + await f.settings.setSettings({ ignorePatterns: ["a.md", "*conflict*"] }); + await f.internals.incorporateFileManifest({ + fileManifestId: 2, + entries: { a: "b.md", other: "a.md" } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); + assert.equal( + new Set(Object.values(f.database.state.local)).size, + Object.keys(f.database.state.local).length + ); +}); + +for (const boundary of ["visible", "durable"] as const) { + for (const action of [ + "move-installed", + "recreate-staged-source" + ] as const) { + test(`${action} at a ${boundary} filesystem boundary keeps original identities and all bytes`, async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let changed = false; + f.disk.boundary = async (label) => { + if (changed || !label.startsWith(`${boundary}:rename:`)) return; + if (action === "move-installed" && label.endsWith("->b.md")) { + changed = true; + await f.disk.userRename("b.md", "later.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "later.md" + }); + } else if ( + action === "recreate-staged-source" && + label.includes("rename:a.md->") + ) { + changed = true; + await f.disk.userWrite("a.md", bytes("new")); + await f.syncer.syncLocallyCreatedFile("a.md"); + } + }; + const next = structuredClone(f.database.state); + next.local = { a: "b.md", b: "a.md" }; + next.fileManifest = { + fileManifestId: 2, + entries: { ...next.local } + }; + await f.files.apply(next); + assert(changed); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))! + .content + ).toString(), + "A" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.b))! + .content + ).toString(), + "B" + ); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((value) => Buffer.from(value).toString()) + .sort(), + action === "move-installed" ? ["A", "B"] : ["A", "B", "new"] + ); + f.disk.crash(true); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.a))! + .content + ).toString(), + "A" + ); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot(f.database.state.local.b))! + .content + ).toString(), + "B" + ); + }); + } +} + +test("rejected content is reported to callers and an explicit retry clears the rejection", async () => { + const f = await fixture(undefined, { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + await f.internals.finishPending(); + await assert.rejects(f.syncer.waitUntilFinished(), /413/); + assert(f.syncer.isBusy); + await f.syncer.retryRejectedRequests(); + assert(await f.internals.preparePush()); +}); + +test("notification saves cannot roll back an uncertain engine commit, even if the first reload fails", async () => { + const key = JSON.stringify(["http://offline.test", "test"]); + const initial = emptyState(key); + initial.initialized = true; + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + let failLoad = false; + const persistence = { + load: async () => { + if (failLoad) { + failLoad = false; + throw new Error("reload failed"); + } + return store.load(); + }, + save: async (state: Parameters[0]) => + store.save(state) + }; + const client = await SyncClient.create({ fs: disk, persistence }); + try { + const internals = client as unknown as { + database: Database; + syncer: Syncer; + }; + const next = structuredClone(initial); + next.lastSeenUpdateId = 2; + store.boundary = (label) => { + if (label === "durable:save") { + store.boundary = () => {}; + throw new Error("durable save failed"); + } + }; + await assert.rejects( + internals.database.commit(next), + /durable save failed/ + ); + await disk.userWrite("a.md", bytes("new")); + failLoad = true; + await assert.rejects( + client.syncLocallyCreatedFile("a.md"), + /reload failed/ + ); + await internals.syncer.flushLocalChanges(); + assert.equal(store.snapshot().database!.lastSeenUpdateId, 2); + await internals.database.recoverPersistence(); + assert.equal(internals.database.state.lastSeenUpdateId, 2); + } finally { + await client.destroy(); + } +}); + +test( + "a settings listener can await client shutdown without deadlocking the lifecycle lock", + { timeout: 1_000 }, + async () => { + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: new MemoryPersistence() + }); + let notified = false; + client.onSettingsChanged.add(async () => { + await client.destroy(); + notified = true; + }); + await client.setSettings({ maxFileSizeMB: 20 }); + assert(notified); + } +); diff --git a/frontend/test-support/event-delivery.test.ts b/frontend/test-support/event-delivery.test.ts new file mode 100644 index 00000000..a31d58e9 --- /dev/null +++ b/frontend/test-support/event-delivery.test.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import type { EventBatch } from "../sync-client/src/services/types/EventBatch"; +import type { EventRecord } from "../sync-client/src/services/types/EventRecord"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +for (const [count, delivery] of [ + [1, "before the loop"], + [3, "before the loop"], + [3, "during the first batch"] +] as const) { + test( + `${count} WS batches delivered ${delivery} drain without polling or another notification`, + { timeout: 5_000 }, + async () => { + const logger = new Logger(); + const initial = emptyState("event-delivery"); + initial.initialized = true; + const persistence = new MemoryPersistence({ database: initial }); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + initial.vaultKey, + async () => + (await persistence.load()).database as StoredDatabase + ); + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + assert.equal(settings.getSettings().syncIntervalMs, 0); + const disk = new MemoryDisk(); + const initializing = Promise.withResolvers(); + const initialized = Promise.withResolvers(); + const firstBatch = Promise.withResolvers(); + const continueBatch = Promise.withResolvers(); + let heldBatch = false; + const applied: number[] = []; + persistence.boundary = async (label) => { + if (label !== "durable:save") return; + const cursor = + persistence.snapshot().database!.lastSeenUpdateId; + if (cursor && applied.at(-1) !== cursor) applied.push(cursor); + if ( + delivery === "during the first batch" && + cursor === 1 && + !heldBatch + ) { + heldBatch = true; + firstBatch.resolve(); + await continueBatch.promise; + } + }; + const config = { + initialize: async () => { + initializing.resolve(); + await initialized.promise; + }, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const events: EventRecord[] = Array.from( + { length: count }, + (_, index) => ({ + eventId: index + 1, + requestId: `manifest-${index + 1}`, + type: "fileManifest", + fileManifest: { fileManifestId: index + 1, entries: {} } + }) + ); + const service = { + // A correct client may drain the queue or catch up over HTTP. Either + // must reach the same head without the test manually waking it. + events: async (after: number): Promise => ({ + headEventId: count, + events: events.filter((event) => event.eventId > after) + }) + } as unknown as SyncService; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => unknown + >() + }; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service, + websocket as unknown as WebSocketManager, + new FileOperations(disk.session(), database, config), + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(1024) + ); + const deliver = async (event: EventRecord) => { + await websocket.onRemoteVaultUpdateReceived.triggerAsync({ + headEventId: event.eventId, + events: [event] + }); + }; + try { + syncer.start(); + await initializing.promise; + if (delivery === "before the loop") { + for (const event of events) await deliver(event); + initialized.resolve(); + } else { + await deliver(events[0]); + initialized.resolve(); + await firstBatch.promise; + for (const event of events.slice(1)) await deliver(event); + continueBatch.resolve(); + } + await syncer.waitUntilFinished(); + assert.equal( + database.state.lastSeenUpdateId, + count, + "Client became idle with already-delivered events unapplied" + ); + assert.deepEqual( + applied, + events.map((event) => event.eventId), + "Every event must be committed in order" + ); + assert.equal(database.state.fileManifest.fileManifestId, count); + assert.equal(database.state.application, undefined); + assert.equal(database.state.pending, undefined); + assert.equal(syncer.isBusy, false); + } finally { + initialized.resolve(); + continueBatch.resolve(); + await syncer.stop(); + } + } + ); +} diff --git a/frontend/test-support/harness.test.ts b/frontend/test-support/harness.test.ts new file mode 100644 index 00000000..e4604b4b --- /dev/null +++ b/frontend/test-support/harness.test.ts @@ -0,0 +1,316 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { NetworkFaults, classifyRequest } from "./network"; +import { + ContentLedger, + assertManifest, + assertSameFiles, + pathDecision +} from "./oracles"; +import { Random } from "./random"; +import { TestRunner } from "../deterministic-tests/src/test-runner"; +import type { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { ManagedWebSocketFactory } from "../deterministic-tests/src/managed-websocket"; + +const bytes = (text: string) => new TextEncoder().encode(text); + +test("consistency oracle rejects self comparison, divergent paths and invalid UTF-8 bytes", () => { + const first = new Map([["a.bin", new Uint8Array([0xff])]]); + assert.throws(() => assertSameFiles([first, first]), /itself/); + assert.throws(() => + assertSameFiles([first, new Map([["a.bin", new Uint8Array([0xfe])]])]) + ); + assert.throws(() => assertSameFiles([first, new Map()])); + assertSameFiles([first, structuredClone(first)]); +}); + +test("ledger catches shared loss, duplicated bytes, and unrelated loss in delete workloads", () => { + const ledger = new ContentLedger(); + ledger.add("first-marker"); + ledger.add("second-marker"); + ledger.removedBy("first-marker", "explicit delete of doc A"); + assert.throws( + () => ledger.assertPreserved(new Map()), + /second-marker.*lost/ + ); + assert.throws( + () => + ledger.assertPreserved( + new Map([["a", bytes("second-marker second-marker")]]) + ), + /duplicated/ + ); + ledger.assertPreserved(new Map([["a", bytes("second-marker")]])); +}); + +test("memory adapter enforces v4 operations and snapshot ownership", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("folder/a", bytes("A")); + const snap = (await disk.readSnapshot("folder/a"))!; + snap.content[0] = 0; + assert.equal( + Buffer.from((await disk.readSnapshot("folder/a"))!.content).toString(), + "A" + ); + await assert.rejects( + disk.write("folder/a", { content: bytes("B") }), + /exists/ + ); + await disk.userWrite("folder/b", bytes("B")); + await assert.rejects(disk.rename("folder/a", "folder/b"), /exists/); + await assert.rejects(disk.delete("folder/a"), /regular file/); + await assert.rejects(disk.delete("folder"), /regular file/); + await assert.rejects(disk.readSnapshot("../outside"), /Unsafe/); + await assert.rejects(disk.stat("folder/a/child"), /ancestor/); + await assert.rejects(disk.readSnapshot("folder"), /directory/); +}); + +for (const phase of ["before", "visible", "durable"] as const) { + for (const powerLoss of [false, true]) { + test(`${powerLoss ? "power loss" : "process death"} at ${phase} rename boundary`, async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a", bytes("A")); + const session = disk.session(); + let fired = false; + disk.boundary = (label) => { + if (label === `${phase}:rename:a->b`) { + fired = true; + disk.crash(powerLoss); + throw new Error("crash"); + } + }; + await assert.rejects(session.rename("a", "b"), /crash/); + assert(fired); + disk.boundary = () => {}; + const moved = + phase === "durable" || (phase === "visible" && !powerLoss); + assert.deepEqual([...disk.userFiles().keys()], [moved ? "b" : "a"]); + assert.throws( + () => session.write("stale", { content: bytes("bad") }), + /crashed/ + ); + }); + } +} + +test("persistence has deep ownership and both old/new uncertain save outcomes", async () => { + for (const phase of ["before", "durable"]) { + const store = new MemoryPersistence({ settings: { vaultName: "old" } }); + const loaded = await store.load(); + loaded.settings!.vaultName = "mutated"; + assert.equal(store.snapshot().settings!.vaultName, "old"); + store.boundary = (label) => { + if (label === `${phase}:save`) throw new Error("save failed"); + }; + await assert.rejects( + store.save({ settings: { vaultName: "new" } }), + /save failed/ + ); + assert.equal( + store.snapshot().settings!.vaultName, + phase === "before" ? "old" : "new" + ); + } +}); + +test("CAS hooks match v4 create, update and manifest and preserve retry payload", async () => { + for (const kind of ["create", "content", "manifest"] as const) { + const net = new NetworkFaults(); + const url = `http://test/vaults/test/${kind === "manifest" ? "file-manifest" : "documents/uuid"}`; + const body = { + requestId: "same", + parentVersionId: kind === "create" ? null : 3 + }; + const init = { method: "PUT", body: JSON.stringify(body) }; + assert.equal(classifyRequest(url, init)?.kind, kind); + let committed = 0; + const wrapped = net.wrap(async () => { + committed++; + return new Response('{"type":"Accepted"}'); + }); + net.arm(kind); + await assert.rejects(wrapped(url, init), /Injected after-commit/); + await net.wait(); + assert.throws(() => net.assertConsumed(), /never retried/); + assert.equal(committed, 1); + await wrapped(url, init); + net.assertConsumed(); + await assert.rejects( + wrapped(url, { + ...init, + body: JSON.stringify({ ...body, changed: true }) + }), + /Retry changed/ + ); + } + assert.equal( + classifyRequest("http://test/documents", { method: "POST" }), + undefined + ); +}); + +test("unconsumed and before-send faults cannot silently pass", async () => { + const net = new NetworkFaults(); + net.arm("manifest", "before"); + assert.throws(() => net.assertConsumed(), /never fired/); + let called = false; + await assert.rejects( + net.wrap(async () => { + called = true; + return new Response(); + })("http://test/file-manifest", { + method: "PUT", + body: '{"requestId":"r"}' + }) + ); + assert(!called); + assert.throws(() => net.assertConsumed(), /never retried/); + await net.wrap(async () => new Response())("http://test/file-manifest", { + method: "PUT", + body: '{"requestId":"r"}' + }); + net.assertConsumed(); +}); + +test("HTTP observation checkpoint blocks catchup, resumes, and respects abort", async () => { + const network = new NetworkFaults(); + const wrapped = network.wrap(async () => new Response("{}")); + network.pauseObservation(); + let delivered = false; + const request = wrapped("http://test/events-since?after=0").then(() => { + delivered = true; + }); + await network.waitForObservation(); + assert(!delivered, "Catchup bypassed the observation checkpoint"); + network.resumeObservation(); + await request; + network.assertConsumed(); + network.pauseObservation(); + const controller = new AbortController(); + const interrupted = wrapped("http://test/vault-snapshot", { + signal: controller.signal + }); + const rejected = assert.rejects(interrupted, /abort/i); + await network.waitForObservation(); + controller.abort(); + await rejected; + network.resumeObservation(); + network.assertConsumed(); +}); + +test("independent path decision truth table covers create, delete and rename", () => { + for (const base of [undefined, "a", "b"]) + for (const local of [undefined, "a", "b"]) + for (const remote of [undefined, "a", "b"]) { + const result = pathDecision(base, local, remote); + assert.equal(result, remote !== base ? remote : local); + } + assertManifest({ a: "folder/é.md", b: "other.bin" }); + for (const entries of [ + { a: "A", b: "a" }, + { a: "é", b: "e\u0301" }, + { a: "x", b: "x/y" } + ]) + assert.throws(() => assertManifest(entries)); +}); + +test("seeded actions are repeatable, including seed zero", () => { + for (const seed of [0, 1, 12345, 0xffffffff]) { + const first = new Random(seed); + const second = new Random(seed); + assert.deepEqual( + Array.from({ length: 100 }, () => first.int(100)), + Array.from({ length: 100 }, () => second.int(100)) + ); + } +}); + +test("runner cleanup errors fail an otherwise successful test", async () => { + const server = { + isRunning: () => true, + resume: () => {} + } as unknown as ServerControl; + const runner = new TestRunner( + server, + new Logger(), + "unused", + "http://unused" + ); + let cleaned = 0; + Object.assign(runner, { + initializeAgents: async () => {}, + agents: [ + { + clientId: 0, + database: () => ({ evidence: "before cleanup" }), + disk: { image: () => ({ bytes: [0, 255] }) }, + network: { requests: [{ body: { requestId: "original" } }] }, + cleanup: async () => { + cleaned++; + throw new Error("late background error"); + } + } + ] + }); + const result = await runner.runTest("cleanup-probe", { + clients: 2, + steps: [] + }); + assert.equal(result.success, false); + assert.match(result.error!, /late background error/); + assert.equal(cleaned, 1, "Failed cleanup was run twice"); + assert.deepEqual(result.diagnostics, [ + { + client: 0, + database: { evidence: "before cleanup" }, + disk: { bytes: [0, 255] }, + requests: [{ body: { requestId: "original" } }] + } + ]); +}); + +test("disposed harness rejects late websocket reconnects", async () => { + const factory = new ManagedWebSocketFactory(); + const Socket = factory.constructorFn; + await factory.finish(); + assert.throws(() => new Socket("ws://unused"), /after harness cleanup/); +}); + +test("runner preserves primary failure and cleanup diagnostics", async () => { + const server = { + isRunning: () => true, + resume: () => {} + } as unknown as ServerControl; + const runner = new TestRunner( + server, + new Logger(), + "unused", + "http://unused" + ); + Object.assign(runner, { + initializeAgents: async () => {}, + executeStep: async () => { + throw new Error("primary assertion failed"); + }, + agents: [ + { + clientId: 0, + database: () => ({}), + disk: { image: () => ({}) }, + network: { requests: [] }, + cleanup: async () => { + throw new Error("cleanup failed too"); + } + } + ] + }); + const result = await runner.runTest("cleanup-probe", { + clients: 2, + steps: [{ type: "barrier" }] + }); + assert.equal(result.success, false); + assert.match(result.error!, /primary assertion failed/); + assert.match(result.error!, /cleanup failed too/); +}); diff --git a/frontend/test-support/journal-interleavings.test.ts b/frontend/test-support/journal-interleavings.test.ts new file mode 100644 index 00000000..93f0bbb0 --- /dev/null +++ b/frontend/test-support/journal-interleavings.test.ts @@ -0,0 +1,407 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, head } from "./sync-fixture"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; + +for (const origin of ["bootstrap", "history recovery"]) + test(`cancelling a replacement retains the ${origin} merge context`, async () => { + const f = await fixture( + { "a.md": "BASE\n" }, + { getDocumentVersionContent: async () => bytes("REMOTE\nBASE\n") } + ); + const recoveryBase = await toStoredSnapshot({ + content: bytes("BASE\n") + }); + if (origin === "bootstrap") + f.database.state.documents.a.bootstrap = true; + else f.database.state.documents.a.recoveryBase = recoveryBase; + await f.database.save(); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("BASE\nLOCAL\n")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + await f.internals.incorporateContent(head("a", 2, "REMOTE\nBASE\n")); + assert(injected); + const deferred = f.database.state.documents.a; + assert.equal(deferred.base, undefined); + if (origin === "bootstrap") assert.equal(deferred.bootstrap, true); + else assert.deepEqual(deferred.recoveryBase, recoveryBase); + f.disk.boundary = () => {}; + await f.settings.setSetting("ignorePatterns", []); + await f.internals.incorporateFileManifest( + f.database.state.fileManifest + ); + const content = Buffer.from( + f.disk.userFiles().get("private.md")! + ).toString(); + assert.equal(content.match(/REMOTE/g)?.length, 1); + assert.equal(content.match(/LOCAL/g)?.length, 1); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + assert.equal(f.database.state.documents.a.recoveryBase, undefined); + }); + +for (const timing of ["before", "durable"]) + test(`a save that exceeds the size limit ${timing} staging keeps its exact bytes`, async () => { + const f = await fixture({ "a.bin": "PUBLIC" }); + await f.settings.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + const large = "PRIVATE".repeat(100); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + `${timing}:rename:a.bin->.vault-link-sync/transactions/` + ) + ) + return; + injected = true; + await f.disk.userWrite("a.bin", bytes(large)); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.bin" }); + }; + await f.files.apply(structuredClone(f.database.state), { + a: { + expected: await toStoredSnapshot({ content: bytes("PUBLIC") }), + replacement: await toStoredSnapshot({ + content: bytes("REMOTE") + }) + } + }); + assert(injected); + assert.equal( + Buffer.from(f.disk.userFiles().get("a.bin")!).toString(), + large + ); + assert.equal(f.database.state.excluded?.a, "a.bin"); + assert.deepEqual(f.database.state.local, { a: "a.bin" }); + }); + +test("an excluded move after installation survives an uncertain journal acknowledgement", async () => { + const f = await fixture({ "a.md": "PUBLIC" }); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + let moved = false, + crashed = false; + f.disk.boundary = async (label) => { + if ( + moved || + !label.startsWith("durable:rename:") || + !label.endsWith(".output->a.md") + ) + return; + moved = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("SECRET")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + f.persistence.boundary = (label) => { + if (!moved || crashed || label !== "durable:save") return; + crashed = true; + throw new Error("lost journal acknowledgement"); + }; + await assert.rejects( + f.files.apply(structuredClone(f.database.state), { + a: { + expected: await toStoredSnapshot({ content: bytes("PUBLIC") }), + replacement: await toStoredSnapshot({ + content: bytes("PUBLIC REMOTE") + }) + } + }), + /lost journal acknowledgement/ + ); + assert(moved && crashed); + f.disk.boundary = () => {}; + f.persistence.boundary = () => {}; + await f.database.recoverPersistence(); + await f.files.recover(); + assert.deepEqual(f.database.state.local, { a: "private.md" }); + assert.equal( + Buffer.from(f.disk.userFiles().get("private.md")!).toString(), + "SECRET" + ); +}); + +test("moving a tracked file into an ignored path during a journal leaves it protected", async () => { + const f = await fixture({ "a.md": "PUBLIC", "b.md": "OTHER" }); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + const next = structuredClone(f.database.state); + next.local.b = "remote-b.md"; + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("SECRET")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + await f.files.apply(next); + assert(injected); + assert.equal( + Buffer.from(f.disk.userFiles().get("private.md") ?? []).toString(), + "SECRET" + ); + assert.equal(f.database.state.local.a, "private.md"); + assert.equal(f.database.state.excluded?.a, "a.md"); +}); + +test("a directory move protects each ignored descendant during a journal", async () => { + const f = await fixture({ "folder/a.md": "PUBLIC", "b.md": "OTHER" }); + await f.settings.setSetting("ignorePatterns", ["private/**"]); + const next = structuredClone(f.database.state); + next.local.b = "remote-b.md"; + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("folder/a.md", "private/a.md"); + await f.disk.userWrite("private/a.md", bytes("SECRET")); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "folder", + relativePath: "private" + }); + }; + await f.files.apply(next); + assert(injected); + assert.equal( + Buffer.from(f.disk.userFiles().get("private/a.md") ?? []).toString(), + "SECRET" + ); + assert.equal(f.database.state.local["folder/a"], "private/a.md"); + assert.equal(f.database.state.excluded?.["folder/a"], "folder/a.md"); +}); + +test("a later editor save replaces earlier unsynced text after a merge was prepared", async () => { + const f = await fixture({ "a.md": "base" }); + const next = structuredClone(f.database.state); + let saves = 0; + f.disk.boundary = async (label) => { + if (saves === 0 && label.startsWith("before:rename:a.md->")) { + saves++; + await f.disk.userWrite("a.md", bytes("base ONE")); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } else if ( + saves === 1 && + label.startsWith("before:write:") && + label.endsWith(".output") + ) { + saves++; + await f.disk.userWrite("a.md", bytes("base TWO")); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + }; + await f.files.apply(next, { + a: { + expected: await toStoredSnapshot({ content: bytes("base") }), + replacement: await toStoredSnapshot({ + content: bytes("base REMOTE") + }) + } + }); + assert.equal(saves, 2); + assert.equal( + Buffer.from(f.disk.userFiles().get("a.md")!).toString(), + "base TWO REMOTE" + ); + assert.deepEqual(f.database.state.local, { a: "a.md" }); +}); + +test("a content save during remote application remains on the same document", async () => { + const f = await fixture({ "a.md": "base" }); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + const next = structuredClone(f.database.state); + next.documents.a.base = { + ...head("a", 2, "base REMOTE"), + hash: (await toStoredSnapshot({ content: bytes("base REMOTE") })).hash + }; + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + "durable:rename:a.md->.vault-link-sync/transactions/" + ) + ) + return; + injected = true; + await f.disk.userWrite("a.md", bytes("base LOCAL")); + await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); + }; + await f.files.apply(next, { + a: { + expected: await toStoredSnapshot({ content: bytes("base") }), + replacement: await toStoredSnapshot({ + content: bytes("base REMOTE") + }) + } + }); + + assert.equal(f.disk.userFiles().size, 1); + const result = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert(result.includes("LOCAL") && result.includes("REMOTE")); +}); + +async function setup() { + const f = await fixture({ "a.md": "ORIGINAL", "b.md": "OTHER" }); + f.database.state.documents.a.base = { + ...head("a", 1, "ORIGINAL"), + hash: f.database.state.documents.a.observedHash! + }; + const next = structuredClone(f.database.state); + next.local = { a: "remote.md", b: "remote-b.md" }; + next.fileManifest = { fileManifestId: 3, entries: { ...next.local } }; + const expected = await toStoredSnapshot({ content: bytes("ORIGINAL") }); + const replacement = await toStoredSnapshot({ + content: bytes("ORIGINAL REMOTE") + }); + next.documents.a.base = { + ...head("a", 2, "ORIGINAL REMOTE"), + hash: replacement.hash + }; + return { f, next, writes: { a: { expected, replacement } } }; +} +test("editor changes preserve exact bytes and identities at every filesystem read, mutation and persistence boundary", async (t) => { + const { f, next, writes } = await setup(); + const labels: string[] = []; + const trace = (l: string) => { + labels.push(l); + }; + f.disk.boundary = trace; + f.persistence.boundary = trace; + await f.files.apply(next, writes); + const failures = []; + let exercised = 0; + for (const kind of ["move", "move-recreate", "delete-recreate", "save"]) + for (let target = 0; target < labels.length; target++) { + const { f, next, writes } = await setup(); + let count = 0, + injected = false, + from: string | undefined; + const boundary = async (l: string) => { + if (injected) return; + if (count++ !== target) return; + const files = f.disk.userFiles(); + from = files.has("a.md") + ? "a.md" + : files.has("remote.md") + ? "remote.md" + : undefined; + if (kind === "save" && !from) from = "a.md"; + if (!from) return; + injected = true; + exercised++; + if (kind === "save") { + await f.disk.userWrite( + from, + bytes( + Buffer.from( + files.get(from) ?? bytes("ORIGINAL") + ).toString() + " LOCAL" + ) + ); + await f.syncer.syncLocallyUpdatedFile({ + relativePath: from + }); + } else if (kind === "delete-recreate") { + await f.disk.userDelete(from); + await f.syncer.syncLocallyDeletedFile(from); + } else { + await f.disk.userRename(from, "mine.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: from, + relativePath: "mine.md" + }); + } + if (kind !== "move" && kind !== "save") { + await f.disk.userWrite(from, bytes("REPLACEMENT")); + await f.syncer.syncLocallyCreatedFile(from); + } + }; + f.disk.boundary = boundary; + f.persistence.boundary = boundary; + let error; + try { + await f.files.apply(next, writes); + } catch (e) { + error = String(e); + } + f.disk.boundary = () => {}; + f.persistence.boundary = () => {}; + try { + await f.database.recoverPersistence(); + await f.files.recover(); + await f.internals.scan(); + } catch (e) { + error += ";retry=" + String(e); + } + if (!injected) continue; + const fs = Object.fromEntries( + [...f.disk.userFiles()].map(([p, b]) => [ + p, + Buffer.from(b).toString() + ]) + ); + const bad = + kind === "save" + ? ![ + "ORIGINAL LOCAL REMOTE", + "ORIGINAL REMOTE LOCAL" + ].includes(fs[f.database.state.local.a]) || + Object.keys(f.database.state.local).length !== 2 + : kind === "delete-recreate" + ? f.database.state.local.a !== undefined + : f.database.state.local.a !== "mine.md" || + fs["mine.md"] !== "ORIGINAL REMOTE"; + if ( + bad || + (kind !== "move" && + kind !== "save" && + fs[from!] !== "REPLACEMENT") + ) + failures.push({ + kind, + target, + label: labels[target], + error, + ids: f.database.state.local, + files: fs + }); + } + assert( + exercised > 50, + "The sweep must actually exercise the injected actions" + ); + t.diagnostic( + `${labels.length} boundaries; ${exercised} editor interleavings` + ); + assert.deepEqual(failures, []); +}); diff --git a/frontend/test-support/journal-protocol.test.ts b/frontend/test-support/journal-protocol.test.ts new file mode 100644 index 00000000..568b9688 --- /dev/null +++ b/frontend/test-support/journal-protocol.test.ts @@ -0,0 +1,612 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; + +for (const exclusion of ["size", "ignore"] as const) + for (const phase of [ + "planned", + ...(exclusion === "size" ? ["staged", "prepared"] : []), + "installed" + ]) + for (const recovery of [ + "none", + "restart", + "before:save", + "durable:save" + ] as const) + test(`reenabling ${exclusion} exclusion at ${phase} preserves the content base through ${recovery}`, async () => { + const f = await fixture(); + try { + const installed = phase === "installed"; + const id = randomUUID(); + const base = await f.put( + "/documents/" + id, + f.content("BASE\n") + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.client.setSettings({ + maxFileSizeMB: + exclusion === "size" ? 100 / (1024 * 1024) : 1, + ignorePatterns: ["private.md"] + }); + const remote = await f.put( + "/documents/" + id, + f.content("REMOTE\nBASE\n", base.vaultUpdateId) + ); + const local = "BASE\n" + "LOCAL\n".repeat(30); + const path = exclusion === "size" ? "a.md" : "private.md"; + let injected = false, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + const atBoundary = + phase === "installed" + ? label.startsWith("durable:rename:") && + label.endsWith(".output->a.md") + : phase === "staged" + ? label.startsWith("durable:rename:a.md->") + : phase === "prepared" + ? label.startsWith("before:write:") && + label.endsWith(".output") + : label.startsWith( + "before:mkdir:.vault-link-sync/transactions/" + ); + if (injected || !atBoundary) return; + injected = true; + if (exclusion === "ignore") + await f.disk.userRename("a.md", path); + // When installation already completed, deleting REMOTE + // is a subsequent local edit, which must not be undone. + await f.disk.userWrite(path, bytes(local)); + await f.client.syncLocallyUpdatedFile({ + relativePath: path, + ...(exclusion === "ignore" + ? { oldPath: "a.md" } + : {}) + }); + armed = true; + }; + f.store.boundary = (label) => { + if (!armed || crashed || label !== recovery) return; + crashed = true; + f.disk.crash(true); + throw new Error("interrupted exclusion save"); + }; + if (recovery.endsWith(":save")) { + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /interrupted exclusion save/ + ); + assert(crashed); + } else await f.client.setSetting("isSyncEnabled", true); + assert(injected); + f.disk.boundary = f.store.boundary = () => {}; + if (recovery !== "none") { + await f.reopenClient(); + await f.client.start(); + } + + assert.deepEqual( + (await f.disk.readSnapshot(path))?.content, + bytes(local) + ); + const stored = f.store.snapshot().database!; + assert.equal( + stored.documents![id].base?.vaultUpdateId, + installed ? remote.vaultUpdateId : base.vaultUpdateId + ); + assert.equal( + stored.remoteHeads![id].vaultUpdateId, + remote.vaultUpdateId + ); + assert.equal( + (await f.get("/documents/" + id)).contentBase64, + Buffer.from("REMOTE\nBASE\n").toString("base64") + ); + await f.client.setSettings({ + maxFileSizeMB: 1, + ignorePatterns: [] + }); + const expected = installed ? local : "REMOTE\n" + local; + const verify = async () => { + assert.deepEqual( + (await f.get("/file-manifest")).entries, + { [id]: path } + ); + assert.deepEqual( + (await f.disk.readSnapshot(path))?.content, + bytes(expected) + ); + assert.equal( + (await f.get("/documents/" + id)).contentBase64, + Buffer.from(expected).toString("base64") + ); + }; + await verify(); + await f.client.reset(); + await verify(); + } finally { + await f.dispose(); + } + }); + +for (const at of [undefined, "before:save", "durable:save"]) + for (const powerLoss of at ? [false, true] : [false]) + test(`repeated saves retain only the latest local edit through ${at ?? "completion"} and powerLoss=${powerLoss}`, async () => { + const f = await fixture(); + try { + const id = randomUUID(); + const base = await f.put("/documents/" + id, f.content("base")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put( + "/documents/" + id, + f.content("base REMOTE", base.vaultUpdateId) + ); + let saves = 0, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + if ( + saves === 0 && + label.startsWith("before:rename:a.md->") + ) { + saves++; + await f.disk.userWrite("a.md", bytes("base ONE")); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.md" + }); + } else if ( + saves === 1 && + label.startsWith("before:write:") && + label.endsWith(".output") + ) { + saves++; + await f.disk.userWrite("a.md", bytes("base TWO")); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.md" + }); + armed = true; + } + }; + f.store.boundary = (label) => { + if (!armed || crashed || label !== at) return; + crashed = true; + f.disk.crash(powerLoss); + throw new Error("injected repeated-save crash"); + }; + if (at) { + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /injected repeated-save crash/ + ); + assert(crashed); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } else await f.client.setSetting("isSyncEnabled", true); + assert.equal(saves, 2); + const verify = async () => { + assert.deepEqual((await f.get("/file-manifest")).entries, { + [id]: "a.md" + }); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base TWO REMOTE") + ); + assert.equal( + Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(), + "base TWO REMOTE" + ); + }; + await verify(); + await f.client.reset(); + await verify(); + } finally { + await f.dispose(); + } + }); + +for (const timing of ["before", "durable"]) + for (const crash of [false, true]) + test(`oversized save ${timing} staging stays local across crash=${crash}`, async () => { + const f = await fixture(); + try { + const id = randomUUID(), + large = "PRIVATE".repeat(100); + const base = await f.put( + "/documents/" + id, + f.content("PUBLIC") + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.bin" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.client.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); + await f.put( + "/documents/" + id, + f.content("REMOTE", base.vaultUpdateId) + ); + let injected = false, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + `${timing}:rename:a.bin->.vault-link-sync/transactions/` + ) + ) + return; + injected = true; + await f.disk.userWrite("a.bin", bytes(large)); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.bin" + }); + armed = true; + }; + f.store.boundary = (label) => { + if (!crash || !armed || crashed || label !== "durable:save") + return; + crashed = true; + f.disk.crash(true); + throw new Error("injected oversized-save crash"); + }; + if (crash) { + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /injected oversized-save crash/ + ); + assert(crashed); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } else await f.client.setSetting("isSyncEnabled", true); + assert(injected); + const verify = async () => { + assert.deepEqual( + (await f.disk.readSnapshot("a.bin"))?.content, + bytes(large) + ); + assert.deepEqual((await f.get("/file-manifest")).entries, { + [id]: "a.bin" + }); + assert.equal( + Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(), + "REMOTE" + ); + }; + await verify(); + await f.client.reset(); + await verify(); + } finally { + await f.dispose(); + } + }); + +test("a tracked file moved into an ignored path during remote application never uploads its new bytes", async () => { + const f = await fixture(); + try { + const a = randomUUID(), + b = randomUUID(); + await f.put("/documents/" + a, f.content("PUBLIC")); + await f.put("/documents/" + b, f.content("OTHER")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [a]: "a.md", [b]: "b.md" } + }); + await f.client.setSetting("ignorePatterns", ["private.md"]); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [a]: "a.md", [b]: "remote-b.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userRename("a.md", "private.md"); + await f.disk.userWrite("private.md", bytes("SECRET")); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "private.md" + }); + }; + await f.client.setSetting("isSyncEnabled", true); + assert(injected); + const manifest = await f.get("/file-manifest"); + const contents = await Promise.all( + Object.keys(manifest.entries).map((id) => f.get("/documents/" + id)) + ); + assert( + !contents.some( + (c) => + Buffer.from(c.contentBase64, "base64").toString() === + "SECRET" + ) + ); + assert.deepEqual( + (await f.disk.readSnapshot("private.md"))?.content, + bytes("SECRET") + ); + assert.equal(manifest.entries[a], "a.md"); + } finally { + await f.dispose(); + } +}); +test("remote creation plus unrelated notification must not delete remote membership on retry", async () => { + const f = await fixture(); + try { + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + const id = randomUUID(); + await f.put("/documents/" + id, f.content("REMOTE")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "remote.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userWrite("local.md", bytes("LOCAL")); + await f.client.syncLocallyCreatedFile("local.md"); + }; + await f.client.setSetting("isSyncEnabled", true); + assert(injected); + f.disk.boundary = () => {}; + await f.client.reset(); + const manifest = await f.get("/file-manifest"); + + assert.equal(manifest.entries[id], "remote.md"); + } finally { + await f.dispose(); + } +}); +test("notified ignored occupant during remote rename must never upload its bytes", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + await f.put("/documents/" + id, f.content("REMOTE")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.setSetting("ignorePatterns", ["private"]); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "private/note.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith("before:mkdir:.vault-link-sync/transactions/") + ) + return; + injected = true; + await f.disk.userWrite("private", bytes("SECRET")); + await f.client.syncLocallyCreatedFile("private"); + }; + await f.client.setSetting("isSyncEnabled", true); + f.disk.boundary = () => {}; + const manifest = await f.get("/file-manifest"); + const contents = await Promise.all( + Object.keys(manifest.entries).map((id) => f.get("/documents/" + id)) + ); + + assert(injected); + assert.deepEqual( + (await f.disk.readSnapshot("private"))?.content, + bytes("SECRET") + ); + assert( + !contents.some( + (c) => + Buffer.from(c.contentBase64, "base64").toString() === + "SECRET" + ) + ); + } finally { + await f.dispose(); + } +}); +test("notified delete/recreate before staging retains replacement file", async () => { + const f = await fixture(); + try { + const id = randomUUID(); + const h = await f.put("/documents/" + id, f.content("ORIGINAL")); + const m = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put( + "/documents/" + id, + f.content("ORIGINAL REMOTE", h.vaultUpdateId) + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: m.fileManifestId, + entries: { [id]: "remote.md" } + }); + let injected = false; + f.disk.boundary = async (label) => { + if ( + injected || + !label.startsWith( + "before:rename:a.md->.vault-link-sync/transactions/" + ) + ) + return; + injected = true; + await f.disk.userDelete("a.md"); + await f.client.syncLocallyDeletedFile("a.md"); + await f.disk.userWrite("a.md", bytes("REPLACEMENT")); + await f.client.syncLocallyCreatedFile("a.md"); + }; + await f.client.setSetting("isSyncEnabled", true); + f.disk.boundary = () => {}; + const manifest = await f.get("/file-manifest"); + + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("REPLACEMENT") + ); + } finally { + await f.dispose(); + } +}); + +for (const mode of ["delete", "move", "save"] as const) + for (const powerLoss of [false, true]) + for (const at of ["before:save", "durable:save"]) + test(`${mode} during staging survives ${at} and ${powerLoss ? "power loss" : "process replacement"}`, async () => { + const f = await fixture(); + try { + const id = randomUUID(); + const h = await f.put( + "/documents/" + id, + f.content("ORIGINAL") + ); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + await f.put( + "/documents/" + id, + f.content("ORIGINAL REMOTE", h.vaultUpdateId) + ); + let injected = false, + armed = false, + crashed = false; + f.disk.boundary = async (label) => { + const point = mode === "save" ? "durable" : "before"; + if ( + injected || + !label.startsWith( + `${point}:rename:a.md->.vault-link-sync/transactions/` + ) + ) + return; + injected = true; + if (mode === "save") { + await f.disk.userWrite( + "a.md", + bytes("ORIGINAL LOCAL") + ); + await f.client.syncLocallyUpdatedFile({ + relativePath: "a.md" + }); + } else { + if (mode === "delete") { + await f.disk.userDelete("a.md"); + await f.client.syncLocallyDeletedFile("a.md"); + } else { + await f.disk.userRename("a.md", "mine.md"); + await f.client.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "mine.md" + }); + } + await f.disk.userWrite( + "a.md", + bytes("REPLACEMENT") + ); + await f.client.syncLocallyCreatedFile("a.md"); + } + armed = true; + }; + f.store.boundary = (label) => { + if (!armed || crashed || label !== at) return; + crashed = true; + f.disk.crash(powerLoss); + throw new Error("injected journal crash"); + }; + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /injected journal crash/ + ); + assert(injected && crashed); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + const manifest = await f.get("/file-manifest"); + if (mode === "save") { + assert.deepEqual(manifest.entries, { [id]: "a.md" }); + const actual = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(actual, "ORIGINAL LOCAL REMOTE"); + } else { + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("REPLACEMENT") + ); + assert.notEqual( + Object.keys(manifest.entries).find( + (key) => manifest.entries[key] === "a.md" + ), + id + ); + if (mode === "move") { + assert.equal(manifest.entries[id], "mine.md"); + assert.deepEqual( + (await f.disk.readSnapshot("mine.md"))?.content, + bytes("ORIGINAL REMOTE") + ); + } else assert.equal(manifest.entries[id], undefined); + } + await f.client.reset(); + assert.deepEqual( + (await f.get("/file-manifest")).entries, + manifest.entries + ); + } finally { + await f.dispose(); + } + }); diff --git a/frontend/test-support/listener-crash-child.ts b/frontend/test-support/listener-crash-child.ts new file mode 100644 index 00000000..bd703090 --- /dev/null +++ b/frontend/test-support/listener-crash-child.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { fixture } from "./sync-fixture"; +(async () => { + const f = await fixture( + {}, + { + events: async () => ({ headEventId: 0, events: [] }) + } + ); + let delivered = 0; + f.syncer.onRemainingOperationsCountChanged.add(() => { + throw new Error("observer failure"); + }); + f.syncer.onRemainingOperationsCountChanged.add(async () => { + throw new Error("async observer failure"); + }); + f.syncer.onRemainingOperationsCountChanged.add(() => { + delivered++; + }); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + const before = delivered; + f.syncer.wake(); + await f.syncer.waitUntilFinished(); + assert(delivered > before, "Background sync must remain usable"); + } finally { + await f.syncer.stop(); + } +})(); diff --git a/frontend/test-support/native-fs.c b/frontend/test-support/native-fs.c new file mode 100644 index 00000000..560c79c6 --- /dev/null +++ b/frontend/test-support/native-fs.c @@ -0,0 +1,27 @@ +/* Test-only POSIX primitives missing from Node's filesystem API. A preflight + * exists() followed by fs.rename() is not an exclusive rename. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#if defined(__linux__) +#include +#endif + +int main(int argc, char **argv) { + if (argc == 4 && strcmp(argv[1], "rename") == 0) { + int result; +#if defined(__APPLE__) + result = renamex_np(argv[2], argv[3], RENAME_EXCL); +#elif defined(__linux__) + result = syscall(SYS_renameat2, AT_FDCWD, argv[2], AT_FDCWD, argv[3], 1 /* RENAME_NOREPLACE */); +#else +#error "The real filesystem tests require macOS or Linux" +#endif + if (result != 0) { perror("exclusive rename"); return 1; } + return 0; + } + fprintf(stderr, "usage: native-fs rename FROM TO\n"); + return 2; +} diff --git a/frontend/test-support/network.ts b/frontend/test-support/network.ts new file mode 100644 index 00000000..7df61083 --- /dev/null +++ b/frontend/test-support/network.ts @@ -0,0 +1,243 @@ +import assert from "node:assert/strict"; + +export type RequestKind = "create" | "content" | "manifest"; +export interface RequestRecord { + kind: RequestKind; + method: string; + url: string; + body: Record; + status?: number; +} + +interface InjectedFault { + kind: RequestKind; + point: "before" | "after"; + fired: boolean; + interrupted?: RequestRecord; + retried: boolean; + reached: Promise; + resolve: () => void; +} + +export function classifyRequest( + input: RequestInfo | URL, + init?: RequestInit +): RequestRecord | undefined { + const url = new URL(input instanceof Request ? input.url : String(input)); + const method = ( + init?.method ?? (input instanceof Request ? input.method : "GET") + ).toUpperCase(); + if (method !== "PUT") return undefined; + // The engine passes JSON strings. Fail loudly if that contract changes. + if (!/\/(documents\/[^/]+|file-manifest)\/?$/.test(url.pathname)) + return undefined; + assert.equal( + typeof init?.body, + "string", + "CAS request body must be observable JSON" + ); + const body = JSON.parse(init!.body as string) as Record; + const kind = /\/file-manifest\/?$/.test(url.pathname) + ? "manifest" + : body.parentVersionId === null + ? "create" + : "content"; + assert.equal( + typeof body.requestId, + "string", + "CAS request missing requestId" + ); + return { kind, method, url: url.href, body }; +} + +export class NetworkFaults { + public readonly requests: RequestRecord[] = []; + private observation?: { + released: boolean; + fired: boolean; + held: Promise; + release: () => void; + reached: Promise; + reach: () => void; + }; + public pauseObservation(): void { + assert( + !this.observation || this.observation.released, + "Observation already paused" + ); + let release!: () => void, reach!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const reached = new Promise((resolve) => { + reach = resolve; + }); + this.observation = { + released: false, + fired: false, + held, + release, + reached, + reach + }; + } + public async waitForObservation(): Promise { + assert(this.observation, "No observation checkpoint armed"); + await this.observation.reached; + } + public resumeObservation(): void { + if (this.observation) { + this.observation.released = true; + this.observation.release(); + } + } + private async observationCheckpoint( + signal?: AbortSignal | null + ): Promise { + const gate = this.observation; + if (!gate || gate.released) return; + gate.fired = true; + gate.reach(); + signal?.throwIfAborted(); + let abort = () => {}; + try { + await Promise.race([ + gate.held, + new Promise((_, reject) => { + abort = () => + reject( + signal?.reason ?? new Error("Observation aborted") + ); + signal?.addEventListener("abort", abort, { once: true }); + }) + ]); + } finally { + signal?.removeEventListener("abort", abort); + } + } + private armed?: InjectedFault; + private readonly faults: InjectedFault[] = []; + public isExpectedFailure(message: string): boolean { + return this.faults.some( + (fault) => + fault.fired && + message.includes( + `Injected ${fault.point}-commit ${fault.kind} network failure` + ) + ); + } + public arm(kind: RequestKind, point: "before" | "after" = "after"): void { + assert( + !this.armed || (this.armed.fired && this.armed.retried), + "Previous network fault was not exercised and retried" + ); + let resolve!: () => void; + const reached = new Promise((r) => { + resolve = r; + }); + this.armed = { + kind, + point, + fired: false, + retried: false, + reached, + resolve + }; + this.faults.push(this.armed); + } + public async wait(): Promise { + assert(this.armed, "No network fault armed"); + await this.armed.reached; + } + public assertConsumed(): void { + assert( + !this.observation || + (this.observation.fired && this.observation.released), + "Observation checkpoint was not exercised and released" + ); + for (const fault of this.faults) { + assert(fault.fired, "Armed network fault never fired"); + assert(fault.retried, "Interrupted CAS request was never retried"); + } + } + public wrap(fetch: typeof globalThis.fetch): typeof globalThis.fetch { + return async (input, init) => { + const url = new URL( + input instanceof Request ? input.url : String(input) + ); + const observing = + ( + init?.method ?? + (input instanceof Request ? input.method : "GET") + ).toUpperCase() === "GET" && !url.pathname.endsWith("/ping"); + const signal = + init?.signal ?? + (input instanceof Request ? input.signal : undefined); + if (observing) await this.observationCheckpoint(signal); + const record = classifyRequest(input, init); + if (record) { + // A client serializes its persisted CAS requests. Its next CAS + // after a lost response must replay the interrupted operation, + // even if it has since observed newer remote state. + const interrupted = this.armed?.interrupted; + if (interrupted && !this.armed!.retried) { + assert.equal( + record.url, + interrupted.url, + "Retry changed target" + ); + assert.deepEqual( + record.body, + interrupted.body, + "Retry changed payload/request identity" + ); + this.armed!.retried = true; + } + const previous = this.requests.find( + (r) => r.body.requestId === record.body.requestId + ); + if (previous) { + assert.equal( + record.url, + previous.url, + "Retry changed target" + ); + assert.deepEqual( + record.body, + previous.body, + "Retry changed payload/request identity" + ); + } + this.requests.push(record); + } + const fault = this.armed; + const matches = + record && fault && !fault.fired && fault.kind === record.kind; + const fail = () => { + fault!.interrupted = structuredClone(record!); + fault!.fired = true; + fault!.resolve(); + throw new TypeError( + `Injected ${fault!.point}-commit ${fault!.kind} network failure` + ); + }; + if (matches && fault.point === "before") fail(); + const response = await fetch(input, init); + if (observing) await this.observationCheckpoint(signal); + if (record) record.status = response.status; + // Only drop accepted replies, never confuse a rejected CAS with commit. + if ( + matches && + !fault.fired && + fault.point === "after" && + response.ok && + ((await response.clone().json()) as { type?: string }).type === + "Accepted" + ) { + await response.body?.cancel(); + fail(); + } + return response; + }; + } +} diff --git a/frontend/test-support/observer.test.ts b/frontend/test-support/observer.test.ts new file mode 100644 index 00000000..8fb2d5e6 --- /dev/null +++ b/frontend/test-support/observer.test.ts @@ -0,0 +1,15 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +test("a status observer exception must not terminate a background sync process", () => { + const result = spawnSync( + process.execPath, + [ + resolve(__dirname, "../node_modules/tsx/dist/cli.mjs"), + resolve(__dirname, "listener-crash-child.ts") + ], + { encoding: "utf8", timeout: 10000 } + ); + assert.equal(result.status, 0, result.stderr); +}); diff --git a/frontend/test-support/oracle-regressions.test.ts b/frontend/test-support/oracle-regressions.test.ts new file mode 100644 index 00000000..fc2916a4 --- /dev/null +++ b/frontend/test-support/oracle-regressions.test.ts @@ -0,0 +1,351 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { test } from "node:test"; +import type { StoredDatabase } from "sync-client"; +import { assertCanonical } from "./canonical"; +import { assertManifest } from "./oracles"; +import { NetworkFaults } from "./network"; +import { MemoryDisk } from "./storage"; +import { DeterministicAgent } from "../deterministic-tests/src/deterministic-agent"; +import { AssertableState } from "../deterministic-tests/src/utils/assertable-state"; +import type { ExpectedDocument } from "../deterministic-tests/src/test-definition"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const id = "00000000-0000-4000-8000-000000000001"; + +test("canonical checks reject repeated clients before reading snapshots", async () => { + const client = { files: () => new Map(), database: () => ({}) }; + await assert.rejects( + assertCanonical([client, client], "http://unused", "unused"), + /itself/ + ); +}); + +test("canonical checks independently verify both persisted content hashes", async (context) => { + const content = bytes("\ufeffexact bytes\r\n"); + const hash = createHash("sha256").update(content).digest("hex"); + const document = { + documentId: id, + vaultUpdateId: 1, + contentSize: content.length, + userId: "user", + deviceId: "device", + updatedDate: "2026-09-20T00:00:00Z" + }; + const manifest = { fileManifestId: 2, entries: { [id]: "a.md" } }; + const state: Partial = { + initialized: true, + local: manifest.entries, + fileManifest: manifest, + lastSeenUpdateId: 2, + documents: { + [id]: { + materialized: true, + observedHash: hash, + base: { ...document, hash } + } + } + }; + context.mock.method( + globalThis, + "fetch", + async (url: string) => + new Response( + JSON.stringify( + url.endsWith("/vault-snapshot") + ? { + headEventId: 2, + fileManifest: manifest, + documents: [document] + } + : url.includes("/events-since") + ? { + headEventId: 2, + events: [ + { eventId: 1, requestId: "create" }, + { eventId: 2, requestId: "manifest" } + ] + } + : { + contentBase64: + Buffer.from(content).toString("base64") + } + ) + ) + ); + const client = (database = state) => ({ + files: () => new Map([["a.md", content.slice()]]), + database: () => structuredClone(database) + }); + await assertCanonical([client(), client()], "http://test", "token"); + for (const field of ["base", "observed"] as const) { + const corrupt = structuredClone(state); + if (field === "base") corrupt.documents![id].base!.hash = "wrong"; + else corrupt.documents![id].observedHash = "wrong"; + await assert.rejects( + assertCanonical( + [client(), client(corrupt)], + "http://test", + "token" + ), + new RegExp(`Wrong ${field} hash`) + ); + } +}); + +test("lost responses require an identical retry, including request identity and target", async () => { + for (const point of ["before", "after"] as const) { + const faults = new NetworkFaults(); + let sent = 0; + const send = faults.wrap(async () => { + sent++; + return new Response('{"type":"Accepted"}'); + }); + const url = "http://test/documents/a"; + const body = { + requestId: "original", + parentVersionId: 3, + content: { type: "Snapshot", value: "YQ==" } + }; + const init = { method: "PUT", body: JSON.stringify(body) }; + faults.arm("content", point); + await assert.rejects(send(url, init), /Injected/); + assert.throws(() => faults.assertConsumed(), /never retried/); + assert.throws( + () => faults.arm("manifest"), + /not exercised and retried/ + ); + const before = sent; + await assert.rejects( + send(url, { + ...init, + body: JSON.stringify({ ...body, requestId: "regenerated" }) + }), + /Retry changed/ + ); + await assert.rejects( + send(`${url}-different`, init), + /Retry changed target/ + ); + await assert.rejects( + send(url, { + ...init, + body: JSON.stringify({ ...body, parentVersionId: 4 }) + }), + /Retry changed/ + ); + assert.equal(sent, before, "Invalid retries must not reach the server"); + assert.throws(() => faults.assertConsumed(), /never retried/); + await send(url, init); + faults.assertConsumed(); + } +}); + +test("an update fault cannot be consumed by creating a new UUID", async () => { + const faults = new NetworkFaults(); + const send = faults.wrap(async () => new Response('{"type":"Accepted"}')); + faults.arm("content"); + await send("http://test/documents/new", { + method: "PUT", + body: '{"requestId":"new","parentVersionId":null}' + }); + assert.throws(() => faults.assertConsumed(), /never fired/); + const update = { + method: "PUT", + body: '{"requestId":"update","parentVersionId":1}' + }; + await assert.rejects( + send("http://test/documents/existing", update), + /Injected/ + ); + await send("http://test/documents/existing", update); + faults.assertConsumed(); +}); + +test("unlink is idempotent even after an uncertain visible deletion", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a", bytes("A")); + disk.boundary = (label) => { + if (label === "visible:unlink:a") throw new Error("uncertain unlink"); + }; + await assert.rejects(disk.deleteFile("a"), /uncertain/); + disk.boundary = () => {}; + await disk.deleteFile("a"); + disk.crash(true); + assert.equal(await disk.exists("a"), false); + await disk.deleteFile("a"); + await disk.createDirectory("directory"); + await assert.rejects(disk.deleteFile("directory"), /directory/); +}); + +test("text assertions preserve BOM, CRLF, empty content and missing final newline", async () => { + const agent = new DeterministicAgent(0, {}, () => {}); + for (const content of [ + "", + "\ufeffhello", + "\ufeffa\r\nb\r\n", + "last line" + ]) { + await agent.disk.userWrite("a.md", bytes(content)); + assert.equal(await agent.getFileContent("a.md"), content); + } +}); + +test("durably pruning a directory also removes its already-unlinked durable children", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("transaction/source", bytes("staged bytes")); + disk.boundary = (label) => { + if (label === "visible:unlink:transaction/source") { + disk.crash(false); + throw new Error("process died after unlink"); + } + }; + await assert.rejects(disk.deleteFile("transaction/source"), /process died/); + disk.boundary = () => {}; + await disk.delete("transaction"); + disk.crash(true); + assert.deepEqual(disk.userFiles(), new Map()); + assert.equal(await disk.exists("transaction"), false); +}); + +test("portable components are limited by UTF-8 bytes, not characters or total path length", () => { + for (const component of ["a".repeat(255), "é".repeat(127) + "a"]) + assertManifest({ [id]: `${component}/${component}/${component}` }); + for (const component of ["a".repeat(256), "é".repeat(128)]) + assert.throws( + () => assertManifest({ [id]: `nested/${component}` }), + /Oversized component/ + ); +}); + +function state(documents: { path: string; content: string; id: string }[]) { + return new AssertableState({ + files: new Map(documents.map((doc) => [doc.path, doc.content])), + clientFiles: [], + manifests: [], + canonical: Object.fromEntries( + documents.map((doc) => [doc.id, doc.path]) + ), + bytes: new Map(documents.map((doc) => [doc.path, bytes(doc.content)])) + }); +} + +test("complete scenario assertions reject collapsed creates, forgotten edits, reverted moves and recycled UUIDs", () => { + const identities = new Map(); + const expected: ExpectedDocument[] = [ + { key: "a", path: "a.md", content: "A\nimportant edit" }, + { key: "b", path: "b.md", content: "B" } + ]; + const original = [ + { id: "id-a", path: "a.md", content: "A\nimportant edit" }, + { id: "id-b", path: "b.md", content: "B" } + ]; + state(original).assertDocuments(expected, identities); + assert.throws( + () => + state([ + { ...original[0], content: "A\nimportant edit\nB" } + ]).assertDocuments(expected, identities), + /Expected 2 file/ + ); + assert.throws( + () => + state([ + { ...original[0], content: "A" }, + original[1] + ]).assertDocuments(expected, identities), + /Byte mismatch/ + ); + const moved = [ + { ...expected[0], path: "b.md" }, + { ...expected[1], path: "a.md" } + ]; + assert.throws( + () => state(original).assertDocuments(moved, identities), + /Byte mismatch/ + ); + assert.throws( + () => + state([ + { ...original[0], id: "id-b" }, + { ...original[1], id: "id-a" } + ]).assertDocuments(expected, identities), + /identity changed/ + ); + state([original[1]]).assertDocuments([expected[1]], identities); + assert.throws( + () => + state(original).assertDocuments( + [{ ...expected[0], key: "recreated" }, expected[1]], + identities + ), + /reused.*deleted UUID/ + ); + assert.equal(identities.has("recreated"), false); + const conflict = [ + { key: "one", path: "same.md", content: "one" }, + { key: "two", path: "same.md", conflict: true, content: "two" } + ]; + state([ + { id: "one", path: "same.md", content: "one" }, + { id: "two", path: "same (conflict two).md", content: "two" } + ]).assertDocuments(conflict, new Map()); +}); + +test("canonical checks follow pages without weakening gap or duplicate-request detection", async (context) => { + const manifest = { fileManifestId: 3, entries: {} }; + const client = () => ({ + files: () => new Map(), + database: () => ({ + initialized: true, + local: {}, + documents: {}, + fileManifest: manifest, + lastSeenUpdateId: 3 + }) + }); + let fault = "none"; + context.mock.method(globalThis, "fetch", async (input: string) => { + if (input.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 3, + fileManifest: manifest, + documents: [] + }); + const after = Number(new URL(input).searchParams.get("after")); + if (after === 0) + return Response.json({ + headEventId: 3, + endEventId: 1, + events: [{ eventId: 1, requestId: "first" }] + }); + assert.equal(after, 1); + return Response.json({ + headEventId: 3, + endEventId: fault === "empty" ? 1 : 3, + events: + fault === "empty" + ? [] + : [ + ...(fault === "gap" + ? [] + : [ + { + eventId: 2, + requestId: + fault === "duplicate" + ? "first" + : "second" + } + ]), + { eventId: 3, requestId: "third" } + ] + }); + }); + await assertCanonical([client(), client()], "http://test", "token"); + for (fault of ["gap", "duplicate", "empty"]) + await assert.rejects( + assertCanonical([client(), client()], "http://test", "token"), + /contiguous|multiple events|progress/ + ); +}); diff --git a/frontend/test-support/oracles.ts b/frontend/test-support/oracles.ts new file mode 100644 index 00000000..f50141ef --- /dev/null +++ b/frontend/test-support/oracles.ts @@ -0,0 +1,173 @@ +import assert from "node:assert/strict"; +import type { StoredDatabase } from "sync-client"; + +export function assertSameFiles( + clients: readonly Map[] +): void { + assert( + clients.length >= 2, + "Consistency requires at least two distinct clients" + ); + assert.equal( + new Set(clients).size, + clients.length, + "Cannot compare a client to itself" + ); + const sorted = (files: Map) => + [...files].sort(([a], [b]) => a.localeCompare(b)); + for (let i = 1; i < clients.length; i++) + assert.deepEqual( + sorted(clients[i]), + sorted(clients[0]), + `Client ${i} differs from client 0 (paths or bytes)` + ); +} + +export function assertManifest(entries: Record): void { + const paths = Object.values(entries); + const folded = paths.map((p) => + p.normalize("NFC").toUpperCase().normalize("NFC") + ); + const directorySpellings = new Map(); + for (const path of paths) { + assert.equal( + path, + path.normalize("NFC"), + `Noncanonical Unicode path: ${path}` + ); + const parts = path.split("/"); + parts.forEach((part, i) => { + assert( + Buffer.byteLength(part, "utf8") <= 255, + `Oversized component: ${part}` + ); + assert( + !/[<>:"\\|?*\p{Cc}]|[. ]$/u.test(part), + `Nonportable component: ${part}` + ); + assert( + !/^(CON|PRN|AUX|NUL|CONIN\$|CONOUT\$|COM[1-9¹²³]|LPT[1-9¹²³])$/u.test( + part.split(".")[0].trimEnd().toUpperCase() + ), + `Reserved component: ${part}` + ); + if (i < parts.length - 1) { + const prefix = parts.slice(0, i + 1).join("/"); + const key = prefix.toUpperCase().normalize("NFC"); + const previous = directorySpellings.get(key); + assert( + previous === undefined || previous === prefix, + `Aliased directory spellings: ${prefix} and ${previous}` + ); + directorySpellings.set(key, prefix); + } + }); + } + assert.equal( + new Set(folded).size, + paths.length, + "Manifest contains aliased/duplicate paths" + ); + for (const path of folded) { + assert( + path && !path.startsWith("/") && !path.includes("\\"), + `Invalid path ${path}` + ); + assert( + path.split("/").every((p) => p && p !== "." && p !== ".."), + `Invalid component ${path}` + ); + assert( + path.split("/")[0] !== ".VAULT-LINK-SYNC", + "Internal path in manifest" + ); + assert( + !folded.some( + (other) => other !== path && other.startsWith(`${path}/`) + ), + `File/ancestor conflict: ${path}` + ); + } +} + +/** Independent per-ID policy oracle, intentionally not importing merge code. */ +export function pathDecision( + base: string | undefined, + local: string | undefined, + remote: string | undefined +): string | undefined { + if (local === base) return remote; + if (remote === base) return local; + return remote; +} + +export function assertQuiescent( + state: Partial, + entries: Record, + head: number +): void { + assert.equal( + state.pending, + undefined, + "Pending CAS request at convergence" + ); + assert.equal( + state.eventReplay, + undefined, + "Unfinished event replay at convergence" + ); + assert.equal( + state.application, + undefined, + "Unfinished filesystem journal at convergence" + ); + assert.deepEqual( + state.local, + entries, + "Local UUID/path map differs from server" + ); + assert.deepEqual( + state.fileManifest?.entries, + entries, + "Client manifest differs from server" + ); + assert.equal( + state.lastSeenUpdateId, + head, + "Client event watermark differs from server" + ); + assertManifest(entries); +} + +/** Removal exemptions name exact markers and the explicit destructive action. + * Deletion mode never disables preservation for unrelated documents. */ +export class ContentLedger { + private readonly markers = new Map(); + public add(marker: string): void { + assert(!this.markers.has(marker), "Duplicate marker"); + this.markers.set(marker, undefined); + } + public removedBy(content: string, action: string): void { + assert(action.length > 0); + for (const marker of this.markers.keys()) + if (content.includes(marker)) this.markers.set(marker, action); + } + public assertPreserved(files: Map): void { + const texts = [...files.values()].map((bytes) => + Buffer.from(bytes).toString("utf8") + ); + for (const [marker, deletion] of this.markers) { + const count = texts.reduce( + (n, text) => n + text.split(marker).length - 1, + 0 + ); + assert(count <= 1, `Marker ${marker} duplicated ${count} times`); + if (!deletion) + assert.equal( + count, + 1, + `Marker ${marker} lost without an explicit delete/overwrite` + ); + } + } +} diff --git a/frontend/test-support/polling.test.ts b/frontend/test-support/polling.test.ts new file mode 100644 index 00000000..37627b5e --- /dev/null +++ b/frontend/test-support/polling.test.ts @@ -0,0 +1,317 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import type { EventBatch } from "../sync-client/src/services/types/EventBatch"; +import type { EventRecord } from "../sync-client/src/services/types/EventRecord"; +import type { PutFileContent } from "../sync-client/src/services/types/PutFileContent"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +const documentId = "00000000-0000-4000-8000-000000000001"; +const interval = 1_000; +const retryInterval = 75; +const bytes = (text: string) => new TextEncoder().encode(text); + +/** Real sync loop over in-memory disk/persistence and an observable version + * store. The transport never wakes the engine itself. */ +async function fixture(syncIntervalMs: number | undefined) { + const disk = new MemoryDisk(); + await disk.userWrite("note.bin", bytes("base")); + let headEventId = 2; // Initial content (1), then manifest (2). + const versions = new Map([[1, bytes("base")]]); + const head = (version: number) => ({ + documentId, + vaultUpdateId: version, + contentSize: versions.get(version)!.length, + updatedDate: "2026-09-20T00:00:00Z", + userId: "user", + deviceId: "device" + }); + let latest = head(1); + const events: EventRecord[] = []; + const writes: PutFileContent[] = []; + let eventReads = 0; + let failNextRead = false; + const publishContent = (content: Uint8Array, requestId: string) => { + const eventId = ++headEventId; + versions.set(eventId, new Uint8Array(content)); + latest = head(eventId); + events.push({ eventId, requestId, type: "content", document: latest }); + return latest; + }; + const batch = (after = 2): EventBatch => + structuredClone({ + headEventId, + events: events.filter((event) => event.eventId > after) + }); + const service: Partial = { + events: async (after) => { + eventReads++; + if (failNextRead) { + failNextRead = false; + throw new Error("offline"); + } + return batch(after); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => { + const content = versions.get(vaultUpdateId); + assert(content, `Missing immutable version ${vaultUpdateId}`); + return content.slice(); + }, + putFileContent: async (id, request) => { + assert.equal(id, documentId); + assert.equal(request.parentVersionId, latest.vaultUpdateId); + assert(request.content.type === "Snapshot"); + writes.push(structuredClone(request)); + return { + type: "Accepted", + ...publishContent( + Buffer.from(request.content.value, "base64"), + request.requestId + ) + }; + } + }; + const initial = emptyState("polling-test"); + initial.initialized = true; + initial.lastSeenUpdateId = 2; + initial.local = { [documentId]: "note.bin" }; + initial.fileManifest = { fileManifestId: 2, entries: { ...initial.local } }; + initial.remoteHeads = { [documentId]: latest }; + const hash = (await toStoredSnapshot({ content: bytes("base") })).hash; + initial.documents[documentId] = { + materialized: true, + observedHash: hash, + base: { ...latest, hash } + }; + const persistence = new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + initial.vaultKey, + async () => (await persistence.load()).database as StoredDatabase + ); + const settings = new Settings( + logger, + { syncIntervalMs, networkRetryIntervalMs: retryInterval }, + async () => {} + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: [] }), + reset: () => {} + } as unknown as ServerConfig; + const websocket = { + onWebSocketStatusChanged: new EventListeners< + (connected: boolean) => unknown + >(), + onRemoteVaultUpdateReceived: new EventListeners< + (batch: EventBatch) => Promise + >() + }; + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket as unknown as WebSocketManager, + new FileOperations(disk.session(), database, config), + config, + new SyncHistory(logger), + new FixedSizeDocumentCache(0) + ); + return { + disk, + database, + syncer, + writes, + batch, + websocket, + reads: () => eventReads, + serverContent: () => versions.get(latest.vaultUpdateId), + failNextRead: () => { + failNextRead = true; + }, + changeRemote: () => { + publishContent(bytes("remote edit"), "remote-content"); + const eventId = ++headEventId; + events.push({ + eventId, + requestId: "remote-rename", + type: "fileManifest", + fileManifest: { + fileManifestId: eventId, + entries: { [documentId]: "renamed.bin" } + } + }); + } + }; +} + +for (const syncIntervalMs of [undefined, 0, interval]) { + for (const side of ["local", "remote"] as const) { + test(`${side} changes without notifications ${syncIntervalMs ? "are recovered by explicit polling" : `remain unobserved when polling is ${syncIntervalMs === 0 ? "zero" : "unset"}`}`, async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + const f = await fixture(syncIntervalMs); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + const reads = f.reads(); + if (side === "local") + await f.disk.userWrite("note.bin", bytes("local edit")); + else f.changeRemote(); + const assertUnobserved = () => { + assert.equal( + f.reads(), + reads, + "A hidden wakeup fetched events" + ); + assert.equal(f.database.state.lastSeenUpdateId, 2); + assert.equal(f.writes.length, 0); + assert.deepEqual( + f.serverContent(), + bytes(side === "local" ? "base" : "remote edit") + ); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + [ + "note.bin", + bytes(side === "local" ? "local edit" : "base") + ] + ]) + ); + }; + context.mock.timers.tick( + syncIntervalMs ? interval - 1 : 60_000 + ); + await f.syncer.waitUntilFinished(); + assertUnobserved(); + if (syncIntervalMs) context.mock.timers.tick(1); + else if (side === "local") + await f.syncer.syncLocallyUpdatedFile({ + relativePath: "note.bin" + }); + else + await f.websocket.onRemoteVaultUpdateReceived.triggerAsync( + f.batch() + ); + await f.syncer.waitUntilFinished(); + const content = bytes( + side === "local" ? "local edit" : "remote edit" + ); + const path = side === "local" ? "note.bin" : "renamed.bin"; + assert.deepEqual(f.serverContent(), content); + assert.deepEqual( + f.disk.userFiles(), + new Map([[path, content]]) + ); + assert.deepEqual(f.database.state.local, { + [documentId]: path + }); + assert.equal( + f.database.state.lastSeenUpdateId, + side === "local" ? 3 : 4 + ); + assert.equal(f.syncer.isBusy, false); + assert.equal(f.writes.length, side === "local" ? 1 : 0); + if (syncIntervalMs) { + const beforeNextPoll = f.reads(); + context.mock.timers.tick(interval); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + beforeNextPoll + 1, + "Polling stopped after its first successful run" + ); + assert.equal( + f.writes.length, + side === "local" ? 1 : 0, + "An idle poll resubmitted unchanged content" + ); + } + await f.syncer.stop(); + const stoppedReads = f.reads(); + context.mock.timers.tick(60_000); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + stoppedReads, + "Polling continued after stop" + ); + } finally { + await f.syncer.stop(); + context.mock.timers.reset(); + } + }); + } +} + +test("transient failures retry with polling disabled, then leave the client idle", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout"] }); + const f = await fixture(0); + try { + f.changeRemote(); + f.failNextRead(); + f.syncer.start(); + await assert.rejects(f.syncer.waitUntilFinished(), /offline/); + const reads = f.reads(); + context.mock.timers.tick(retryInterval - 1); + await assert.rejects(f.syncer.waitUntilFinished(), /offline/); + assert.equal(f.reads(), reads); + context.mock.timers.tick(1); + await f.syncer.waitUntilFinished(); + assert.equal(f.reads(), reads + 1); + assert.deepEqual( + f.disk.userFiles(), + new Map([["renamed.bin", bytes("remote edit")]]) + ); + assert.equal(f.database.state.lastSeenUpdateId, 4); + assert.equal(f.syncer.isBusy, false); + context.mock.timers.tick(60_000); + await f.syncer.waitUntilFinished(); + assert.equal( + f.reads(), + reads + 1, + "Network retries became clean-state polling" + ); + } finally { + await f.syncer.stop(); + context.mock.timers.reset(); + } +}); + +test("a failed WebSocket connection catches up over HTTP even when polling is disabled", async () => { + const f = await fixture(0); + try { + f.syncer.start(); + await f.syncer.waitUntilFinished(); + f.changeRemote(); + f.websocket.onWebSocketStatusChanged.trigger(false); + await f.syncer.waitUntilFinished(); + assert.equal(f.database.state.lastSeenUpdateId, 4); + assert.deepEqual( + f.disk.userFiles(), + new Map([["renamed.bin", bytes("remote edit")]]) + ); + } finally { + await f.syncer.stop(); + } +}); diff --git a/frontend/test-support/protocol-audit.test.ts b/frontend/test-support/protocol-audit.test.ts new file mode 100644 index 00000000..2cb7d782 --- /dev/null +++ b/frontend/test-support/protocol-audit.test.ts @@ -0,0 +1,392 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { resolve, join } from "node:path"; +import { tmpdir } from "node:os"; +import { mkdtemp, rm } from "node:fs/promises"; +const root = resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const bytes = (s: string) => new TextEncoder().encode(s); +const headers = { + Authorization: `Bearer ${token}`, + "Device-Id": "audit", + "Content-Type": "application/json" +}; +async function put(url: string, body: unknown) { + const r = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(body) + }); + assert(r.ok, await r.clone().text()); + return r.json(); +} +const content = (s: string) => ({ + requestId: randomUUID(), + parentVersionId: null, + content: { type: "Snapshot", value: Buffer.from(s).toString("base64") } +}); + +test( + "connection admission rejects excess clients and releases disconnected slots", + { timeout: 15000 }, + async () => { + const dir = await mkdtemp(join(tmpdir(), "vault-link-admission-")); + const config = join(dir, "config.yml"); + await writeFile( + config, + ( + await readFile(root + "/sync-server/config-e2e.yml", "utf8") + ).replace("max_clients_per_vault: 256", "max_clients_per_vault: 1") + ); + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + config, + new Logger() + ); + const sockets: WebSocket[] = []; + try { + await server.start(); + const url = server.remoteUri + "/vaults/" + randomUUID(); + await put(url + "/documents/" + randomUUID(), content("event")); + const connect = async () => { + const ws = new WebSocket(url.replace("http:", "ws:") + "/ws"); + sockets.push(ws); + const accepted = await new Promise((resolve) => { + ws.onopen = () => + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId: randomUUID(), + lastSeenVaultUpdateId: 0 + }) + ); + ws.onmessage = () => resolve(true); + ws.onerror = () => resolve(false); + ws.onclose = () => resolve(false); + }); + return { ws, accepted }; + }; + const first = await connect(); + assert(first.accepted); + assert.equal((await connect()).accepted, false); + await new Promise((resolve) => { + first.ws.onclose = () => resolve(); + first.ws.close(); + }); + // A completed close releases the server's owned connection permit. + assert.equal((await connect()).accepted, true); + } finally { + for (const ws of sockets) ws.close(); + await server.stop(); + await rm(dir, { recursive: true, force: true }); + } + } +); + +test( + "an unauthenticated upgraded socket expires without a handshake", + { timeout: 10000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let ws: WebSocket | undefined; + try { + await server.start(); + ws = new WebSocket( + server.remoteUri.replace("http:", "ws:") + + "/vaults/" + + randomUUID() + + "/ws" + ); + const socket = ws; + const closed = await new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), 6500); + socket.onclose = () => { + clearTimeout(timer); + resolve(true); + }; + socket.onerror = () => {}; + }); + assert(closed, "handshake deadline must release idle sockets"); + } finally { + ws?.close(); + await server.stop(); + } + } +); + +test( + "bootstrap notifications adopt the existing remote document identity", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let client: SyncClient | undefined; + try { + await server.start(); + const vault = randomUUID(), + id = randomUUID(); + const url = server.remoteUri + "/vaults/" + vault; + await put(url + "/documents/" + id, content("REMOTE")); + await put(url + "/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0 + } + }); + client = await SyncClient.create({ fs: disk, persistence: store }); + await disk.userWrite("a.md", bytes("LOCAL")); + await client.syncLocallyCreatedFile("a.md"); + await disk.userWrite("b.md", bytes("B")); + await client.syncLocallyCreatedFile("b.md"); + await client.start(); + const files = [...disk.userFiles().keys()]; + assert.equal(files.length, 2); + } finally { + await client?.destroy(); + await server.stop(); + } + } +); + +test( + "local undo survives a lost accepted reply followed by a rejected retry", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + let client: SyncClient | undefined; + class SilentWebSocket { + static OPEN = 1; + readyState = 1; + onopen: ((event: object) => void) | null = null; + onclose: + | ((event: { code: number; reason: string }) => void) + | null = null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: ((event: object) => void) | null = null; + constructor(_url: string | URL) { + queueMicrotask(() => this.onopen?.({})); + } + send(_data: string) {} + close() { + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + try { + await server.start(); + const vault = randomUUID(), + id = randomUUID(); + const url = server.remoteUri + "/vaults/" + vault; + await put(url + "/documents/" + id, content("base")); + await put(url + "/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0, + networkRetryIntervalMs: 0 + } + }); + let mode: "normal" | "lose-reply" | "reject-retry" = "normal"; + let lostId: string | undefined; + client = await SyncClient.create({ + fs: disk, + persistence: store, + webSocket: SilentWebSocket as unknown as typeof WebSocket, + fetch: async (input, init) => { + if (init?.method === "PUT" && mode === "reject-retry") { + assert.equal( + JSON.parse(String(init.body)).requestId, + lostId + ); + mode = "normal"; + return new Response("proxy body limit changed", { + status: 413 + }); + } + const response = await fetch(input, init); + if (init?.method === "PUT" && mode === "lose-reply") { + assert.equal( + (await response.clone().json()).type, + "Accepted" + ); + lostId = JSON.parse(String(init.body)).requestId; + mode = "normal"; + throw new Error("injected lost accepted response"); + } + return response; + } + }); + await client.start(); + mode = "lose-reply"; + await disk.userWrite("a.md", bytes("base A")); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + await assert.rejects(client.waitUntilFinished(), /lost accepted/); + const original = await ( + await fetch(url + "/documents/" + id, { headers }) + ).json(); + await put(url + "/documents/" + id, { + ...content("base A REMOTE"), + parentVersionId: original.vaultUpdateId + }); + mode = "reject-retry"; + await disk.userWrite("a.md", bytes("base")); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + await client.waitUntilFinished(); + const result = Buffer.from( + (await disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(result, "base REMOTE"); + } finally { + await client?.destroy(); + await server.stop(); + } + } +); + +test( + "closing an old connection cannot erase its replacement's cursors", + { timeout: 15000 }, + async () => { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + const sockets: WebSocket[] = []; + try { + await server.start(); + const url = server.remoteUri + "/vaults/" + randomUUID(); + const id = randomUUID(); + await put(url + "/documents/" + id, content("example")); + const connect = async (deviceId: string) => { + const ws = new WebSocket(url.replace("http:", "ws:") + "/ws"); + sockets.push(ws); + await new Promise((resolve, reject) => { + ws.onopen = () => + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId, + lastSeenVaultUpdateId: 0 + }) + ); + ws.onmessage = () => resolve(); + ws.onerror = () => reject(new Error("connection failed")); + }); + return ws; + }; + const observer = await connect("observer"); + const first = await connect("same-device"); + const sendCursor = (ws: WebSocket, start: number) => + ws.send( + JSON.stringify({ + type: "cursorPositions", + documentsWithCursors: [ + { + document_id: id, + relative_path: "note.md", + vault_update_id: 1, + cursors: [{ start, end: start }] + } + ] + }) + ); + const seen = (start: number) => + new Promise((resolve) => { + observer.onmessage = (event) => { + const message = JSON.parse(String(event.data)); + if ( + message.type === "cursorPositions" && + message.clients.some( + (client: { + deviceId: string; + documentsWithCursors: { + cursors: { start: number }[]; + }[]; + }) => + client.deviceId === "same-device" && + client.documentsWithCursors[0]?.cursors[0] + ?.start === start + ) + ) + resolve(); + }; + }); + let observed = seen(1); + sendCursor(first, 1); + await observed; + const replacement = await connect("same-device"); + observed = seen(2); + sendCursor(replacement, 2); + await observed; + await new Promise((resolve) => { + first.onclose = () => resolve(); + first.close(); + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const current = new Promise< + { + deviceId: string; + documentsWithCursors: { cursors: { start: number }[] }[]; + }[] + >((resolve) => { + observer.onmessage = (event) => { + const message = JSON.parse(String(event.data)); + if (message.type === "cursorPositions") + resolve(message.clients); + }; + }); + observer.send( + JSON.stringify({ + type: "cursorPositions", + documentsWithCursors: [] + }) + ); + const clients = await current; + assert.equal( + clients.find((client) => client.deviceId === "same-device") + ?.documentsWithCursors[0]?.cursors[0]?.start, + 2 + ); + } finally { + for (const ws of sockets) ws.close(); + await server.stop(); + } + } +); diff --git a/frontend/test-support/protocol.test.ts b/frontend/test-support/protocol.test.ts new file mode 100644 index 00000000..d0652d79 --- /dev/null +++ b/frontend/test-support/protocol.test.ts @@ -0,0 +1,420 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { Worker } from "node:worker_threads"; +import * as path from "node:path"; +import { Logger } from "sync-client"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { MemoryDisk, type DiskImage, type StoredClient } from "./storage"; +import { getJson, type CanonicalSnapshot } from "./canonical"; +import type { RequestRecord } from "./network"; +import type { CrashWorkerInput } from "../deterministic-tests/src/crash-worker"; + +const root = path.resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const headers = { + Authorization: `Bearer ${token}`, + "Device-Id": "protocol-test", + "Content-Type": "application/json" +}; +const makeServer = () => + new ServerControl( + path.join(root, "sync-server/target/release/sync_server"), + path.join(root, "sync-server/config-e2e.yml"), + new Logger() + ); +async function put(url: string, body: unknown) { + const response = await fetch(url, { + method: "PUT", + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(5_000) + }); + assert( + response.ok, + `HTTP ${response.status}: ${await response.clone().text()}` + ); + return (await response.json()) as { + type: string; + vaultUpdateId: number; + fileManifestId: number; + }; +} +const content = (text: string, parentVersionId: number | null = null) => ({ + requestId: randomUUID(), + parentVersionId, + content: { type: "Snapshot", value: Buffer.from(text).toString("base64") } +}); + +interface WorkerSnapshot { + type: string; + disk: DiskImage; + stored: StoredClient; + requests: RequestRecord[]; + editedBase?: { documentId: string; vaultUpdateId: number }; +} +async function runWorker(input: CrashWorkerInput): Promise { + const worker = new Worker( + path.join(root, "frontend/deterministic-tests/dist/crash-worker.js"), + { workerData: input } + ); + let timeout: ReturnType | undefined; + try { + return await new Promise((resolve, reject) => { + timeout = setTimeout( + () => + reject(new Error("Worker checkpoint/completion timed out")), + 15_000 + ); + worker.once("error", reject); + worker.once("exit", (code) => + reject( + new Error( + `Worker exited (${code}) before expected checkpoint` + ) + ) + ); + worker.on( + "message", + (message: WorkerSnapshot & { error?: string }) => { + if (message.type === "error") + reject(new Error(message.error)); + else { + assert.equal( + message.type, + input.stopAt ? "checkpoint" : "finished" + ); + resolve(message); + } + } + ); + }); + } finally { + clearTimeout(timeout); + // This is the fault injection, not graceful shutdown masquerading as it. + await worker.terminate(); + } +} + +for (const [kind, format, originalPath] of [ + ["create", "Snapshot", "original.md"], + ["content", "Snapshot", "original.bin"], + ["content", "Diff", "original.md"], + ["manifest", undefined, "original.md"] +] as const) { + test( + `abrupt client death after accepted ${kind} ${format ?? ""} reply, newer head, server SIGKILL, durable retry`, + { timeout: 30_000 }, + async () => { + const server = makeServer(); + try { + await server.start(); + const vault = `crash-${randomUUID()}`; + const url = `${server.remoteUri}/vaults/${vault}`; + const disk = new MemoryDisk(); + await disk.userWrite( + originalPath, + new TextEncoder().encode("preserve-original\n") + ); + let input: CrashWorkerInput = { + disk: disk.image(), + stored: { + settings: { + isSyncEnabled: true, + enableTelemetry: false, + remoteUri: server.remoteUri, + vaultName: vault, + token, + requestTimeoutMs: 5_000, + syncIntervalMs: 0 + } + } as StoredClient + }; + let originalId: string | undefined; + let originalParent: number | undefined; + let uploaded = "preserve-original\n"; + if (kind === "content" && format === "Diff") { + // Keep the established base in the worker's real content + // cache; a cold restart intentionally falls back to Snapshot. + uploaded += "local-update\n"; + input.editAfterSync = { + path: originalPath, + content: uploaded + }; + } else if (kind === "content") { + const initialized = await runWorker(input); + originalId = Object.keys( + initialized.stored.database!.local! + )[0]; + originalParent = + initialized.stored.database!.documents![originalId] + .base!.vaultUpdateId; + disk.restore(initialized.disk); + uploaded += "local-update\n"; + await disk.userWrite( + originalPath, + new TextEncoder().encode(uploaded) + ); + input = { disk: disk.image(), stored: initialized.stored }; + } + const checkpoint = await runWorker({ ...input, stopAt: kind }); + if (format === "Diff") { + assert( + checkpoint.editedBase, + "Warm update baseline was not recorded" + ); + originalId = checkpoint.editedBase.documentId; + originalParent = checkpoint.editedBase.vaultUpdateId; + } + const pending = checkpoint.stored.database!.pending!; + assert( + pending, + "The uncertain request was not persisted before sending" + ); + const interrupted = checkpoint.requests.at(-1)!; + assert.equal( + interrupted.kind, + kind, + "Wrong CAS was interrupted" + ); + if (kind !== "manifest") { + assert.equal(pending.type, "content"); + assert(pending.type === "content"); + assert.equal(pending.request.content.type, format); + assert.equal( + pending.request.parentVersionId, + kind === "create" ? null : originalParent + ); + if (originalId) + assert.equal( + pending.documentId, + originalId, + "Update created a replacement UUID" + ); + } + const accepted = await getJson( + `${url}/vault-snapshot`, + token + ); + // Unpublished content has no manifest membership yet, so it is + // deliberately absent from /vault-snapshot's document list. + const doc = + pending.type === "content" + ? await getJson<{ + documentId: string; + vaultUpdateId: number; + }>(`${url}/documents/${pending.documentId}`, token) + : accepted.documents[0]; + assert(doc); + const remote = await put( + `${url}/documents/${doc.documentId}`, + content(`${uploaded}remote-update\n`, doc.vaultUpdateId) + ); + assert.equal(remote.type, "Accepted"); + const remotePath = originalPath.replace("original", "remote"); + const renamed = await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: accepted.fileManifest.fileManifestId, + entries: { [doc.documentId]: remotePath } + }); + assert.equal(renamed.type, "Accepted"); + const before = await getJson( + `${url}/vault-snapshot`, + token + ); + await server.crash(); + await server.restart(); + assert.deepEqual( + await getJson(`${url}/vault-snapshot`, token), + before, + "Server restart lost acknowledged state" + ); + const restarted = await runWorker({ + disk: checkpoint.disk, + stored: checkpoint.stored + }); + const replay = restarted.requests.find( + (r) => r.body.requestId === pending.request.requestId + ); + assert(replay, "Restart never retried the uncertain request"); + assert.equal( + replay.url, + interrupted.url, + "Restart changed the request target" + ); + assert.deepEqual( + replay.body, + pending.request, + "Restart changed the persisted request" + ); + assert.equal(restarted.stored.database!.pending, undefined); + assert.equal(restarted.stored.database!.application, undefined); + const recovered = new MemoryDisk(); + recovered.restore(restarted.disk); + assert.deepEqual( + recovered.userFiles(), + new Map([ + [ + remotePath, + new TextEncoder().encode( + `${uploaded}remote-update\n` + ) + ] + ]) + ); + assert.deepEqual(restarted.stored.database!.local, { + [doc.documentId]: remotePath + }); + const events = await getJson<{ + events: { requestId: string }[]; + }>(`${url}/events-since?after=0`, token); + assert.equal( + events.events.filter( + (e) => e.requestId === pending.request.requestId + ).length, + 1, + "Retry emitted a duplicate event" + ); + } finally { + await server.stop(); + } + } + ); +} + +test( + "concurrent CAS commits have identical HTTP/event-log/WebSocket total order and reconnect catchup", + { timeout: 20_000 }, + async () => { + const server = makeServer(); + let ws: WebSocket | undefined; + try { + await server.start(); + const url = `${server.remoteUri}/vaults/order-${randomUUID()}`; + const received: number[] = []; + const connect = async (after: number) => { + ws = new WebSocket(`${url.replace(/^http/, "ws")}/ws`); + ws.addEventListener("message", (message) => { + const batch = JSON.parse(String(message.data)) as { + type: string; + events: { eventId: number }[]; + }; + if (batch.type === "vaultEvents") + received.push( + ...batch.events.map((event) => event.eventId) + ); + }); + await new Promise((resolve, reject) => { + ws!.onopen = () => resolve(); + ws!.onerror = () => reject(new Error("WebSocket failed")); + }); + ws.send( + JSON.stringify({ + type: "handshake", + token, + deviceId: "order-observer", + lastSeenVaultUpdateId: after + }) + ); + }; + await connect(0); + const ids = Array.from({ length: 12 }, () => randomUUID()); + const requests = ids.map((id, i) => ({ + id, + body: content(`bytes-${i}`) + })); + const replies = await Promise.all( + requests.map(({ id, body }) => + put(`${url}/documents/${id}`, body) + ) + ); + assert.deepEqual( + replies.map((r) => r.vaultUpdateId).sort((a, b) => a - b), + ids.map((_, i) => i + 1) + ); + const emptyManifest = await getJson<{ + entries: Record; + }>(`${url}/file-manifest`, token); + assert.deepEqual( + emptyManifest.entries, + {}, + "Content-only CAS changed membership" + ); + const entries = Object.fromEntries( + ids.map((id, i) => [id, `${i}.md`]) + ); + await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: 0, + entries + }); + // All retries must return original receipts without emitting events. + assert.deepEqual( + await Promise.all( + requests.map(({ id, body }) => + put(`${url}/documents/${id}`, body) + ) + ), + replies + ); + const waitForEvents = async (head: number) => { + const deadline = Date.now() + 5_000; + while (received.length < head && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.deepEqual( + received, + Array.from({ length: head }, (_, i) => i + 1), + "WebSocket reordered, lost or duplicated an event" + ); + }; + await waitForEvents(13); + await new Promise((resolve) => { + ws!.onclose = () => resolve(); + ws!.close(); + }); + await put( + `${url}/documents/${ids[0]}`, + content("next", replies[0].vaultUpdateId) + ); + await connect(13); + await waitForEvents(14); + const log = await getJson<{ events: { eventId: number }[] }>( + `${url}/events-since?after=0`, + token + ); + assert.deepEqual( + received, + log.events.map((event) => event.eventId) + ); + const stale = await put(`${url}/file-manifest`, { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: {} + }); + assert.equal(stale.type, "StaleBase"); + const invalid = await fetch(`${url}/file-manifest`, { + method: "PUT", + headers, + body: JSON.stringify({ + requestId: randomUUID(), + parentFileManifestId: 13, + entries: { [ids[0]]: "same", [ids[1]]: "same" } + }) + }); + assert.equal(invalid.status, 422); + assert.equal( + ( + await getJson( + `${url}/vault-snapshot`, + token + ) + ).headEventId, + 14, + "Rejected CAS emitted an event" + ); + } finally { + ws?.close(); + await server.stop(); + } + } +); diff --git a/frontend/test-support/random.ts b/frontend/test-support/random.ts new file mode 100644 index 00000000..f451ae47 --- /dev/null +++ b/frontend/test-support/random.ts @@ -0,0 +1,25 @@ +/** A seed controls harness choices; production UUIDs and OS scheduling remain + * nondeterministic. The concrete action trace is the stronger replay artifact. */ +export class Random { + private state: number; + public constructor(seed: number) { + if (!Number.isInteger(seed) || seed < 0 || seed > 0xffffffff) + throw new Error("Seed must be a uint32"); + this.state = seed; + } + public next(): number { + this.state = (this.state + 0x6d2b79f5) >>> 0; + let n = this.state; + n = Math.imul(n ^ (n >>> 15), n | 1); + n ^= n + Math.imul(n ^ (n >>> 7), n | 61); + return ((n ^ (n >>> 14)) >>> 0) / 4294967296; + } + public int(max: number): number { + if (!Number.isSafeInteger(max) || max < 1) + throw new Error("Empty random choice"); + return Math.floor(this.next() * max); + } + public pick(values: readonly T[]): T { + return values[this.int(values.length)]; + } +} diff --git a/frontend/test-support/real-disk.test.ts b/frontend/test-support/real-disk.test.ts new file mode 100644 index 00000000..3b917c81 --- /dev/null +++ b/frontend/test-support/real-disk.test.ts @@ -0,0 +1,137 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { RealDisk } from "./real-disk"; +import { Database, emptyState } from "../sync-client/src/persistence/database"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Logger } from "../sync-client/src/tracing/logger"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; + +test( + "real filesystem: exclusive operations, unsafe entries, and journal swaps", + { timeout: 20_000 }, + async () => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vault-link-real-fs-") + ); + try { + const root = path.join(directory, "vault"); + await fs.mkdir(root); + const helper = path.join(directory, "native-fs"); + await promisify(execFile)("cc", [ + "-Wall", + "-Wextra", + "-O2", + path.join(__dirname, "native-fs.c"), + "-o", + helper + ]); + const disk = new RealDisk(root, helper); + const snapshot = (text: string) => ({ + content: new TextEncoder().encode(text) + }); + const writes = await Promise.allSettled([ + disk.write("raced.bin", snapshot("first")), + disk.write("raced.bin", snapshot("second")) + ]); + assert.equal( + writes.filter((result) => result.status === "fulfilled").length, + 1, + "Exclusive create overwrote a racing writer" + ); + assert( + ["first", "second"].includes( + Buffer.from( + (await disk.readSnapshot("raced.bin"))!.content + ).toString() + ) + ); + await disk.write("a.md", snapshot("A")); + await disk.write("b.md", snapshot("B")); + // Recovery must flush the old source's ancestors even if another + // writer has replaced one of those directories with a file. + assert.equal(await disk.stat("a.md/absent.md"), undefined); + assert.equal(await disk.readSnapshot("a.md/absent.md"), undefined); + await disk.flushPaths(["a.md/absent.md"]); + await assert.rejects(disk.rename("a.md", "b.md")); + await assert.rejects(disk.delete("a.md"), /regular file/); + await fs.symlink("a.md", path.join(root, "symlink")); + await assert.rejects(disk.readSnapshot("symlink"), /Symlink/); + await fs.unlink(path.join(root, "symlink")); + await fs.link(path.join(root, "a.md"), path.join(root, "hardlink")); + await assert.rejects(disk.readSnapshot("a.md"), /hardlink/); + await fs.unlink(path.join(root, "hardlink")); + // The host may be case/normalization insensitive. Whichever model + // it exposes, an alias must never silently overwrite existing bytes. + for (const [original, alias] of [ + ["Case.bin", "case.bin"], + ["é.bin", "e\u0301.bin"] + ]) { + await disk.write(original, snapshot("original bytes")); + if (await disk.exists(alias)) + await assert.rejects( + disk.write(alias, snapshot("alias bytes")) + ); + else { + await disk.write(alias, snapshot("alias bytes")); + assert.equal( + Buffer.from( + (await disk.readSnapshot(alias))!.content + ).toString(), + "alias bytes" + ); + } + assert.equal( + Buffer.from( + (await disk.readSnapshot(original))!.content + ).toString(), + "original bytes" + ); + } + const initial = emptyState("real"); + initial.local = { a: "a.md", b: "b.md" }; + for (const [id, relative] of Object.entries(initial.local)) + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot( + (await disk.readSnapshot(relative))! + ) + ).hash + }; + const database = new Database( + new Logger(), + initial, + async () => {}, + "real", + async () => initial + ); + const operations = new FileOperations(disk, database, { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) + } as ServerConfig); + const next = structuredClone(initial); + next.local = { a: "b.md", b: "a.md" }; + await operations.apply(next); + assert.equal( + Buffer.from( + (await disk.readSnapshot("a.md"))!.content + ).toString(), + "B" + ); + assert.equal( + Buffer.from( + (await disk.readSnapshot("b.md"))!.content + ).toString(), + "A" + ); + assert.equal(database.state.application, undefined); + } finally { + await fs.rm(directory, { recursive: true }); + } + } +); diff --git a/frontend/test-support/real-disk.ts b/frontend/test-support/real-disk.ts new file mode 100644 index 00000000..d3c42639 --- /dev/null +++ b/frontend/test-support/real-disk.ts @@ -0,0 +1,215 @@ +import assert from "node:assert/strict"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { constants } from "node:fs"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { randomUUID } from "node:crypto"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +const exec = promisify(execFile); +/** Small integration fixture, NOT a supported application adapter. Safety + * preflights detect static symlinks/hardlinks; they are not an openat sandbox + * against a hostile process replacing ancestors concurrently. */ +export class RealDisk implements FileSystemOperations { + public constructor( + private readonly root: string, + private readonly helper: string + ) {} + private async resolve(relative: string): Promise { + assert( + relative === "" || + relative + .split("/") + .every( + (p) => p && p !== "." && p !== ".." && !p.includes("\\") + ), + "Unsafe path" + ); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + current = path.join(current, component); + const entry = await this.info(current); + if (entry) + assert( + entry.isDirectory() || + (entry.isFile() && entry.nlink === 1), + "Symlink, hardlink or special file" + ); + } + return current; + } + private async info(absolute: string) { + try { + return await fs.lstat(absolute); + } catch (error) { + if ( + ["ENOENT", "ENOTDIR"].includes( + (error as NodeJS.ErrnoException).code ?? "" + ) + ) + return undefined; + throw error; + } + } + private async flush(absolute: string): Promise { + const handle = await fs.open( + absolute, + constants.O_RDONLY | constants.O_NOFOLLOW + ); + try { + await handle.sync(); + } finally { + await handle.close(); + } + } + public async stat(relative: string) { + const info = await this.info(await this.resolve(relative)); + return info + ? { + kind: info.isFile() + ? ("file" as const) + : ("directory" as const), + size: info.size + } + : undefined; + } + public async exists(relative: string) { + return (await this.stat(relative)) !== undefined; + } + public async readSnapshot( + relative: string + ): Promise { + const absolute = await this.resolve(relative); + let handle; + try { + handle = await fs.open( + absolute, + constants.O_RDONLY | constants.O_NOFOLLOW + ); + } catch (error) { + if ( + ["ENOENT", "ENOTDIR"].includes( + (error as NodeJS.ErrnoException).code ?? "" + ) + ) + return undefined; + throw error; + } + try { + const before = await handle.stat({ bigint: true }); + assert( + before.isFile() && before.nlink === 1n, + "Not an independent regular file" + ); + const content = new Uint8Array(await handle.readFile()); + const after = await handle.stat({ bigint: true }); + const atPath = await fs.lstat(absolute, { bigint: true }); + assert( + before.ino === after.ino && + after.ino === atPath.ino && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs, + "File changed during read" + ); + return { content }; + } finally { + await handle.close(); + } + } + public async listFilesRecursively(root = ""): Promise { + const result: string[] = []; + for (const name of await fs.readdir(await this.resolve(root))) { + const relative = root ? `${root}/${name}` : name; + if ((await this.stat(relative))!.kind === "directory") + result.push(...(await this.listFilesRecursively(relative))); + else result.push(relative); + } + return result.sort(); + } + public async createDirectory(relative: string): Promise { + await this.resolve(relative); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + const next = path.join(current, component); + try { + await fs.mkdir(next); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") + throw error; + } + assert((await fs.lstat(next)).isDirectory()); + await this.flush(next); + await this.flush(current); + current = next; + } + } + public async write( + relative: string, + snapshot: FileSnapshot + ): Promise { + assert( + !snapshot.cursors?.length, + "This disk-only fixture has no editor selections" + ); + const destination = await this.resolve(relative); + const staging = ".vault-link-sync/adapter-staging"; + await this.createDirectory(staging); + const source = path.join(this.root, staging, randomUUID()); + const handle = await fs.open(source, "wx", 0o600); + try { + await handle.writeFile(snapshot.content); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await exec(this.helper, ["rename", source, destination]); + await this.flush(path.dirname(source)); + await this.flush(path.dirname(destination)); + } finally { + await fs.unlink(source).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }); + } + } + public async rename(from: string, to: string): Promise { + const source = await this.resolve(from), + destination = await this.resolve(to); + assert((await this.info(source))?.isFile()); + await this.flush(source); + await exec(this.helper, ["rename", source, destination]); + await this.flush(path.dirname(source)); + await this.flush(path.dirname(destination)); + } + public async flushPaths(paths: readonly string[]): Promise { + for (const relative of paths) { + let absolute = await this.resolve(relative); + for (;;) { + if (await this.info(absolute)) await this.flush(absolute); + if (absolute === this.root) break; + absolute = path.dirname(absolute); + } + } + } + public async delete(relative: string): Promise { + assert(relative, "Cannot prune root"); + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isDirectory(), "Cannot delete regular file"); + for (const child of await fs.readdir(absolute)) + await this.delete(`${relative}/${child}`); + await fs.rmdir(absolute); + await this.flush(path.dirname(absolute)); + } + public async deleteFile(relative: string): Promise { + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); + await fs.unlink(absolute); + await this.flush(path.dirname(absolute)); + } +} diff --git a/frontend/test-support/recovery.test.ts b/frontend/test-support/recovery.test.ts new file mode 100644 index 00000000..8da22c4a --- /dev/null +++ b/frontend/test-support/recovery.test.ts @@ -0,0 +1,695 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { isDeepStrictEqual } from "node:util"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import { mergeFileManifests } from "../sync-client/src/sync-operations/file-manifest"; +import { assertManifest, pathDecision } from "./oracles"; +import { + scanLocalFiles, + type LocalChange +} from "../sync-client/src/sync-operations/scan"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const config = { + getConfig: async () => ({ mergeableFileExtensions: ["md"] }) +} as ServerConfig; + +async function fixture( + contents: Record = { + a: { path: "a.md", text: "A" }, + b: { path: "b.md", text: "B" } + } +) { + const disk = new MemoryDisk(); + const initial = emptyState("recovery-test"); + for (const [id, { path, text }] of Object.entries(contents)) { + await disk.userWrite(path, bytes(text)); + initial.local[id] = path; + } + for (const [id, path] of Object.entries(initial.local)) + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot((await disk.readSnapshot(path))!) + ).hash + }; + const persistence = new MemoryPersistence({ database: initial }); + const restart = (changes: LocalChange[] = []) => { + const database = new Database( + new Logger(), + persistence.snapshot().database, + async (next) => persistence.save({ database: next }), + "recovery-test", + async () => (await persistence.load()).database as StoredDatabase + ); + return { + database, + files: new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} } + ) + }; + }; + return { + disk, + persistence, + restart, + initial, + permanentFault: (_label: string): void => {}, + expectedLocal: undefined as Record | undefined + }; +} + +async function swapFixture() { + const current = await fixture(); + const next = structuredClone(current.initial); + next.local = { a: "b.md", b: "a.md", c: "nested/c.md" }; + next.documents.c = { materialized: false }; + const writes = { + c: { replacement: await toStoredSnapshot({ content: bytes("C") }) } + }; + const expected = new Map([ + ["a.md", bytes("B")], + ["b.md", bytes("A")], + ["nested/c.md", bytes("C")] + ]); + return { ...current, next, writes, expected, retained: [] as string[] }; +} + +async function crossDirectoryFixture() { + const current = await fixture({ + a: { path: "src/a.md", text: "A" }, + // Keep src nonempty: deleting an empty source directory would mask an + // unflushed source unlink in the staging rename. + b: { path: "src/keep.md", text: "untouched sibling" } + }); + const next = structuredClone(current.initial); + next.local.a = "dst/a.md"; + const expected = new Map([ + ["dst/a.md", bytes("A")], + ["src/keep.md", bytes("untouched sibling")] + ]); + return { ...current, next, writes: {}, expected, retained: [] as string[] }; +} + +async function mergeFixture() { + const base = "first paragraph\n\nlast paragraph\n"; + const local = "local first paragraph\n\nlast paragraph\n"; + const remote = "first paragraph\n\nremote last paragraph\n"; + const current = await fixture({ a: { path: "note.md", text: local } }); + return { + ...current, + next: structuredClone(current.initial), + writes: { + a: { + expected: await toStoredSnapshot({ content: bytes(base) }), + replacement: await toStoredSnapshot({ content: bytes(remote) }) + } + }, + expected: new Map([ + [ + "note.md", + bytes("local first paragraph\n\nremote last paragraph\n") + ] + ]), + retained: [local] + }; +} + +async function deletionFixture() { + const current = await fixture({ + a: { path: "a.bin", text: "private binary edit\0" }, + b: { path: "deleted.md", text: "unsent deleted bytes" } + }); + const next = structuredClone(current.initial); + delete next.local.b; + return { + ...current, + next, + writes: { + a: { + replacement: await toStoredSnapshot({ + content: bytes("remote binary\0") + }) + } + }, + expected: new Map([["a.bin", bytes("remote binary\0")]]), + retained: ["private binary edit\0", "unsent deleted bytes"] + }; +} + +async function permanentDestinationFixture() { + const current = await fixture(); + const next = structuredClone(current.initial); + next.local = { a: "unsupported/a.md", b: "good.md" }; + return { + ...current, + next, + permanentFault: (label: string) => { + if (label === "before:mkdir:unsupported") + throw Object.assign(new Error("Unsupported destination"), { + code: "ENAMETOOLONG" + }); + }, + writes: { + a: { + expected: await toStoredSnapshot({ content: bytes("A") }), + replacement: await toStoredSnapshot({ + content: bytes("REMOTE") + }) + } + }, + expectedLocal: { a: "Recovered a.md", b: "good.md" }, + expected: new Map([ + ["Recovered a.md", bytes("REMOTE")], + ["good.md", bytes("B")] + ]), + retained: ["A"] + }; +} + +for (const [name, create] of [ + ["swap and create", swapFixture], + ["cross-directory move with surviving sibling", crossDirectoryFixture], + ["merge during recovery", mergeFixture], + ["binary replacement, deletion and retained bytes", deletionFixture], + ["permanent destination replan", permanentDestinationFixture] +] as const) { + test(`${name}: interrupt every apply and replay boundary, then power-cycle again`, async (t) => { + const baseline = await create(); + const labels: string[] = []; + const trace = (label: string) => { + if (!label.startsWith("read:")) labels.push(label); + }; + baseline.disk.boundary = (label) => { + baseline.permanentFault(label); + trace(label); + }; + baseline.persistence.boundary = trace; + await baseline.restart().files.apply(baseline.next, baseline.writes); + assert.deepEqual(baseline.disk.userFiles(), baseline.expected); + assert(labels.some((label) => label.startsWith("visible:rename:"))); + assert(labels.includes("durable:save")); + let replayCrashes = 0; + for (const powerLoss of [false, true]) { + for (let index = 0; index < labels.length; index++) { + await t.test( + `${powerLoss ? "power" : "process"} crash #${index}: ${labels[index]}`, + async () => { + const current = await create(); + const clearFaults = () => { + current.disk.boundary = current.permanentFault; + current.persistence.boundary = () => {}; + }; + const interrupt = async ( + action: () => Promise, + at: number, + power: boolean + ) => { + let position = 0, + fired = false; + const fault = (label: string) => { + if (label.startsWith("read:")) return; + if (position++ === at) { + fired = true; + current.disk.crash(power); + throw new Error("Injected interruption"); + } + }; + current.disk.boundary = (label) => { + current.permanentFault(label); + fault(label); + }; + current.persistence.boundary = fault; + try { + await assert.rejects( + action(), + /Injected interruption/ + ); + assert(fired, "Fault index did not occur"); + } finally { + clearFaults(); + } + }; + const verify = ( + runtime: ReturnType + ) => { + assert.equal( + runtime.database.state.application, + undefined + ); + assert.deepEqual( + runtime.database.state.local, + current.expectedLocal ?? current.next.local + ); + assert.deepEqual( + current.disk.userFiles(), + current.expected, + "Exact files must survive interrupted replay" + ); + const retained = current.disk + .image() + .visible.filter( + ([path, entry]) => + entry && + path.startsWith( + ".vault-link-sync/recovery/" + ) && + path.endsWith(".content") + ) + .map(([, entry]) => + Buffer.from(entry!.content).toString() + ) + .sort(); + assert.deepEqual( + retained, + [...current.retained].sort(), + "Recovery archives lost or duplicated displaced bytes" + ); + assert( + !current.disk + .image() + .visible.some( + ([path, entry]) => + entry && + path.startsWith( + ".vault-link-sync/transactions/" + ) + ), + "Completed recovery left transaction artifacts" + ); + }; + const finish = async (resubmit: boolean) => { + let runtime = current.restart(); + await runtime.files.recover(); + if (resubmit) + await runtime.files.apply( + current.next, + current.writes + ); + await runtime.files.recover(); + verify(runtime); + current.disk.crash(true); + runtime = current.restart(); + await runtime.files.recover(); + verify(runtime); + }; + await interrupt( + () => + current + .restart() + .files.apply(current.next, current.writes), + index, + powerLoss + ); + const saved = current.persistence.snapshot(); + const diskImage = current.disk.image(); + const intentWasNotSaved = isDeepStrictEqual( + saved.database, + current.initial + ); + // Trace replay from the actual interrupted state, including + // recovery-only branches such as already-visible installs. + const replayLabels: string[] = []; + const traceReplay = (label: string) => { + if (!label.startsWith("read:")) + replayLabels.push(label); + }; + current.disk.boundary = (label) => { + current.permanentFault(label); + traceReplay(label); + }; + current.persistence.boundary = traceReplay; + await current.restart().files.recover(); + clearFaults(); + await finish(intentWasNotSaved); + for (const secondPowerLoss of [false, true]) { + for ( + let replay = 0; + replay < replayLabels.length; + replay++ + ) { + current.disk.crash(); + current.disk.restore(diskImage); + await current.persistence.save(saved); + try { + await interrupt( + () => current.restart().files.recover(), + replay, + secondPowerLoss + ); + await finish(false); + replayCrashes++; + } catch (error) { + throw new Error( + `Second ${secondPowerLoss ? "power" : "process"} crash #${replay}: ${replayLabels[replay]}`, + { cause: error } + ); + } + } + } + } + ); + } + } + assert(replayCrashes > 0, "No interrupted recovery was exercised"); + t.diagnostic(`${replayCrashes} second crashes during journal replay`); + }); +} + +test("production manifest merge obeys independently specified per-document policy", () => { + const paths = [undefined, "a.md", "b.md", "nested/c.md"]; + const map = (path: string | undefined): Record => + path === undefined + ? {} + : { "00000000-0000-4000-8000-000000000001": path }; + for (const base of paths) + for (const local of paths) + for (const remote of paths) { + assert.deepEqual( + mergeFileManifests(map(base), map(local), map(remote)), + map(pathDecision(base, local, remote)) + ); + } + const conflicts = mergeFileManifests( + {}, + { a: "folder" }, + { b: "folder/note.md" } + ); + assertManifest(conflicts); + assert.equal( + conflicts.b, + "folder/note.md", + "Canonical path must win collision" + ); + assert.equal( + Object.keys(conflicts).length, + 2, + "Collision must not discard an ID" + ); +}); + +test("exhaustive two-document manifest states preserve IDs and canonical collision priority", () => { + const ids = [ + "00000000-0000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000002" + ]; + const states: Record[] = []; + for (const a of [undefined, "a.md", "b.md"]) + for (const b of [undefined, "a.md", "b.md"]) { + if (a && a === b) continue; + states.push({ + ...(a ? { [ids[0]]: a } : {}), + ...(b ? { [ids[1]]: b } : {}) + }); + } + for (const base of states) + for (const local of states) + for (const remote of states) { + const expected = Object.fromEntries( + ids + .map((id) => [ + id, + pathDecision(base[id], local[id], remote[id]) + ]) + .filter(([, value]) => value !== undefined) + ); + const merged = mergeFileManifests(base, local, remote); + assertManifest(merged); + assert.deepEqual( + Object.keys(merged).sort(), + Object.keys(expected).sort(), + "Merge discarded or invented an identity" + ); + for (const id of ids) + if ( + expected[id] !== undefined && + remote[id] === expected[id] + ) + assert.equal( + merged[id], + remote[id], + "Canonical claimant lost its path" + ); + assert.deepEqual( + mergeFileManifests(remote, merged, remote), + merged, + "Manifest merge is not idempotent" + ); + } +}); + +test("scan failures cannot masquerade as deletions", async () => { + for (const point of ["read:list:", "read:snapshot:a.md"]) { + const current = await fixture(); + const runtime = current.restart(); + const original = structuredClone(runtime.database.state); + let fired = false; + current.disk.boundary = (label) => { + if (label === point) { + fired = true; + throw new Error("permission denied"); + } + }; + await assert.rejects( + scanLocalFiles({ + next: structuredClone(original), + changes: [], + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }), + /permission denied/ + ); + assert(fired); + assert.deepEqual(runtime.database.state, original); + assert.deepEqual(current.persistence.snapshot().database, original); + } +}); + +for (const phase of ["before", "durable"]) { + test(`delete/recreate identity survives ${phase}-save failure and retry`, async () => { + const current = await fixture(); + await current.disk.userDelete("a.md"); + await current.disk.userWrite("a.md", bytes("new incarnation")); + const changes: LocalChange[] = [ + { type: "delete", path: "a.md" }, + { type: "create", path: "a.md" } + ]; + const scan = async () => { + const runtime = current.restart(); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + return runtime.database.state; + }; + current.persistence.boundary = (label) => { + if (label === `${phase}:save`) throw new Error("save failure"); + }; + await assert.rejects(scan(), /save failure/); + current.persistence.boundary = () => {}; + const state = await scan(); + assert.equal( + state.local.a, + undefined, + "Retry reused the deleted document's UUID" + ); + assert.equal( + Object.values(state.local).filter((path) => path === "a.md").length, + 1 + ); + }); +} + +async function scanChanges( + runtime: { database: Database; files: FileOperations }, + changes: LocalChange[] +): Promise { + await runtime.database.recoverPersistence(); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); +} + +for (const editedContent of ["new bytes", "B"]) { + test(`explicit rename then edit to ${JSON.stringify(editedContent)} preserves both UUIDs`, async () => { + const current = await fixture(); + const runtime = current.restart(); + await current.disk.userRename("a.md", "moved.md"); + await current.disk.userWrite("moved.md", bytes(editedContent)); + const changes: LocalChange[] = [ + { type: "move", oldPath: "a.md", relativePath: "moved.md" } + ]; + await scanChanges(runtime, changes); + assert.deepEqual( + runtime.database.state.local, + { a: "moved.md", b: "b.md" }, + "Matching another live document's bytes must not steal its UUID" + ); + assert.deepEqual( + current.disk.userFiles(), + new Map([ + ["moved.md", bytes(editedContent)], + ["b.md", bytes("B")] + ]) + ); + assert.equal(changes.length, 0); + }); +} + +for (const equalContent of [false, true]) { + for (const phase of ["before", "durable"]) { + test(`${equalContent ? "equal" : "different"}-content swap retains UUIDs after ${phase}-save rejection`, async () => { + const current = await fixture({ + a: { path: "a.md", text: "A" }, + b: { path: "b.md", text: equalContent ? "A" : "B" } + }); + const runtime = current.restart(); + const changes: LocalChange[] = []; + for (const [oldPath, relativePath] of [ + ["a.md", "temporary.md"], + ["b.md", "a.md"], + ["temporary.md", "b.md"] + ]) { + await current.disk.userRename(oldPath, relativePath); + changes.push({ type: "move", oldPath, relativePath }); + } + const expected = { a: "b.md", b: "a.md" }; + const expectedFiles = new Map([ + ["a.md", bytes(equalContent ? "A" : "B")], + ["b.md", bytes("A")] + ]); + let fired = false; + current.persistence.boundary = (label) => { + if (!fired && label === `${phase}:save`) { + fired = true; + throw new Error("uncertain save"); + } + }; + await assert.rejects( + scanChanges(runtime, changes), + /uncertain save/ + ); + assert(fired, "The requested save failure must be exercised"); + assert.deepEqual( + current.persistence.snapshot().database!.local, + phase === "durable" ? expected : current.initial.local, + "The fault must occur on the intended side of the durable save" + ); + current.persistence.boundary = () => {}; + // Model an uncertain save in a live client: durable state reloads, + // but the original editor notifications remain in memory. Do not + // clear, reconstruct or deduplicate that queue in the harness. + await scanChanges(runtime, changes); + assert.deepEqual( + runtime.database.state.local, + expected, + "Retry replayed an already committed swap and reversed its identities" + ); + assert.equal(changes.length, 0); + await scanChanges(runtime, changes); + assert.deepEqual(runtime.database.state.local, expected); + const restarted = current.restart(); + await scanChanges(restarted, []); + assert.deepEqual(restarted.database.state.local, expected); + assert.deepEqual(current.disk.userFiles(), expectedFiles); + }); + } +} + +test("normalization-sensitive disk does not lose a distinct decomposed filename", async () => { + const current = await fixture(); + await current.disk.userWrite("é.md", bytes("composed")); + const runtime = current.restart(); + const next = structuredClone(runtime.database.state); + next.local.composed = "é.md"; + next.documents.composed = { materialized: true }; + await runtime.database.commit(next); + await current.disk.userWrite("e\u0301.md", bytes("decomposed")); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes: [], + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + assert.equal( + Object.keys(runtime.database.state.local).length, + 4, + "Distinct Unicode file vanished from the scan" + ); +}); + +test("notified editor move during a remote swap keeps the moved document's UUID", async () => { + const current = await fixture(); + const changes: LocalChange[] = []; + const runtime = current.restart(changes); + let fired = false; + current.persistence.boundary = async (label) => { + if ( + !fired && + label === "durable:save" && + current.persistence.snapshot().database?.application + ) { + fired = true; + await current.disk.userRename("b.md", "editor.md"); + changes.push({ + type: "move", + oldPath: "b.md", + relativePath: "editor.md" + }); + } + }; + const next = structuredClone(runtime.database.state); + next.local = { a: "b.md", b: "a.md" }; + await runtime.files.apply(next); + assert(fired); + await scanLocalFiles({ + next: structuredClone(runtime.database.state), + changes, + files: runtime.files, + guard: () => {}, + ignored: () => false, + oversized: () => false, + commit: (next) => runtime.database.commit(next) + }); + assert.equal( + runtime.database.state.local.b, + "editor.md", + "Editor move was attributed to the wrong document" + ); + assert.equal( + Buffer.from( + current.disk.userFiles().get(runtime.database.state.local.b)! + ).toString(), + "B" + ); +}); diff --git a/frontend/test-support/server-restore.test.ts b/frontend/test-support/server-restore.test.ts new file mode 100644 index 00000000..d8598429 --- /dev/null +++ b/frontend/test-support/server-restore.test.ts @@ -0,0 +1,179 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +import { readdir, copyFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +for (const mode of [ + "same-version", + "behind", + "dirty", + "pending", + "legacy-state", + ...["before:save", "durable:save"].flatMap((boundary) => + [1, 2, 3, 4, 5].map((ordinal) => `${boundary}:${ordinal}`) + ) +] as const) + test(`restored history recovers ${mode} without false convergence or lost local work`, async () => { + const f = await fixture(); + const crashRecovery = mode.includes(":save:"); + const hasLocalEdit = + mode === "dirty" || mode === "pending" || crashRecovery; + try { + const id = randomUUID(); + const h = await f.put("/documents/" + id, f.content("BASE")); + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "a.md" } + }); + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + const vaults = join(f.server.databaseDirectory, "vaults"); + const db = join( + vaults, + (await readdir(vaults)).find((p) => p.endsWith(".sqlite"))! + ); + const backup = db + ".backup"; + execFileSync("python3", [ + "-c", + "import sqlite3,sys; s=sqlite3.connect(sys.argv[1]); d=sqlite3.connect(sys.argv[2]); s.backup(d); d.close(); s.close()", + db, + backup + ]); + const old = await f.put( + "/documents/" + id, + f.content("OLD BRANCH", h.vaultUpdateId) + ); + await f.client.setSetting("isSyncEnabled", true); + await f.client.setSetting("isSyncEnabled", false); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(), + "OLD BRANCH" + ); + if (hasLocalEdit) { + await f.disk.userWrite("a.md", bytes("OLD BRANCH LOCAL")); + await f.client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + if (mode === "pending") { + // Lose an acknowledgement after the server committed, leaving a + // durable request that must not be retried against the restored DB. + f.setFetchHook(async (input, init) => { + const response = await fetch(input, init); + if (init?.method === "PUT") + throw new Error("lost acknowledgement"); + return response; + }); + await assert.rejects( + f.client.setSetting("isSyncEnabled", true), + /lost acknowledgement/ + ); + await f.client.setSetting("isSyncEnabled", false); + f.setFetchHook(fetch); + } + await f.server.crash(); + await copyFile(backup, db); + await rm(db + "-wal", { force: true }); + await rm(db + "-shm", { force: true }); + await f.server.restart(); + const fork = + mode === "behind" + ? h + : await f.put( + "/documents/" + id, + f.content("NEW BRANCH", h.vaultUpdateId) + ); + if (mode !== "behind") + assert.equal(fork.vaultUpdateId, old.vaultUpdateId); + let interrupted = false, + mismatch = false; + if (crashRecovery) { + const boundary = mode.split(":").slice(0, 2).join(":"); + const ordinal = Number(mode.split(":")[2]); + let saves = 0; + f.setFetchHook(async (input, init) => { + const response = await fetch(input, init); + if ( + response.headers.get( + "x-vault-link-history-mismatch" + ) === "1" + ) + mismatch = true; + return response; + }); + f.store.boundary = (label) => { + if ( + mismatch && + !interrupted && + label === boundary && + ++saves === ordinal + ) { + // Sweep the scan, durable reset intent, checkpoint + // replacement and final bootstrap-enabling saves. + interrupted = true; + throw new Error("interrupted recovery save"); + } + }; + await assert.rejects( + f.client.setSetting("isSyncEnabled", true) + ); + assert(interrupted && mismatch); + f.store.boundary = () => {}; + await f.reopenClient(); + await f.client.start(); + } + if (mode === "legacy-state") { + await f.client.destroy(); + const stored = f.store.snapshot() as ReturnType< + typeof f.store.snapshot + > & { historyCheckpoint?: unknown }; + delete stored.historyCheckpoint; + await f.store.save(stored); + await f.reopenClient(); + await f.client.start(); + } + await f.client.setSetting("isSyncEnabled", true); + const actual = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + const remote = Buffer.from( + (await f.get("/documents/" + id)).contentBase64, + "base64" + ).toString(); + + assert.equal(actual, remote); + assert.equal(remote, mode === "behind" ? "BASE" : "NEW BRANCH"); + if (hasLocalEdit) { + const manifest = await f.get("/file-manifest"); + const recovered = Object.keys(manifest.entries).filter( + (other) => other !== id + ); + assert.equal(recovered.length, 1); + assert.equal( + Buffer.from( + (await f.get("/documents/" + recovered[0])) + .contentBase64, + "base64" + ).toString(), + "OLD BRANCH LOCAL" + ); + assert.equal( + Buffer.from( + f.disk.userFiles().get(manifest.entries[recovered[0]])! + ).toString(), + "OLD BRANCH LOCAL" + ); + } + await f.client.reset(); + assert.equal( + Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(), + remote + ); + } finally { + await f.dispose(); + } + }); diff --git a/frontend/test-support/storage.ts b/frontend/test-support/storage.ts new file mode 100644 index 00000000..816dc122 --- /dev/null +++ b/frontend/test-support/storage.ts @@ -0,0 +1,320 @@ +import assert from "node:assert/strict"; +import type { + FileSnapshot, + FileSystemOperations, + StoredDatabase, + SyncSettings +} from "sync-client"; + +export type StoredClient = Partial<{ + settings: Partial; + database: Partial; +}>; +export type Boundary = (label: string) => void | Promise; +type Entry = FileSnapshot | null; // null is a directory; absence is not a directory. +export interface DiskImage { + visible: [string, Entry][]; + durable: [string, Entry][]; +} +const parent = (path: string) => path.split("/").slice(0, -1).join("/"); +export const isInternal = (path: string) => + path.normalize("NFC").toUpperCase().split("/")[0] === ".VAULT-LINK-SYNC"; + +/** The adapter and the editor are deliberately separate APIs. Engine writes do + * not generate editor notifications. All returned snapshots are owned copies. + * + * This models atomic namespace operations, not a particular filesystem's + * journal. At a visible boundary a power loss may roll back the operation; + * a process death alone does not roll back the kernel's dirty state. + */ +export class MemoryDisk implements FileSystemOperations { + private visible = new Map([["", null]]); + private durable = new Map([["", null]]); + private generation = 0; + public boundary: Boundary = () => {}; + + public image(): DiskImage { + return structuredClone({ + visible: [...this.visible], + durable: [...this.durable] + }); + } + + public restore(image: DiskImage): void { + this.visible = new Map(structuredClone(image.visible)); + this.durable = new Map(structuredClone(image.durable)); + } + + public crash(powerLoss = false): void { + this.generation++; + if (powerLoss) this.visible = structuredClone(this.durable); + } + + /** Fence an abandoned runtime, including calls it makes after a restart. */ + public session(): FileSystemOperations { + const generation = this.generation; + return new Proxy(this, { + get: (target, key) => { + const value = Reflect.get(target, key); + if (typeof value !== "function") return value; + return (...args: unknown[]) => { + assert.equal( + this.generation, + generation, + "Access from crashed filesystem session" + ); + return value.apply(target, args); + }; + } + }); + } + + private check(path: string): void { + assert( + !path.startsWith("/") && + !path.includes("\\") && + !path.includes("\0"), + `Unsafe path: ${path}` + ); + assert( + path === "" || + path + .split("/") + .every( + (part) => part !== "" && part !== "." && part !== ".." + ), + `Unsafe path: ${path}` + ); + for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor)) + assert( + !this.visible.has(ancestor) || + this.visible.get(ancestor) === null, + `File ancestor: ${ancestor}` + ); + } + + private async change( + label: string, + paths: string[], + mutate: () => void + ): Promise { + const generation = this.generation; + await this.boundary(`before:${label}`); + assert.equal( + this.generation, + generation, + "Mutation resumed after crash" + ); + mutate(); + await this.boundary(`visible:${label}`); + assert.equal(this.generation, generation, "Flush resumed after crash"); + for (const path of paths) { + if (this.visible.has(path)) + this.durable.set( + path, + structuredClone(this.visible.get(path)!) + ); + else this.durable.delete(path); + } + await this.boundary(`durable:${label}`); + assert.equal( + this.generation, + generation, + "Operation returned after crash" + ); + } + + public async stat(path: string) { + this.check(path); + await this.boundary(`read:stat:${path}`); + const entry = this.visible.get(path); + return entry === undefined + ? undefined + : entry === null + ? { kind: "directory" as const, size: 0 } + : { kind: "file" as const, size: entry.content.length }; + } + + public async exists(path: string): Promise { + return (await this.stat(path)) !== undefined; + } + + public async readSnapshot(path: string): Promise { + this.check(path); + await this.boundary(`read:snapshot:${path}`); + const entry = this.visible.get(path); + assert(entry !== null, `Cannot read directory: ${path}`); + return structuredClone(entry); + } + + public async listFilesRecursively(root = ""): Promise { + this.check(root); + await this.boundary(`read:list:${root}`); + assert.equal( + this.visible.get(root), + null, + `Cannot list non-directory: ${root}` + ); + return [...this.visible] + .filter( + ([p, entry]) => + entry !== null && (!root || p.startsWith(`${root}/`)) + ) + .map(([p]) => p) + .sort(); + } + + public userFiles(): Map { + return new Map( + [...this.visible] + .filter(([p, e]) => e !== null && !isInternal(p)) + .map(([p, e]) => [p, e!.content.slice()]) + ); + } + + public async createDirectory(path: string): Promise { + this.check(path); + const paths: string[] = []; + for (let p = path; p; p = parent(p)) paths.unshift(p); + await this.change(`mkdir:${path}`, paths, () => { + for (const p of paths) + assert( + !this.visible.has(p) || this.visible.get(p) === null, + `File blocks directory: ${p}` + ); + for (const p of paths) this.visible.set(p, null); + }); + } + + public async write(path: string, snapshot: FileSnapshot): Promise { + this.check(path); + await this.change(`write:${path}`, [path], () => { + assert(!this.visible.has(path), `Destination exists: ${path}`); + assert.equal( + this.visible.get(parent(path)), + null, + `Parent missing: ${path}` + ); + this.visible.set(path, structuredClone(snapshot)); + }); + } + + public async rename(from: string, to: string): Promise { + this.check(from); + this.check(to); + await this.change(`rename:${from}->${to}`, [from, to], () => { + const source = this.visible.get(from); + assert(source, `Source is not a regular file: ${from}`); + assert(!this.visible.has(to), `Destination exists: ${to}`); + assert.equal( + this.visible.get(parent(to)), + null, + `Parent missing: ${to}` + ); + this.visible.set(to, source); + this.visible.delete(from); + }); + } + + public async flushPaths(paths: readonly string[]): Promise { + const affected = new Set(paths); + for (const path of paths) + for (let p = parent(path); p; p = parent(p)) affected.add(p); + await this.change(`flush:${paths.join(",")}`, [...affected], () => {}); + } + + public async delete(path: string): Promise { + this.check(path); + assert(path !== "", "Cannot remove vault root"); + const visiblePaths = [...this.visible.keys()].filter( + (p) => p === path || p.startsWith(`${path}/`) + ); + // A durable rmdir removes access to the whole subtree. A child whose + // unlink was visible but unflushed cannot reappear beneath a durably + // removed directory when this flat namespace is restored. + const paths = [ + ...new Set([ + ...visiblePaths, + ...[...this.durable.keys()].filter( + (p) => p === path || p.startsWith(`${path}/`) + ) + ]) + ]; + await this.change(`rmdir:${path}`, paths, () => { + for (const p of visiblePaths) + assert.equal( + this.visible.get(p), + null, + `Cannot unlink regular file: ${p}` + ); + for (const p of visiblePaths) this.visible.delete(p); + }); + } + + public async deleteFile(path: string): Promise { + this.check(path); + await this.change(`unlink:${path}`, [path], () => { + const entry = this.visible.get(path); + assert(entry !== null, `Cannot unlink directory: ${path}`); + this.visible.delete(path); + }); + } + + /** External editor mutations are durable by default and may replace files. */ + public async userWrite(path: string, content: Uint8Array): Promise { + this.check(path); + await this.createDirectory(parent(path)); + assert( + this.visible.get(path) !== null, + `Cannot overwrite directory: ${path}` + ); + this.visible.set(path, { content: content.slice() }); + this.durable.set(path, { content: content.slice() }); + } + + public async userDelete(path: string): Promise { + this.check(path); + assert(this.visible.get(path), `Cannot delete missing file: ${path}`); + this.visible.delete(path); + this.durable.delete(path); + } + + public async userRename(from: string, to: string): Promise { + this.check(from); + this.check(to); + await this.createDirectory(parent(to)); + const source = this.visible.get(from); + assert(source, `Source is not a regular file: ${from}`); + assert( + this.visible.get(to) !== null, + `Cannot overwrite directory: ${to}` + ); + // Editors may explicitly replace a destination. This is NOT the + // engine's rename API, which must never overwrite it. + this.visible.delete(from); + this.durable.delete(from); + this.visible.set(to, structuredClone(source)); + this.durable.set(to, structuredClone(source)); + } +} + +/** Atomic replacement with both definite failure and uncertain success. */ +export class MemoryPersistence { + private durable: StoredClient; + public boundary: Boundary = () => {}; + public constructor(initial: StoredClient = {}) { + this.durable = structuredClone(initial); + } + public async load(): Promise { + return structuredClone(this.durable); + } + public snapshot(): StoredClient { + return structuredClone(this.durable); + } + public async save(next: StoredClient): Promise { + const owned = structuredClone(next); + await this.boundary("before:save"); + this.durable = owned; + await this.boundary("durable:save"); + } +} diff --git a/frontend/test-support/sync-audit.test.ts b/frontend/test-support/sync-audit.test.ts new file mode 100644 index 00000000..ae842a83 --- /dev/null +++ b/frontend/test-support/sync-audit.test.ts @@ -0,0 +1,900 @@ +import assert from "node:assert/strict"; +import { PermanentSyncError } from "../sync-client/src/errors/errors"; +import { test } from "node:test"; +import type { Database } from "../sync-client/src/persistence/database"; +import { emptyState } from "../sync-client/src/persistence/database"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +import { fixture, bytes, head } from "./sync-fixture"; + +test("ignored rename must not produce a duplicate submitted path", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + for (const [id, path] of Object.entries(f.database.state.local)) { + f.database.state.documents[id].base = { + ...head(id, 1, id.toUpperCase()), + hash: f.database.state.documents[id].observedHash! + }; + } + await f.disk.userRename("a.md", "ignored.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "ignored.md" + }); + await f.disk.userRename("b.md", "a.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "a.md" + }); + await f.settings.setSettings({ ignorePatterns: ["ignored.md"] }); + await f.internals.scan(); + f.internals.stopped = false; + await f.internals.preparePush(); + const p = f.database.state.pending; + assert(p?.type === "fileManifest"); + assert.equal( + new Set(Object.values(p.request.entries)).size, + Object.keys(p.request.entries).length + ); +}); + +test("ignored untracked ancestor must remain in place", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("remote") } + ); + await f.disk.userWrite("private", bytes("LOCAL PRIVATE")); + await f.settings.setSettings({ ignorePatterns: ["private"] }); + f.database.state.remoteHeads.a = head("a", 2, "remote"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "private/note.md" } + }); + f.internals.stopped = false; + await f.internals.preparePush(); + assert.deepEqual(await f.disk.readSnapshot("private"), { + content: bytes("LOCAL PRIVATE") + }); +}); + +test("oversized untracked file must remain in place", async () => { + const f = await fixture( + {}, + { getDocumentVersionContent: async () => bytes("R") } + ); + await f.disk.userWrite("a.md", bytes("LOCAL OVERSIZED")); + await f.settings.setSettings({ maxFileSizeMB: 5 / (1024 * 1024) }); + f.database.state.remoteHeads.a = head("a", 2, "R"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "a.md" } + }); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("LOCAL OVERSIZED") + ); +}); + +test("rejected unknown success must use submitted content as merge base", async () => { + const f = await fixture( + { "a.md": "base" }, + { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes( + vaultUpdateId === 1 + ? "base" + : vaultUpdateId === 2 + ? "base A" + : "base A REMOTE" + ) + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("base A")); + f.internals.stopped = false; + await f.internals.preparePush(); + const req = f.database.state.pending!.request.requestId; + await f.internals.finishPending(); + await f.disk.userWrite("a.md", bytes("base")); + await f.internals.incorporateEventBatch({ + headEventId: 3, + events: [ + { + eventId: 2, + requestId: req, + type: "content", + document: head("a", 2, "base A") + }, + { + eventId: 3, + requestId: "other", + type: "content", + document: head("a", 3, "base A REMOTE") + } + ] + }); + const result = Buffer.from( + (await f.disk.readSnapshot("a.md"))!.content + ).toString(); + assert.equal(result, "base REMOTE"); +}); + +test("bootstrap adopts same-path identity even after multiple notifications", async () => { + const f = await fixture({}); + f.database.state.initialized = false; + f.database.state.fileManifest = { fileManifestId: 0, entries: {} }; + await f.disk.userWrite("a.md", bytes("local a")); + await f.syncer.syncLocallyCreatedFile("a.md"); + await f.disk.userWrite("b.md", bytes("local b")); + await f.syncer.syncLocallyCreatedFile("b.md"); + const initial = { + headEventId: 2, + fileManifest: { fileManifestId: 2, entries: { remoteA: "a.md" } }, + documents: [head("remoteA", 1, "remote a")] + }; + await f.internals.scan(initial); + assert.equal(f.database.state.local.remoteA, "a.md"); +}); + +test("oversized remote head must not cause identical stale uploads forever", async () => { + let attempts = 0; + const remoteText = "R".repeat(20); + const f = await fixture( + { "a.md": "edit" }, + { + putFileContent: async () => { + attempts++; + return { + type: "StaleBase", + ...head("a", 2, remoteText), + contentBase64: Buffer.from(remoteText).toString("base64") + }; + } + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 2, remoteText); + await f.settings.setSettings({ maxFileSizeMB: 10 / (1024 * 1024) }); + f.internals.stopped = false; + // Even a small local edit must wait until its required remote base is readable. + assert.equal(await f.internals.preparePush(), false); + assert.equal(attempts, 0); + assert.equal(f.database.state.pending, undefined); +}); + +test("local undo of accepted-but-rejected manifest must survive replay", async () => { + const f = await fixture( + { "a.md": "A" }, + { + pushFileManifest: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async () => bytes("A") + } + ); + f.database.state.lastSeenUpdateId = 1; + f.database.state.documents.a.base = { + ...head("a", 1, "A"), + hash: f.database.state.documents.a.observedHash! + }; + await f.disk.userRename("a.md", "b.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "a.md", + relativePath: "b.md" + }); + await f.internals.scan(); + f.internals.stopped = false; + await f.internals.preparePush(); + const req = f.database.state.pending!.request.requestId; + f.internals.stopped = true; + await f.internals.finishPending(); + await f.disk.userRename("b.md", "a.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "a.md" + }); + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId: req, + type: "fileManifest", + fileManifest: { fileManifestId: 2, entries: { a: "b.md" } } + } + ] + }); + assert.equal(f.database.state.local.a, "a.md"); +}); + +test("disabled startup should recover an active journal before returning", async () => { + const disk = new MemoryDisk(); + const key = JSON.stringify(["http://offline.test", "test"]); + const state = emptyState(key); + state.initialized = true; + state.local = { a: "a.md" }; + state.documents = { a: { materialized: true } }; + const next = structuredClone(state); + next.local = { a: "b.md" }; + const prefix = ".vault-link-sync/transactions/active/a"; + await disk.userWrite(prefix + ".source", bytes("ONLY COPY")); + state.application = { + id: "active", + extensions: ["md"], + next, + steps: [ + { + documentId: "a", + from: "a.md", + to: "b.md", + staged: prefix + ".source", + output: prefix + ".output", + phase: "staged" + } + ] + }; + const store = new MemoryPersistence({ + settings: { + remoteUri: "http://offline.test", + vaultName: "test", + isSyncEnabled: false + }, + database: state + }); + const client = await SyncClient.create({ fs: disk, persistence: store }); + try { + await client.start(); + assert.deepEqual( + (await disk.readSnapshot("b.md"))?.content, + bytes("ONLY COPY") + ); + } finally { + await client.destroy(); + } +}); + +test("a settings update resumes the enabled engine", async () => { + const f = await settingsLifecycleFixture(); + try { + await f.client.setSettings({ maxFileSizeMB: 20 }); + assert.equal(f.client.getSettings().maxFileSizeMB, 20); + assert.equal(f.store.snapshot().settings?.maxFileSizeMB, 20); + const before = f.reads(); + await f.client.syncLocallyUpdatedFile({ relativePath: "absent.md" }); + await f.client.waitUntilFinished(); + assert( + f.reads() > before, + "notifications must still wake the enabled engine" + ); + } finally { + await f.client.destroy(); + } +}); + +test("a dirty cursor update must invalidate old precise positions", async () => { + const { CursorTracker } = await import( + "../sync-client/src/sync-operations/cursor-tracker" + ); + const { FileChangeNotifier } = await import( + "../sync-client/src/sync-operations/file-change-notifier" + ); + const f = await fixture({ "a.md": "base" }); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + const ws = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteCursorsUpdateReceived: new EventListeners(), + updateLocalCursors: () => {} + }; + const tracker = new CursorTracker( + f.database, + ws as unknown as WebSocketManager, + f.files, + new FileChangeNotifier() + ); + let latest: import("../sync-client/src/types/maybe-outdated-client-cursors").MaybeOutdatedClientCursors[] = + []; + tracker.onRemoteCursorsUpdated.add((cursors) => { + latest = cursors; + }); + const clean = { + userName: "other", + deviceId: "other", + documentsWithCursors: [ + { + document_id: "a", + relative_path: "a.md", + vault_update_id: 1, + cursors: [{ start: 0, end: 0 }] + } + ] + }; + await ws.onRemoteCursorsUpdateReceived.triggerAsync([clean]); + await ws.onRemoteCursorsUpdateReceived.triggerAsync([ + { + ...clean, + documentsWithCursors: [ + { + ...clean.documentsWithCursors[0], + vault_update_id: null, + cursors: [{ start: 100, end: 100 }] + } + ] + } + ]); + assert.equal(latest[0]?.isOutdated, true); + tracker.reset(); +}); + +test("silent open WebSocket must eventually trigger catchup or reconnect", async (context) => { + context.mock.timers.enable({ apis: ["setTimeout", "setInterval"] }); + let opens = 0, + eventReads = 0; + class SilentWebSocket { + static OPEN = 1; + readyState = 1; + onopen: ((event: object) => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = + null; + onmessage: ((event: { data: string }) => void) | null = null; + onerror: ((event: object) => void) | null = null; + constructor(_url: string | URL) { + opens++; + queueMicrotask(() => this.onopen?.({})); + } + send(_message: string) {} + close() { + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + const store = new MemoryPersistence({ + settings: { + isSyncEnabled: true, + remoteUri: "http://silent.test", + vaultName: "test", + token: "test", + syncIntervalMs: 0 + } + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: store, + webSocket: SilentWebSocket as unknown as typeof WebSocket, + fetch: async (input) => { + const path = String(input); + if (path.endsWith("/ping")) + return Response.json({ + supportedApiVersion: 5, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (path.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + if (path.includes("/events-since")) { + eventReads++; + return Response.json({ headEventId: 0, events: [] }); + } + throw new Error("Unexpected request " + path); + } + }); + try { + await client.start(); + const before = eventReads; + context.mock.timers.tick(45_000); + context.mock.timers.tick(3500); + for (let i = 0; i < 30; i++) await Promise.resolve(); + assert(opens > 1 || eventReads > before); + } finally { + await client.destroy(); + } +}); + +test("oversized files should be filtered before reading complete snapshots", async () => { + const f = await fixture({}); + await f.disk.userWrite("huge.bin", bytes("too big")); + await f.settings.setSettings({ maxFileSizeMB: 1 / (1024 * 1024) }); + let reads = 0; + f.disk.boundary = (label) => { + if (label === "read:snapshot:huge.bin") reads++; + }; + await f.internals.scan(); + assert.equal(reads, 0); +}); + +test("permanent destination failure replans the affected file and finishes the batch", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + f.disk.boundary = (label) => { + if (label === "before:mkdir:unsupported") + throw Object.assign(new Error("filesystem path limit"), { + code: "ENAMETOOLONG" + }); + }; + const next = structuredClone(f.database.state); + next.local = { a: "unsupported/a.md", b: "good.md" }; + next.fileManifest = { fileManifestId: 2, entries: { ...next.local } }; + await f.files.apply(next); + await f.settings.setSettings({ ignorePatterns: ["unsupported/**"] }); + await f.files.recover(); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((b) => Buffer.from(b).toString()) + .sort(), + ["A", "B"] + ); +}); + +test("conflict allocation terminates when a truncated extension ends in a space", async () => { + const { spawnSync } = await import("node:child_process"); + const modulePath = require.resolve( + "../sync-client/src/utils/portable-path" + ); + const result = spawnSync( + process.execPath, + [ + "--require", + require.resolve("tsx/cjs"), + "-e", + ` + const {allocatePortablePath,validatePortablePaths} = require(${JSON.stringify(modulePath)}); + const name = 'a.' + 'x'.repeat(62) + ' ' + 'x'.repeat(100); + const allocated = allocatePortablePath(name, 'new', {old:name}); + validatePortablePaths([name, allocated]); + if (Buffer.byteLength(allocated) > 255) process.exit(2); + ` + ], + { timeout: 3000, encoding: "utf8" } + ); + assert.equal(result.status, 0, String(result.error ?? result.stderr)); +}); + +test("paged catchup folds remote edits and reversions before touching local bytes", async () => { + const f = await fixture( + { "a.md": "base LOCAL" }, + { getDocumentVersionContent: async () => bytes("base") } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 1, "base"); + f.database.state.lastSeenUpdateId = 1; + const first = { + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId: "remote-1", + type: "content" as const, + document: head("a", 2, "base REMOTE") + } + ] + }; + await f.internals.incorporateEventBatch(first); + assert.equal( + f.database.state.lastSeenUpdateId, + 1, + "partial history must not become an incorporated base" + ); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); + assert.equal( + await f.internals.preparePush(), + false, + "do not submit against a partially replayed history" + ); + const last = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "remote-2", + type: "content" as const, + document: head("a", 3, "base") + } + ] + }; + await f.internals.incorporateEventBatch(last); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); +}); + +test("an ignored occupant arriving after the scan keeps its path and privacy", async () => { + let f: Awaited>; + f = await fixture( + {}, + { + getDocumentVersionContent: async () => { + await f.disk.userWrite("private", bytes("SECRET")); + return bytes("remote"); + } + } + ); + await f.settings.setSettings({ ignorePatterns: ["private"] }); + f.database.state.remoteHeads.a = head("a", 2, "remote"); + await f.internals.incorporateFileManifest({ + fileManifestId: 3, + entries: { a: "private/note.md" } + }); + assert.deepEqual( + (await f.disk.readSnapshot("private"))?.content, + bytes("SECRET") + ); + f.internals.stopped = false; + await f.internals.preparePush(); + const { pending } = f.database.state; + assert( + pending?.type !== "content" || + Buffer.from(pending.snapshot.contentBase64, "base64").toString() !== + "SECRET" + ); +}); + +test("content notifications during every snapshot do not starve a complete namespace scan", async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let notifications = 0; + f.disk.boundary = async (label) => { + if ( + label.startsWith("read:snapshot:") && + !label.includes(".vault-link-sync") + ) { + notifications++; + await f.syncer.syncLocallyUpdatedFile({ + relativePath: label.slice("read:snapshot:".length) + }); + } + }; + await f.internals.scan(); + assert.equal(notifications, 2); + assert.deepEqual(f.database.state.local, { a: "a.md", b: "b.md" }); +}); + +for (const boundary of ["before:save", "durable:save"]) { + test(`receipt incorporation after paged replay survives ${boundary} and process replacement`, async () => { + const service: Partial = { + putFileContent: async () => { + throw new PermanentSyncError("HTTP 413"); + }, + getDocumentVersionContent: async ({ vaultUpdateId }) => + bytes( + vaultUpdateId === 1 + ? "base" + : vaultUpdateId === 2 + ? "base A" + : "base A REMOTE" + ) + }; + let f = await fixture({ "a.md": "base" }, service); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("base A")); + f.internals.stopped = false; + await f.internals.preparePush(); + const { requestId } = f.database.state.pending!.request; + await f.internals.finishPending(); + await f.disk.userWrite("a.md", bytes("base")); + await f.internals.incorporateEventBatch({ + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "base A") + } + ] + }); + const page = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "other", + type: "content" as const, + document: head("a", 3, "base A REMOTE") + } + ] + }; + f.persistence.boundary = (label) => { + if (label === boundary) throw new Error("injected crash"); + }; + await assert.rejects( + f.internals.incorporateEventBatch(page), + /injected crash/ + ); + f.persistence.boundary = () => {}; + f.disk.crash(true); + f = await fixture({}, service, f); + await f.files.recover(); + assert.equal(f.database.state.eventReplay?.after, 2); + await f.internals.incorporateEventBatch(page); + assert.equal(f.database.state.eventReplay, undefined); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base REMOTE") + ); + }); +} + +test("initial vault settings bind the engine and survive restart", async () => { + const disk = new MemoryDisk(); + const persistence = new MemoryPersistence({ + settings: { + remoteUri: "http://test", + vaultName: "old", + isSyncEnabled: false + } + }); + class Socket { + static OPEN = 1; + readyState = 3; + onopen = null; + onclose = null; + onerror = null; + onmessage = null; + send() {} + close() {} + } + const options = { + fs: disk, + persistence, + webSocket: Socket as unknown as typeof WebSocket, + fetch: async (input: RequestInfo | URL) => { + if (String(input).endsWith("/ping")) + return Response.json({ + supportedApiVersion: 5, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (String(input).endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + return Response.json({ headEventId: 0, events: [] }); + } + }; + let client = await SyncClient.create(options); + try { + await client.setSettings({ vaultName: "new", isSyncEnabled: true }); + await client.start(); + await client.destroy(); + client = await SyncClient.create(options); + assert.equal(client.getSettings().vaultName, "new"); + await client.start(); + } finally { + await client.destroy(); + } +}); + +test("disabling sync persists across reset without resuming the transport", async () => { + const f = await settingsLifecycleFixture(); + try { + const before = f.reads(); + await f.client.setSettings({ isSyncEnabled: false }); + assert.equal(f.store.snapshot().settings?.isSyncEnabled, false); + await f.client.reset(); + assert.equal(f.client.getSettings().isSyncEnabled, false); + assert.equal(f.reads(), before); + } finally { + await f.client.destroy(); + } +}); + +for (const receipt of [false, true]) { + test(`${receipt ? "replayed" : "direct"} upload acknowledgement survives a local deletion`, async () => { + const f = await fixture( + { "a.md": "base" }, + { + putFileContent: async () => { + if (receipt) + throw new PermanentSyncError( + "HTTP 413 after lost success" + ); + return { type: "Accepted", ...head("a", 2, "edited") }; + } + } + ); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: f.database.state.documents.a.observedHash! + }; + f.database.state.lastSeenUpdateId = 1; + await f.disk.userWrite("a.md", bytes("edited")); + f.internals.stopped = false; + await f.internals.preparePush(); + const { requestId } = f.database.state.pending!.request; + if (receipt) await f.internals.finishPending(); + // Recovery folds queued deletions into the durable namespace before + // an outstanding request or its delayed receipt is handled. + f.internals.stopped = true; + await f.disk.userDelete("a.md"); + await f.syncer.syncLocallyDeletedFile("a.md"); + await f.internals.scan(); + assert.deepEqual(f.database.state.local, {}); + if (receipt) { + await f.internals.incorporateEventBatch({ + headEventId: 2, + events: [ + { + eventId: 2, + requestId, + type: "content", + document: head("a", 2, "edited") + } + ] + }); + assert.equal(f.database.state.lastSeenUpdateId, 2); + assert.equal(f.database.state.unconfirmed?.length ?? 0, 0); + } else await f.internals.finishPending(); + assert.equal(f.database.state.pending, undefined); + assert.deepEqual(f.database.state.local, {}); + assert.equal(await f.disk.readSnapshot("a.md"), undefined); + assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); + }); +} + +async function settingsLifecycleFixture() { + let eventReads = 0; + let onClose = () => {}; + class Socket { + static OPEN = 1; + readyState = 1; + onopen: (() => void) | null = null; + onclose: ((event: { code: number; reason: string }) => void) | null = + null; + constructor() { + queueMicrotask(() => this.onopen?.()); + } + send() {} + close() { + onClose(); + this.readyState = 3; + this.onclose?.({ code: 1000, reason: "" }); + } + } + const store = new MemoryPersistence({ + settings: { + remoteUri: "http://test", + vaultName: "test", + isSyncEnabled: true + } + }); + const client = await SyncClient.create({ + fs: new MemoryDisk(), + persistence: store, + webSocket: Socket as unknown as typeof WebSocket, + fetch: async (input) => { + const path = String(input); + if (path.endsWith("/ping")) + return Response.json({ + supportedApiVersion: 5, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" + }); + if (path.endsWith("/vault-snapshot")) + return Response.json({ + headEventId: 0, + fileManifest: { fileManifestId: 0, entries: {} }, + documents: [] + }); + if (path.includes("/events-since")) { + eventReads++; + return Response.json({ headEventId: 0, events: [] }); + } + throw new Error("Unexpected request " + path); + } + }); + await client.start(); + return { + client, + store, + reads: () => eventReads, + onClose: (fn: () => void) => { + onClose = fn; + } + }; +} + +for (const boundary of ["before:save", "durable:save"]) { + test(`settings recovery resumes after the rejection-reset save fails at ${boundary}`, async () => { + const f = await settingsLifecycleFixture(); + try { + const { database } = f.client as unknown as { database: Database }; + await database.commit({ + ...database.state, + rejectedManifest: { + entries: { retired: "old.md" }, + message: "retired rejection" + } + }); + let saves = 0, + failed = false; + f.store.boundary = (label) => { + if (label === "before:save") saves++; + if (saves === 2 && label === boundary && !failed) { + failed = true; + throw new Error("rejection-reset save failed"); + } + }; + await assert.rejects( + f.client.setSettings({ maxFileSizeMB: 20 }), + /rejection-reset save failed/ + ); + assert(failed); + assert.equal(f.client.getSettings().maxFileSizeMB, 20); + const before = f.reads(); + await f.client.syncLocallyUpdatedFile({ + relativePath: "absent.md" + }); + await f.client.waitUntilFinished(); + assert( + f.reads() > before, + "the enabled engine must resume after any settings-phase save failure" + ); + } finally { + await f.client.destroy(); + } + }); +} + +test("a caller cannot mutate settings while transports are being paused", async () => { + const f = await settingsLifecycleFixture(); + const change = { + maxFileSizeMB: 20, + vaultName: "test", + ignorePatterns: ["private/**"] + }; + try { + f.onClose(() => { + change.vaultName = "wrong-vault"; + change.ignorePatterns.push("**"); + }); + await f.client.setSettings(change); + assert.equal(f.client.getSettings().vaultName, "test"); + assert.deepEqual(f.client.getSettings().ignorePatterns, ["private/**"]); + assert.equal( + f.store.snapshot().database?.vaultKey, + JSON.stringify(["http://test", "test"]) + ); + } finally { + await f.client.destroy(); + } +}); diff --git a/frontend/test-support/sync-fixture.ts b/frontend/test-support/sync-fixture.ts new file mode 100644 index 00000000..da54142a --- /dev/null +++ b/frontend/test-support/sync-fixture.ts @@ -0,0 +1,136 @@ +import { + Database, + emptyState, + type StoredDatabase +} from "../sync-client/src/persistence/database"; +import { Settings } from "../sync-client/src/persistence/settings"; +import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Syncer } from "../sync-client/src/sync-operations/syncer"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { Logger } from "../sync-client/src/tracing/logger"; +import { SyncHistory } from "../sync-client/src/tracing/sync-history"; +import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; +import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; +import { globsToRegexes } from "../sync-client/src/utils/globs-to-regexes"; +import type { ServerConfig } from "../sync-client/src/services/server-config"; +import type { SyncService } from "../sync-client/src/services/sync-service"; +import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +import type { LocalChange } from "../sync-client/src/sync-operations/local-changes"; + +export const bytes = (text: string) => new TextEncoder().encode(text); +export const head = (id: string, version: number, content: string) => ({ + documentId: id, + vaultUpdateId: version, + contentSize: bytes(content).length, + updatedDate: "2026-09-19T00:00:00Z", + userId: "user", + deviceId: "device" +}); + +export async function fixture( + contents: Record = { "a.md": "local" }, + service: Partial = {}, + recovered?: { disk: MemoryDisk; persistence: MemoryPersistence } +) { + const disk = recovered?.disk ?? new MemoryDisk(); + const initial = recovered + ? (recovered.persistence.snapshot().database as StoredDatabase) + : emptyState("test"); + if (!recovered) { + initial.initialized = true; + for (const [path, content] of Object.entries(contents)) { + await disk.userWrite(path, bytes(content)); + const id = path.split(".")[0]; + initial.local[id] = path; + initial.documents[id] = { + materialized: true, + observedHash: ( + await toStoredSnapshot({ content: bytes(content) }) + ).hash + }; + } + initial.fileManifest = { + fileManifestId: 1, + entries: { ...initial.local } + }; + } + const persistence = + recovered?.persistence ?? new MemoryPersistence({ database: initial }); + const logger = new Logger(); + const database = new Database( + logger, + initial, + async (next) => persistence.save({ database: next }), + "test", + async () => (await persistence.load()).database as StoredDatabase + ); + const config = { + initialize: async () => {}, + getConfig: async () => ({ mergeableFileExtensions: ["md"] }), + reset: () => {} + } as unknown as ServerConfig; + const settings = new Settings( + logger, + { syncIntervalMs: 0 }, + async () => {} + ); + const changes: LocalChange[] = []; + const files = new FileOperations( + disk.session(), + database, + config, + undefined, + { entries: () => changes, flush: async () => {} }, + (path, size) => + size > settings.getSettings().maxFileSizeMB * 1024 * 1024 || + globsToRegexes(settings.getSettings().ignorePatterns, logger).some( + (pattern) => pattern.test(path) + ) + ); + const websocket = { + onWebSocketStatusChanged: new EventListeners(), + onRemoteVaultUpdateReceived: new EventListeners() + } as unknown as WebSocketManager; + const contentCache = new FixedSizeDocumentCache(1024); + const syncer = new Syncer( + "device", + logger, + database, + settings, + service as SyncService, + websocket, + files, + config, + new SyncHistory(logger), + contentCache, + { entries: changes, save: async () => {} } + ); + const internals = syncer as unknown as { + finishPending: () => Promise; + preparePush: () => Promise; + scan: ( + initial?: import("../sync-client/src/services/types/VaultSnapshot").VaultSnapshot + ) => Promise; + incorporateEventBatch: ( + batch: import("../sync-client/src/services/types/EventBatch").EventBatch + ) => Promise; + stopped: boolean; + incorporateFileManifest: (remote: { + fileManifestId: number; + entries: Record; + }) => Promise; + incorporateContent: (remote: ReturnType) => Promise; + }; + return { + disk, + database, + files, + settings, + syncer, + internals, + persistence, + changes, + contentCache + }; +} diff --git a/frontend/test-support/sync-server-fixture.ts b/frontend/test-support/sync-server-fixture.ts new file mode 100644 index 00000000..8e0d438f --- /dev/null +++ b/frontend/test-support/sync-server-fixture.ts @@ -0,0 +1,94 @@ +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { randomUUID } from "node:crypto"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { Logger } from "sync-client"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; +const root = resolve(__dirname, "../.."); +const token = "test-token-change-me"; +export const bytes = (s: string) => new TextEncoder().encode(s); +const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "Device-Id": "audit" +}; +export async function fixture() { + const server = new ServerControl( + root + "/sync-server/target/release/sync_server", + root + "/sync-server/config-e2e.yml", + new Logger() + ); + await server.start(); + const vault = randomUUID(), + url = server.remoteUri + "/vaults/" + vault; + const put = async (path: string, data: unknown) => { + const r = await fetch(url + path, { + method: "PUT", + headers, + body: JSON.stringify(data) + }); + assert(r.ok, await r.clone().text()); + return r.json(); + }; + const get = async (path: string) => { + const r = await fetch(url + path, { headers }); + assert(r.ok); + return r.json(); + }; + const disk = new MemoryDisk(); + const store = new MemoryPersistence({ + settings: { + remoteUri: server.remoteUri, + vaultName: vault, + token, + isSyncEnabled: true, + syncIntervalMs: 0, + networkRetryIntervalMs: 0, + requestTimeoutMs: 5000 + } + }); + let fetchHook: typeof fetch | undefined; + const createClient = () => + SyncClient.create({ + fs: disk, + persistence: store, + fetch: (...args) => + fetchHook ? fetchHook(...args) : fetch(...args) + }); + let client = await createClient(); + const content = (text: string, parentVersionId: number | null = null) => ({ + requestId: randomUUID(), + parentVersionId, + content: { + type: "Snapshot", + value: Buffer.from(text).toString("base64") + } + }); + return { + server, + disk, + store, + get client() { + return client; + }, + reopenClient: async () => { + await client.destroy(); + client = await createClient(); + }, + url, + headers, + put, + get, + content, + setFetchHook: (hook: typeof fetch) => { + fetchHook = hook; + }, + dispose: async () => { + await client.destroy(); + await server.stop(); + } + }; +} + +export { randomUUID }; diff --git a/frontend/test-support/tsconfig.json b/frontend/test-support/tsconfig.json new file mode 100644 index 00000000..42eb5312 --- /dev/null +++ b/frontend/test-support/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "skipLibCheck": true, + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "node", + "esModuleInterop": true, + "types": ["node"], + "lib": ["DOM", "ES2024"] + }, + "include": ["*.ts"] +} diff --git a/logs/harness-review/deterministic-results.json b/logs/harness-review/deterministic-results.json new file mode 100644 index 00000000..7ac2c0b3 --- /dev/null +++ b/logs/harness-review/deterministic-results.json @@ -0,0 +1,1121 @@ +[ + { + "name": "v4-binary-bytes-and-independent-identities", + "result": { + "success": true, + "duration": 100 + } + }, + { + "name": "v4-three-file-cycle-preserves-identities", + "result": { + "success": true, + "duration": 170 + } + }, + { + "name": "v4-case-alias-and-file-directory-conflicts", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "v4-create-before-commit-network-loss", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "v4-create-after-commit-network-loss", + "result": { + "success": true, + "duration": 64 + } + }, + { + "name": "v4-content-before-commit-network-loss", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "v4-content-after-commit-network-loss", + "result": { + "success": true, + "duration": 91 + } + }, + { + "name": "v4-manifest-before-commit-network-loss", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "v4-manifest-after-commit-network-loss", + "result": { + "success": true, + "duration": 72 + } + }, + { + "name": "rename-create-conflict", + "result": { + "success": true, + "duration": 121 + } + }, + { + "name": "rename-chain", + "result": { + "success": true, + "duration": 52 + } + }, + { + "name": "rename-update-conflict", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "delete-rename-conflict", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "multi-file-operations", + "result": { + "success": true, + "duration": 129 + } + }, + { + "name": "delete-recreate-same-path", + "result": { + "success": true, + "duration": 63 + } + }, + { + "name": "offline-rename-and-edit", + "result": { + "success": true, + "duration": 79 + } + }, + { + "name": "simultaneous-create-delete-same-path", + "result": { + "success": true, + "duration": 54 + } + }, + { + "name": "sequential-create-duplicate-content", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "mc-three-client-rename-offline-update", + "result": { + "success": true, + "duration": 102 + } + }, + { + "name": "mc-multi-delete-offline-rename", + "result": { + "success": true, + "duration": 115 + } + }, + { + "name": "mc-cross-create-rename-same-target", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "mc-delete-then-offline-rename", + "result": { + "success": true, + "duration": 79 + } + }, + { + "name": "offline-mixed-operations", + "result": { + "success": true, + "duration": 117 + } + }, + { + "name": "offline-concurrent-renames", + "result": { + "success": true, + "duration": 76 + } + }, + { + "name": "offline-multiple-edits", + "result": { + "success": true, + "duration": 74 + } + }, + { + "name": "rename-swap", + "result": { + "success": true, + "duration": 93 + } + }, + { + "name": "rename-circular", + "result": { + "success": true, + "duration": 113 + } + }, + { + "name": "rename-roundtrip", + "result": { + "success": true, + "duration": 95 + } + }, + { + "name": "offline-rename-remote-create-old-path", + "result": { + "success": true, + "duration": 83 + } + }, + { + "name": "offline-edit-remote-rename", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "rename-chain-then-delete", + "result": { + "success": true, + "duration": 86 + } + }, + { + "name": "offline-delete-remote-rename", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "overlapping-edits-same-section", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "rapid-updates-after-merge", + "result": { + "success": true, + "duration": 117 + } + }, + { + "name": "delete-recreate-concurrent-update", + "result": { + "success": true, + "duration": 90 + } + }, + { + "name": "move-and-concurrent-remote-update", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "double-offline-cycle", + "result": { + "success": true, + "duration": 104 + } + }, + { + "name": "offline-update-both-then-delete-one", + "result": { + "success": true, + "duration": 100 + } + }, + { + "name": "offline-create-same-path-mergeable", + "result": { + "success": true, + "duration": 68 + } + }, + { + "name": "three-client-rename-create-delete", + "result": { + "success": true, + "duration": 106 + } + }, + { + "name": "rename-to-path-of-unconfirmed-delete", + "result": { + "success": true, + "duration": 94 + } + }, + { + "name": "offline-edit-then-move-same-content", + "result": { + "success": true, + "duration": 92 + } + }, + { + "name": "delete-recreate-different-content", + "result": { + "success": true, + "duration": 102 + } + }, + { + "name": "offline-move-then-remote-delete", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "reset-clears-recently-deleted-resurrection", + "result": { + "success": true, + "duration": 75 + } + }, + { + "name": "move-then-delete-stale-path", + "result": { + "success": true, + "duration": 65 + } + }, + { + "name": "offline-delete-vs-remote-update", + "result": { + "success": true, + "duration": 89 + } + }, + { + "name": "update-does-not-survive-remote-delete", + "result": { + "success": true, + "duration": 64 + } + }, + { + "name": "move-preserves-remote-update", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "recently-deleted-cleared-on-reconnect", + "result": { + "success": true, + "duration": 107 + } + }, + { + "name": "watermark-advances-on-skip", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "watermark-gap-remote-update-not-recorded", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "queue-reset-loses-coalesced-local-edit", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "rename-to-pending-path-fallback", + "result": { + "success": true, + "duration": 55 + } + }, + { + "name": "move-remote-update-reverts-rename", + "result": { + "success": true, + "duration": 99 + } + }, + { + "name": "local-edit-lost-during-create-merge", + "result": { + "success": true, + "duration": 52 + } + }, + { + "name": "create-rename-response-skips-file", + "result": { + "success": true, + "duration": 73 + } + }, + { + "name": "online-create-rename-concurrent-create-orphan", + "result": { + "success": true, + "duration": 69 + } + }, + { + "name": "concurrent-rename-first-wins", + "result": { + "success": true, + "duration": 90 + } + }, + { + "name": "binary-to-text-transition", + "result": { + "success": true, + "duration": 151 + } + }, + { + "name": "text-pending-create-not-displaced", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "binary-pending-create-not-displaced", + "result": { + "success": true, + "duration": 80 + } + }, + { + "name": "coalesce-update-remote-update-data-loss", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "coalesced-remote-update-watermark-loss", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "concurrent-delete-during-remote-update", + "result": { + "success": true, + "duration": 66 + } + }, + { + "name": "concurrent-edit-exact-same-position", + "result": { + "success": true, + "duration": 97 + } + }, + { + "name": "concurrent-rename-and-create-at-target-rename-first", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "concurrent-rename-and-create-at-target-create-first", + "result": { + "success": true, + "duration": 109 + } + }, + { + "name": "concurrent-rename-same-target", + "result": { + "success": true, + "duration": 110 + } + }, + { + "name": "concurrent-update-diff-consistency", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "user-parenthesized-file-not-deleted", + "result": { + "success": true, + "duration": 116 + } + }, + { + "name": "create-delete-noop", + "result": { + "success": true, + "duration": 30 + } + }, + { + "name": "create-merge-delete", + "result": { + "success": true, + "duration": 73 + } + }, + { + "name": "move-identical-content-ambiguity", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "create-during-reconciliation", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "create-merge-preserves-renamed-update", + "result": { + "success": true, + "duration": 106 + } + }, + { + "name": "create-rename-create-same-path", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "move-chain-three-files", + "result": { + "success": true, + "duration": 116 + } + }, + { + "name": "delete-by-other-client-then-recreate", + "result": { + "success": true, + "duration": 81 + } + }, + { + "name": "online-delete-recreate-rapid-cycle", + "result": { + "success": true, + "duration": 169 + } + }, + { + "name": "online-edit-vs-delete-convergence", + "result": { + "success": true, + "duration": 84 + } + }, + { + "name": "rapid-edit-delete-online-convergence", + "result": { + "success": true, + "duration": 144 + } + }, + { + "name": "online-both-create-same-path-deconflict", + "result": { + "success": true, + "duration": 92 + } + }, + { + "name": "online-create-update-while-other-creates-same-path", + "result": { + "success": true, + "duration": 82 + } + }, + { + "name": "displaced-file-not-marked-deleted", + "result": { + "success": true, + "duration": 112 + } + }, + { + "name": "remote-update-resurrects-deleted-doc", + "result": { + "success": false, + "error": "Convergence timed out after 60000ms: Client 1 differs from client 0 (paths or bytes)\n+ actual - expected\n\n+ [\n+ [\n+ 'P.md',\n+ Uint8Array(17) [\n+ 118,\n+ 50,\n+ 49,\n+ 32,\n+ 99,\n+ 111,\n+ 110,\n+ 116,\n+ 101,\n+ 110,\n+ 116,\n+ 32,\n+ 40,\n+ 68,\n+ 50,\n+ 41,\n+ 10\n+ ]\n+ ]\n+ ]\n- []\n", + "diagnostics": [ + { + "client": 0, + "database": { + "vaultKey": "[\"http://localhost:62272\",\"test-df2b81ab-03e9-432a-878a-d6aa1c6c9147\"]", + "initialized": true, + "fileManifest": { + "fileManifestId": 4, + "entries": {} + }, + "local": {}, + "documents": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "materialized": true, + "base": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40, + "hash": "229b36522c531631df2139e070b94a0e1b8d8833ae49d4662cb0d7a8bd158e9b" + }, + "bootstrap": false, + "observedHash": "229b36522c531631df2139e070b94a0e1b8d8833ae49d4662cb0d7a8bd158e9b" + } + }, + "remoteHeads": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40 + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17 + } + }, + "lastSeenUpdateId": 5 + }, + "disk": { + "visible": [ + [ + "", + null + ], + [ + ".vault-link-sync", + null + ], + [ + ".vault-link-sync/transactions", + null + ] + ], + "durable": [ + [ + "", + null + ], + [ + ".vault-link-sync", + null + ], + [ + ".vault-link-sync/transactions", + null + ] + ] + }, + "requests": [ + { + "kind": "content", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "d970973e-d042-4383-9edd-6be699df0c0f", + "parentVersionId": 3, + "content": { + "type": "Diff", + "value": [ + -11, + "v17local edit by client 0", + 15 + ] + } + }, + "status": 200 + } + ] + }, + { + "client": 1, + "database": { + "vaultKey": "[\"http://localhost:62272\",\"test-df2b81ab-03e9-432a-878a-d6aa1c6c9147\"]", + "initialized": true, + "fileManifest": { + "fileManifestId": 6, + "entries": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + } + }, + "local": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + }, + "documents": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "materialized": true, + "bootstrap": false, + "observedHash": "106554e74ef5b4dc8c8668c00ff9e822ebb2573232e6af3c71849bf200685e26", + "base": { + "vaultUpdateId": 3, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.876666Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 26, + "hash": "106554e74ef5b4dc8c8668c00ff9e822ebb2573232e6af3c71849bf200685e26" + } + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "materialized": true, + "bootstrap": false, + "observedHash": "8dd4b135f5ee76639d62ae49fbfc7404962fc5a67f8f60ced508d5cb8ce5f4df", + "base": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17, + "hash": "8dd4b135f5ee76639d62ae49fbfc7404962fc5a67f8f60ced508d5cb8ce5f4df" + } + } + }, + "remoteHeads": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": { + "vaultUpdateId": 7, + "documentId": "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "updatedDate": "2026-09-19T21:30:07.925900Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (f64bfac0-58b8-4ce3-8a1d-40f1f2df7011; MacIntel)", + "contentSize": 40 + }, + "c6316ddf-539d-4604-8039-5ff6022efef2": { + "vaultUpdateId": 5, + "documentId": "c6316ddf-539d-4604-8039-5ff6022efef2", + "updatedDate": "2026-09-19T21:30:07.889537Z", + "userId": "admin", + "deviceId": "vault-link/0.14.0 (29416abc-2c3c-430c-bd21-f0c5b1aacdac; MacIntel)", + "contentSize": 17 + } + }, + "lastSeenUpdateId": 7 + }, + "disk": { + "visible": [ + [ + "", + null + ], + [ + "P.md", + { + "content": { + "0": 118, + "1": 50, + "2": 49, + "3": 32, + "4": 99, + "5": 111, + "6": 110, + "7": 116, + "8": 101, + "9": 110, + "10": 116, + "11": 32, + "12": 40, + "13": 68, + "14": 50, + "15": 41, + "16": 10 + } + } + ] + ], + "durable": [ + [ + "", + null + ], + [ + "P.md", + { + "content": { + "0": 118, + "1": 50, + "2": 49, + "3": 32, + "4": 99, + "5": 111, + "6": 110, + "7": 116, + "8": 101, + "9": 110, + "10": 116, + "11": 32, + "12": 40, + "13": 68, + "14": 50, + "15": 41, + "16": 10 + } + } + ] + ] + }, + "requests": [ + { + "kind": "create", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "f2abf8a1-8c5d-49d7-8ca8-4d4df56a2aff", + "parentVersionId": null, + "content": { + "type": "Snapshot", + "value": "djggY29udGVudAo=" + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "41f31b38-1edc-4f97-8519-6973e4a04f9f", + "parentFileManifestId": 0, + "entries": { + "1460ea0a-a39b-4a55-8772-02ffa0b0ffc6": "P.md" + } + }, + "status": 200 + }, + { + "kind": "content", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/1460ea0a-a39b-4a55-8772-02ffa0b0ffc6", + "body": { + "requestId": "4bd1dcf9-c5d2-42f4-aa96-f1b921362e55", + "parentVersionId": 1, + "content": { + "type": "Diff", + "value": [ + -2, + "v17", + 8, + " from client 1", + 1 + ] + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "3b0cc3a0-b288-47d0-998b-d608f885d4b0", + "parentFileManifestId": 2, + "entries": {} + }, + "status": 200 + }, + { + "kind": "create", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/documents/c6316ddf-539d-4604-8039-5ff6022efef2", + "body": { + "requestId": "edb11740-8c0c-4407-91d2-bd264e26281f", + "parentVersionId": null, + "content": { + "type": "Snapshot", + "value": "djIxIGNvbnRlbnQgKEQyKQo=" + } + }, + "status": 200 + }, + { + "kind": "manifest", + "method": "PUT", + "url": "http://localhost:62272/vaults/test-df2b81ab-03e9-432a-878a-d6aa1c6c9147/file-manifest", + "body": { + "requestId": "c9615094-e693-4b11-8a3d-5c415d3dcdd0", + "parentFileManifestId": 4, + "entries": { + "c6316ddf-539d-4604-8039-5ff6022efef2": "P.md" + } + }, + "status": 200 + } + ] + } + ], + "duration": 60283 + } + }, + { + "name": "local-update-survives-remote-rename", + "result": { + "success": true, + "duration": 58 + } + }, + { + "name": "catchup-create-and-update-not-skipped", + "result": { + "success": true, + "duration": 55 + } + }, + { + "name": "remote-rename-collides-with-pending-local-create", + "result": { + "success": true, + "duration": 53 + } + }, + { + "name": "remote-quick-write-rename-before-record", + "result": { + "success": true, + "duration": 42 + } + }, + { + "name": "v4-server-sigkill-retains-acknowledged-state", + "result": { + "success": true, + "duration": 220 + } + }, + { + "name": "idempotency-after-server-pause", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "server-pause-both-clients-create", + "result": { + "success": true, + "duration": 60 + } + }, + { + "name": "server-pause-update-and-create", + "result": { + "success": true, + "duration": 87 + } + }, + { + "name": "server-pause-rename-edit-resume", + "result": { + "success": true, + "duration": 70 + } + }, + { + "name": "delete-during-pending-create", + "result": { + "success": true, + "duration": 43 + } + }, + { + "name": "rapid-create-update-delete-cycle", + "result": { + "success": true, + "duration": 51 + } + }, + { + "name": "server-pause-both-edit-same-file", + "result": { + "success": true, + "duration": 98 + } + }, + { + "name": "update-during-create-processing", + "result": { + "success": true, + "duration": 65 + } + }, + { + "name": "interrupted-delete-retry", + "result": { + "success": true, + "duration": 67 + } + }, + { + "name": "rename-pending-create-before-response", + "result": { + "success": true, + "duration": 78 + } + }, + { + "name": "create-update-coalesce-server-pause", + "result": { + "success": true, + "duration": 67 + } + }, + { + "name": "server-pause-delete-recreate", + "result": { + "success": true, + "duration": 71 + } + }, + { + "name": "merging-update-response-survives-user-rename", + "result": { + "success": true, + "duration": 88 + } + }, + { + "name": "local-rename-survives-remote-rename", + "result": { + "success": true, + "duration": 82 + } + }, + { + "name": "rename-chain-during-pending-create", + "result": { + "success": true, + "duration": 57 + } + }, + { + "name": "remote-update-survives-user-rename", + "result": { + "success": true, + "duration": 111 + } + }, + { + "name": "same-doc-id-collapse-on-local-create-after-remote-create", + "result": { + "success": true, + "duration": 54 + } + }, + { + "name": "renamed-pending-create-reused-path-then-delete", + "result": { + "success": true, + "duration": 185 + } + }, + { + "name": "rename-pending-create-onto-pending-delete-path", + "result": { + "success": true, + "duration": 166 + } + }, + { + "name": "rename-overwrites-pending-create-then-delete", + "result": { + "success": true, + "duration": 80 + } + }, + { + "name": "same-doc-id-collapse-after-remote-quick-write-and-pending-rename", + "result": { + "success": true, + "duration": 220 + } + }, + { + "name": "delete-recreated-pending-create-with-stale-deleting-record", + "result": { + "success": true, + "duration": 234 + } + }, + { + "name": "queued-create-delete-does-not-hijack-reused-path", + "result": { + "success": true, + "duration": 173 + } + }, + { + "name": "self-merge-pending-rename-aliases-second-create", + "result": { + "success": true, + "duration": 186 + } + } +] \ No newline at end of file diff --git a/logs/harness-review/result-42.json b/logs/harness-review/result-42.json new file mode 100644 index 00000000..fa5492ce --- /dev/null +++ b/logs/harness-review/result-42.json @@ -0,0 +1,4 @@ +{ + "success": true, + "duration": 262 +} \ No newline at end of file diff --git a/logs/harness-review/result-43.json b/logs/harness-review/result-43.json new file mode 100644 index 00000000..cbf0df26 --- /dev/null +++ b/logs/harness-review/result-43.json @@ -0,0 +1,4 @@ +{ + "success": true, + "duration": 273 +} \ No newline at end of file diff --git a/logs/harness-review/results.json b/logs/harness-review/results.json new file mode 100644 index 00000000..b14cc528 --- /dev/null +++ b/logs/harness-review/results.json @@ -0,0 +1,111 @@ +{ + "options": { + "workers": 2, + "seed": 42, + "iterations": 3, + "timeout": 180000 + }, + "results": [ + { + "name": "build-server", + "command": "cargo", + "args": [ + "build", + "--release" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "build-clients", + "command": "npm", + "args": [ + "run", + "build", + "--workspace", + "sync-client", + "--workspace", + "deterministic-tests", + "--workspace", + "test-client" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "harness-self-tests", + "command": "npm", + "args": [ + "run", + "test:harness" + ], + "code": 1, + "signal": null, + "timedOut": false, + "success": false + }, + { + "name": "crash-and-protocol", + "command": "npm", + "args": [ + "run", + "test:protocol" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "deterministic", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/deterministic-tests/dist/cli.js", + "--concurrency", + "2" + ], + "code": 1, + "signal": null, + "timedOut": false, + "success": false + }, + { + "name": "fuzz-42", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/test-client/dist/cli.js", + "--seed", + "42", + "--iterations", + "3", + "--artifacts", + "/Users/schmelczer/Projects/vault-link/logs/harness-review" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + }, + { + "name": "fuzz-43", + "command": "/Users/schmelczer/.nvm/versions/node/v25.2.1/bin/node", + "args": [ + "/Users/schmelczer/Projects/vault-link/frontend/test-client/dist/cli.js", + "--seed", + "43", + "--iterations", + "3", + "--artifacts", + "/Users/schmelczer/Projects/vault-link/logs/harness-review" + ], + "code": 0, + "signal": null, + "timedOut": false, + "success": true + } + ] +} \ No newline at end of file diff --git a/logs/harness-review/trace-42.json b/logs/harness-review/trace-42.json new file mode 100644 index 00000000..c27de220 --- /dev/null +++ b/logs/harness-review/trace-42.json @@ -0,0 +1,368 @@ +{ + "version": 1, + "seed": 42, + "iterations": 3, + "steps": [ + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-0/café/a.md", + "content": "[seed-42-round-0-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-0/café/b.md", + "content": "[seed-42-round-0-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-0/café/a.md", + "key": "round-0/café/a.md" + }, + { + "type": "remember-identity", + "path": "round-0/café/b.md", + "key": "round-0/café/b.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/a.md", + "newPath": "round-0/café/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/c.md", + "newPath": "round-0/café/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/café/a.md", + "newPath": "round-0/café/c.md" + }, + { + "type": "update", + "client": 0, + "path": "round-0/café/c.md", + "content": "[seed-42-round-0-A]\nrapid final edit" + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-0/café/c.md", + "key": "round-0/café/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-0/café/c.md": "[seed-42-round-0-A]\nrapid final edit", + "round-0/café/b.md": "[seed-42-round-0-B]" + }, + "absent": [ + "round-0/café/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-1/日本語/a.md", + "content": "[seed-42-round-1-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-1/日本語/b.md", + "content": "[seed-42-round-1-B]" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-1/日本語/a.md", + "key": "round-1/日本語/a.md" + }, + { + "type": "remember-identity", + "path": "round-1/日本語/b.md", + "key": "round-1/日本語/b.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/a.md", + "newPath": "round-1/日本語/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/c.md", + "newPath": "round-1/日本語/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/日本語/a.md", + "newPath": "round-1/日本語/c.md" + }, + { + "type": "update", + "client": 0, + "path": "round-1/日本語/c.md", + "content": "[seed-42-round-1-A]\nrapid final edit" + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-1/日本語/c.md", + "key": "round-1/日本語/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-1/日本語/c.md": "[seed-42-round-1-A]\nrapid final edit", + "round-1/日本語/b.md": "[seed-42-round-1-B]" + }, + "absent": [ + "round-1/日本語/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]", + "[seed-42-round-1-A]", + "[seed-42-round-1-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-2/café/a.md", + "content": "[seed-42-round-2-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-2/café/b.md", + "content": "[seed-42-round-2-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-2/café/a.md", + "key": "round-2/café/a.md" + }, + { + "type": "remember-identity", + "path": "round-2/café/b.md", + "key": "round-2/café/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/a.md", + "newPath": "round-2/café/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/b.md", + "newPath": "round-2/café/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/café/c.md", + "newPath": "round-2/café/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-2/café/b.md", + "content": "[seed-42-round-2-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-2/café/b.md", + "key": "round-2/café/a.md" + }, + { + "type": "assert-identity", + "path": "round-2/café/a.md", + "key": "round-2/café/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-2/café/a.md": "[seed-42-round-2-B]", + "round-2/café/b.md": "[seed-42-round-2-A]\nextra line\n" + }, + "absent": [ + "round-2/café/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-42-round-0-A]", + "[seed-42-round-0-B]", + "[seed-42-round-1-A]", + "[seed-42-round-1-B]", + "[seed-42-round-2-A]", + "[seed-42-round-2-B]" + ], + "removed": [] + } + ] +} \ No newline at end of file diff --git a/logs/harness-review/trace-43.json b/logs/harness-review/trace-43.json new file mode 100644 index 00000000..4d764671 --- /dev/null +++ b/logs/harness-review/trace-43.json @@ -0,0 +1,404 @@ +{ + "version": 1, + "seed": 43, + "iterations": 3, + "steps": [ + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-0/space folder/a.md", + "content": "[seed-43-round-0-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-0/space folder/b.md", + "content": "[seed-43-round-0-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "drop-response", + "client": 0, + "kind": "create", + "point": "after" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-0/space folder/a.md", + "key": "round-0/space folder/a.md" + }, + { + "type": "remember-identity", + "path": "round-0/space folder/b.md", + "key": "round-0/space folder/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-0/space folder/a.md", + "newPath": "round-0/space folder/c.md" + }, + { + "type": "update", + "client": 1, + "path": "round-0/space folder/a.md", + "content": "[seed-43-round-0-A]\nremote addition\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-0/space folder/c.md", + "key": "round-0/space folder/a.md" + }, + { + "type": "assert-files", + "expected": { + "round-0/space folder/c.md": "[seed-43-round-0-A]\nremote addition\n", + "round-0/space folder/b.md": "[seed-43-round-0-B]" + }, + "absent": [ + "round-0/space folder/a.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]" + ], + "removed": [] + }, + { + "type": "reset", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-1/space folder/a.md", + "content": "[seed-43-round-1-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-1/space folder/b.md", + "content": "[seed-43-round-1-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-1/space folder/a.md", + "key": "round-1/space folder/a.md" + }, + { + "type": "remember-identity", + "path": "round-1/space folder/b.md", + "key": "round-1/space folder/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/a.md", + "newPath": "round-1/space folder/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/b.md", + "newPath": "round-1/space folder/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-1/space folder/c.md", + "newPath": "round-1/space folder/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-1/space folder/b.md", + "content": "[seed-43-round-1-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-1/space folder/b.md", + "key": "round-1/space folder/a.md" + }, + { + "type": "assert-identity", + "path": "round-1/space folder/a.md", + "key": "round-1/space folder/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-1/space folder/a.md": "[seed-43-round-1-B]", + "round-1/space folder/b.md": "[seed-43-round-1-A]\nextra line\n" + }, + "absent": [ + "round-1/space folder/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]", + "[seed-43-round-1-A]", + "[seed-43-round-1-B]" + ], + "removed": [] + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "disable-sync", + "client": 1 + }, + { + "type": "delay-notifications", + "client": 0 + }, + { + "type": "delay-notifications", + "client": 1 + }, + { + "type": "create", + "client": 0, + "path": "round-2/nested/a.md", + "content": "[seed-43-round-2-A]" + }, + { + "type": "create", + "client": 0, + "path": "round-2/nested/b.md", + "content": "[seed-43-round-2-B]" + }, + { + "type": "flush-notifications", + "client": 0 + }, + { + "type": "flush-notifications", + "client": 1 + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "sync", + "client": 0 + }, + { + "type": "enable-sync", + "client": 1 + }, + { + "type": "barrier" + }, + { + "type": "remember-identity", + "path": "round-2/nested/a.md", + "key": "round-2/nested/a.md" + }, + { + "type": "remember-identity", + "path": "round-2/nested/b.md", + "key": "round-2/nested/b.md" + }, + { + "type": "disable-sync", + "client": 0 + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/a.md", + "newPath": "round-2/nested/c.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/b.md", + "newPath": "round-2/nested/a.md" + }, + { + "type": "rename", + "client": 0, + "oldPath": "round-2/nested/c.md", + "newPath": "round-2/nested/b.md" + }, + { + "type": "update", + "client": 0, + "path": "round-2/nested/b.md", + "content": "[seed-43-round-2-A]\nextra line\n" + }, + { + "type": "enable-sync", + "client": 0 + }, + { + "type": "barrier" + }, + { + "type": "assert-identity", + "path": "round-2/nested/b.md", + "key": "round-2/nested/a.md" + }, + { + "type": "assert-identity", + "path": "round-2/nested/a.md", + "key": "round-2/nested/b.md" + }, + { + "type": "assert-files", + "expected": { + "round-2/nested/a.md": "[seed-43-round-2-B]", + "round-2/nested/b.md": "[seed-43-round-2-A]\nextra line\n" + }, + "absent": [ + "round-2/nested/c.md" + ] + }, + { + "type": "assert-markers", + "markers": [ + "[seed-43-round-0-A]", + "[seed-43-round-0-B]", + "[seed-43-round-1-A]", + "[seed-43-round-1-B]", + "[seed-43-round-2-A]", + "[seed-43-round-2-B]" + ], + "removed": [] + } + ] +} \ No newline at end of file diff --git a/sync-server/README.md b/sync-server/README.md index fc54337c..27e68c20 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -1,22 +1,23 @@ # Sync server -## API v4 +## API v5 The server is a compare-and-swap store. Content versions do not contain paths or deletions. A separately versioned file manifest owns the namespace and membership. All routes below are under `/vaults/:vault_id` and require bearer authentication (except the WebSocket, which authenticates its first message). -| Endpoint | Purpose | -| --- | --- | -| `PUT /documents/:document_id` | CAS an immutable content version. | -| `GET /documents/:document_id` | Current content version, including base64 bytes. | -| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. | -| `GET /file_manifest` | Current file manifest. | -| `PUT /file_manifest` | CAS the complete file manifest. | -| `GET /vault_snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. | -| `GET /events_since?after=N` | Every event after N, in order, with the current watermark. | -| `GET /ws` | Ordered event delivery and ephemeral cursor positions. | +| Endpoint | Purpose | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| `PUT /documents/:document_id` | CAS an immutable content version. | +| `GET /documents/:document_id/metadata` | Current version metadata and size without content bytes. | +| `GET /documents/:document_id` | Current content version, including base64 bytes. | +| `GET /documents/:document_id/versions/:version_id/content` | Immutable raw bytes. | +| `GET /file-manifest` | Current file manifest. | +| `PUT /file-manifest` | CAS the complete file manifest. | +| `GET /vault-snapshot` | File manifest, referenced content heads, and event watermark from one read transaction. | +| `GET /events-since?after=N` | Every event after N, in order, with the current watermark. | +| `GET /ws` | Ordered event delivery and ephemeral cursor positions. | A content push includes a `Device-Id` header and this JSON body: @@ -54,13 +55,14 @@ UUID-to-path mapping in `file_manifest_entries`. Each version has a header row even when its entries are empty. Entries are unique by document ID and exact path within a version; portable path validation also rejects aliases and directory conflicts. Reads reconstruct the complete map, and vault snapshots join these -rows to the latest content versions. The version, entries, and complete file -manifest event in `events.event_json` commit atomically. Event replay and retry -acknowledgements continue to use `event_json`. +rows to the latest content versions. Event replay and retry acknowledgements are +also reconstructed from normalized content/manifest rows, without storing a +second complete map in the event table. Both pushes return a tagged `Accepted` or `StaleBase` response. An accepted file manifest response returns its assigned file manifest ID. A stale content response -includes bytes; a stale file manifest response includes the complete map. +includes metadata and byte length only; clients fetch immutable bytes after applying +their size and exclusion policy. A stale file manifest response includes the complete map. Accepted requests store their unique request ID and fingerprint in the event row, atomically with the version. Retrying the exact request reconstructs the original `Accepted` response from that event before checking the current head, even after @@ -70,8 +72,8 @@ identity is not part of the fingerprint. File manifest validation rejects duplicate UUID keys (including aliases), missing content, path aliases, inconsistent directory spelling, and file/ancestor -conflicts. Paths use `/`, NFC normalization, at most 240 UTF-8 bytes, and portable -Windows filename restrictions. Alias comparison is NFC → Unicode uppercase → +conflicts. Paths use `/`, NFC normalization, and portable Windows filename +restrictions without imposing a universal total-path byte limit. Alias comparison is NFC → Unicode uppercase → NFC. `.vault-link-sync` and its aliases are reserved. The server never sanitizes or allocates a replacement name. @@ -87,11 +89,19 @@ determine event ordering. The WebSocket handshake retains `lastSeenVaultUpdateId`. The server sends `{type: "vaultEvents", headEventId, events}` in increasing order, including events originated by that client. Each connection drains the database through one sender. -In-memory notifications only wake it; polling covers missed wakes and cancellation -after commit. Reconnect with the last durably processed event ID. The HTTP replay -endpoint uses the same history, so losing the final notification is recoverable. -A fresh client reads `/vault_snapshot`, applies it durably, then replays after its -watermark. +In-memory notifications only wake it; polling covers missed wakes and requests +interrupted after commit. Reconnect with the last durably processed event ID. The +HTTP replay endpoint uses the same history, so losing the final notification is +recoverable. +A fresh client reads `/vault-snapshot`, applies it durably, then replays after its +watermark. Successful HTTP responses include `X-Vault-Link-History`, containing +an event ID and a random event-incarnation token. Clients persist and echo this +checkpoint on later HTTP requests. The server returns HTTP 409 with +`X-Vault-Link-History-Mismatch: 1` before executing a request whose checkpoint is +absent from the current database. Restored backups retain their original prefix; +new events get new tokens even when numeric IDs or request UUIDs are reused. +Restore databases while the server is stopped. Live replacement of open SQLite +files is not supported. Cursor notifications are ephemeral and do not consume event IDs. Cursor expiry and disconnect broadcast the remaining clients, including an empty list when the last client leaves. @@ -116,8 +126,20 @@ flushes database directory entries; the filesystem must support those durability operations. Content, file manifests, and events (including request fingerprints) are retained indefinitely. -API v4 requires fresh vault databases and upgraded clients. Existing database -migration checksums intentionally reject earlier schemas; there is no automatic -erasure or migration. Use a new database directory and preserve old data. +Deploy API v5 clients and server together. The v4 database schema migrates by +adding event-incarnation tokens without deleting content or receipts. Schemas +from before v4 still require a separate database directory and explicit recovery. +Vault databases open and migrate independently on first access; one corrupt or +incompatible vault does not prevent healthy vaults from serving requests. + +Diffs are limited to 10,000 items and reconstructed in one pass over Unicode +scalars on a blocking worker, before taking the SQLite writer transaction. The +transaction rechecks request identity and the parent version before committing. +Clients send a snapshot when an edit exceeds the diff-operation budget. Decoded +content is bounded by `max_body_size_mb` as well as the encoded request limit. + +A missing configuration file is initialized with durably stored credentials. +Existing configurations must explicitly supply `users.user_configs`; omitted +credentials are rejected instead of generating an unpersisted token at startup. Base64 increases snapshot body size by about one third; request limits apply to the encoded JSON body. diff --git a/sync-server/src/app_state/cursors.rs b/sync-server/src/app_state/cursors.rs index 65b941ed..e535d5b2 100644 --- a/sync-server/src/app_state/cursors.rs +++ b/sync-server/src/app_state/cursors.rs @@ -2,6 +2,7 @@ use core::time::Duration; use std::{collections::HashMap, sync::Arc}; use tokio::sync::Mutex; +use uuid::Uuid; use super::{ database::models::{DeviceId, VaultId}, @@ -19,6 +20,7 @@ pub struct Cursors { config: DatabaseConfig, broadcasts: Broadcasts, vault_to_cursors: Arc>>>, + sessions: Arc>>>, } impl Cursors { @@ -27,16 +29,39 @@ impl Cursors { config: config.clone(), broadcasts: broadcasts.clone(), vault_to_cursors: Arc::new(Mutex::new(HashMap::new())), + sessions: Arc::default(), } } + pub async fn register_connection(&self, vault_id: &str, device_id: &str) -> Uuid { + let mut sessions = self.sessions.lock().await; + let session = Uuid::new_v4(); + sessions + .entry(vault_id.to_owned()) + .or_default() + .insert(device_id.to_owned(), session); + self.remove_device_cursors(vault_id, device_id).await; + session + } + pub async fn update_cursors( &self, vault_id: VaultId, user_name: String, device_id: &DeviceId, + session: Uuid, document_to_cursors: Vec, ) { + // A reconnect can overlap an old connection's buffered messages and + // cleanup. Only the most recently authenticated session owns presence. + let sessions = self.sessions.lock().await; + if sessions + .get(&vault_id) + .and_then(|devices| devices.get(device_id)) + != Some(&session) + { + return; + } let mut vault_to_cursors = self.vault_to_cursors.lock().await; let all_device_cursors = vault_to_cursors @@ -100,7 +125,22 @@ impl Cursors { .await; } - pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { + pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str, session: Uuid) { + let mut sessions = self.sessions.lock().await; + let Some(devices) = sessions.get_mut(vault_id) else { + return; + }; + if devices.get(device_id) != Some(&session) { + return; + } + devices.remove(device_id); + if devices.is_empty() { + sessions.remove(vault_id); + } + self.remove_device_cursors(vault_id, device_id).await; + } + + async fn remove_device_cursors(&self, vault_id: &str, device_id: &str) { let mut vault_to_cursors = self.vault_to_cursors.lock().await; if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { @@ -147,32 +187,42 @@ mod tests { async fn disconnect_broadcasts_remaining_clients_and_the_final_empty_state() { let broadcasts = Broadcasts::new(&ServerConfig::default()); let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + let mut sessions = HashMap::new(); for device in ["one", "two"] { + let session = cursors.register_connection("vault", device).await; + sessions.insert(device, session); cursors .update_cursors( "vault".to_owned(), "user".to_owned(), &device.to_owned(), + session, vec![], ) .await; } let mut receiver = broadcasts.get_receiver("vault".to_owned()).await; let mut other = broadcasts.get_receiver("other".to_owned()).await; - cursors.remove_cursors_of_device("vault", "one").await; + cursors + .remove_cursors_of_device("vault", "one", sessions["one"]) + .await; let Notification::Cursors(update) = receiver.try_recv().unwrap() else { panic!("expected cursors") }; assert_eq!(update.clients.len(), 1); assert_eq!(update.clients[0].device_id, "two"); - cursors.remove_cursors_of_device("vault", "two").await; + cursors + .remove_cursors_of_device("vault", "two", sessions["two"]) + .await; let Notification::Cursors(update) = receiver.try_recv().unwrap() else { panic!("expected cursors") }; assert!(update.clients.is_empty()); assert!(other.try_recv().is_err()); - cursors.remove_cursors_of_device("vault", "two").await; + cursors + .remove_cursors_of_device("vault", "two", sessions["two"]) + .await; assert!(receiver.try_recv().is_err()); } @@ -182,11 +232,13 @@ mod tests { let config = DatabaseConfig::default(); let cursors = Cursors::new(&config, &broadcasts); for (vault, device) in [("vault", "expired"), ("vault", "live"), ("other", "live")] { + let session = cursors.register_connection(vault, device).await; cursors .update_cursors( vault.to_owned(), "user".to_owned(), &device.to_owned(), + session, vec![], ) .await; @@ -229,4 +281,39 @@ mod tests { assert!(receiver.try_recv().is_err()); assert!(other.try_recv().is_err()); } + #[tokio::test] + async fn replacement_session_fences_old_updates_and_disconnects() { + let broadcasts = Broadcasts::new(&ServerConfig::default()); + let cursors = Cursors::new(&DatabaseConfig::default(), &broadcasts); + let vault = "vault".to_owned(); + let device = "device".to_owned(); + let old = cursors.register_connection(&vault, &device).await; + cursors + .update_cursors(vault.clone(), "old".to_owned(), &device, old, vec![]) + .await; + let new = cursors.register_connection(&vault, &device).await; + cursors + .update_cursors(vault.clone(), "new".to_owned(), &device, new, vec![]) + .await; + let mut receiver = broadcasts.get_receiver(vault.clone()).await; + cursors + .update_cursors( + vault.clone(), + "late old update".to_owned(), + &device, + old, + vec![], + ) + .await; + cursors.remove_cursors_of_device(&vault, &device, old).await; + assert!(receiver.try_recv().is_err()); + assert_eq!(cursors.get_cursors(&vault).await[0].user_name, "new"); + cursors.remove_cursors_of_device(&vault, &device, new).await; + let Notification::Cursors(update) = receiver.try_recv().unwrap() else { + panic!("expected cursors") + }; + assert!(update.clients.is_empty()); + assert!(cursors.get_cursors(&vault).await.is_empty()); + assert!(cursors.sessions.lock().await.is_empty()); + } } diff --git a/sync-server/src/app_state/database.rs b/sync-server/src/app_state/database.rs index 9aa99db5..eca69a44 100644 --- a/sync-server/src/app_state/database.rs +++ b/sync-server/src/app_state/database.rs @@ -2,7 +2,7 @@ use core::time::Duration; use std::{ collections::HashMap, path::{Path, PathBuf}, - sync::Arc, + sync::{Arc, Weak}, }; use anyhow::{Context as _, Result}; @@ -48,6 +48,7 @@ impl std::fmt::Debug for PoolWithTimestamp { pub struct Database { config: DatabaseConfig, connection_pools: Arc>>, + opening_pools: Arc>>>>, } pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; @@ -66,25 +67,16 @@ impl Database { ) })?; - info!("Applying pending database migrations"); + // SQLite flushes its own files, not ancestors created by this process. + Self::flush_directory_ancestors(&vault_directory).await?; - // Filenames encode only a digest, so their vault names cannot be - // recovered here. Check migrations now and open pools lazily by name. - let mut entries = tokio::fs::read_dir(&vault_directory).await?; - while let Some(entry) = entries.next_entry().await? { - if entry.file_type().await?.is_file() - && entry.path().extension().is_some_and(|ext| ext == "sqlite") - { - Self::open_database(config, &entry.path()) - .await? - .close() - .await; - } - } + // Open and migrate each vault lazily under its own lock. A corrupt or + // incompatible database must fail that vault, not global startup. let database = Self { config: config.clone(), connection_pools: Arc::default(), + opening_pools: Arc::default(), }; // Start background task to cleanup idle connection pools @@ -127,9 +119,32 @@ impl Database { Self::run_migrations(&pool).await?; + if let Some(directory) = file_name.parent() { + Self::flush_directory_ancestors(directory).await?; + } + Ok(pool) } + async fn flush_directory_ancestors(directory: &Path) -> Result<()> { + #[cfg(unix)] + { + let directory = tokio::fs::canonicalize(directory).await?; + tokio::task::spawn_blocking(move || -> Result<()> { + for ancestor in directory.ancestors() { + std::fs::File::open(ancestor)?.sync_all().with_context(|| { + format!("Cannot flush directory {}", ancestor.display()) + })?; + } + Ok(()) + }) + .await??; + } + #[cfg(not(unix))] + let _ = directory; + Ok(()) + } + async fn run_migrations(pool: &Pool) -> Result<()> { sqlx::migrate!("src/app_state/database/migrations") .run(pool) @@ -140,28 +155,49 @@ impl Database { async fn get_connection_pool(&self, vault: &VaultId) -> Result> { let vault = normalize_string(vault); validate_vault_id(&vault)?; - let mut pools = self.connection_pools.lock().await; - - if !pools.contains_key(&vault) { - let file_name = Self::database_path(&self.config, &vault); - let pool = Self::open_database(&self.config, &file_name).await?; - pools.insert( - vault.clone(), - PoolWithTimestamp { - pool, - last_accessed: Instant::now(), - }, - ); + if let Some(pool) = self.existing_pool(&vault).await { + return Ok(pool); } - let pool_with_timestamp = pools - .get_mut(&vault) - .expect("Pool was just inserted or already exists"); + // Serialize initialization per vault. SQLite lock waits, migrations and + // directory flushes must never hold the registry for unrelated vaults. + // Weak entries also allow cancelled/failed opens to be retried safely. + let opening = { + let mut openings = self.opening_pools.lock().await; + openings.retain(|_, opening| opening.strong_count() > 0); + if let Some(opening) = openings.get(&vault).and_then(Weak::upgrade) { + opening + } else { + let opening = Arc::new(Mutex::new(())); + openings.insert(vault.clone(), Arc::downgrade(&opening)); + opening + } + }; + let _opening = opening.lock().await; + if let Some(pool) = self.existing_pool(&vault).await { + return Ok(pool); + } + let file_name = Self::database_path(&self.config, &vault); + let pool = Self::open_database(&self.config, &file_name).await?; + self.connection_pools.lock().await.insert( + vault, + PoolWithTimestamp { + pool: pool.clone(), + last_accessed: Instant::now(), + }, + ); + Ok(pool) + } - // Update last accessed time - pool_with_timestamp.last_accessed = Instant::now(); - - Ok(pool_with_timestamp.pool.clone()) + async fn existing_pool(&self, vault: &VaultId) -> Option> { + self.connection_pools + .lock() + .await + .get_mut(vault) + .map(|entry| { + entry.last_accessed = Instant::now(); + entry.pool.clone() + }) } /// Attempting to write from this transaction might result in a @@ -196,16 +232,22 @@ impl Database { .iter() .filter(|(_, pool_with_timestamp)| { now.duration_since(pool_with_timestamp.last_accessed) > idle_timeout + && pool_with_timestamp.pool.num_idle() + == pool_with_timestamp.pool.size() as usize }) .map(|(vault_id, _)| vault_id.clone()) .collect(); - // Close and remove idle pools - for vault_id in &vaults_to_remove { - if let Some(pool_with_timestamp) = pools.remove(vault_id) { - info!("Closing idle database connection pool for vault `{vault_id}`"); - pool_with_timestamp.pool.close().await; - } + let closing: Vec<_> = vaults_to_remove + .into_iter() + .filter_map(|vault| pools.remove(&vault).map(|entry| (vault, entry.pool))) + .collect(); + drop(pools); + // A checkout racing cleanup can delay close. It must never retain the + // mutex used to access every other vault in the process. + for (vault, pool) in closing { + info!("Closing idle database connection pool for vault `{vault}`"); + pool.close().await; } } diff --git a/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql b/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql new file mode 100644 index 00000000..dc96b259 --- /dev/null +++ b/sync-server/src/app_state/database/migrations/20260920000000_history_checkpoint.sql @@ -0,0 +1,4 @@ +-- Tokens identify an event's incarnation, including when a restored backup +-- reuses its numeric ID or a client resubmits the same request UUID. +ALTER TABLE events ADD COLUMN event_token TEXT; +UPDATE events SET event_token = lower(hex(randomblob(16))); diff --git a/sync-server/src/app_state/database/models.rs b/sync-server/src/app_state/database/models.rs index 57c4cd94..edb3f885 100644 --- a/sync-server/src/app_state/database/models.rs +++ b/sync-server/src/app_state/database/models.rs @@ -123,5 +123,9 @@ pub struct VaultSnapshot { pub struct EventBatch { #[ts(as = "f64")] pub head_event_id: VaultUpdateId, + /// Last event in this page; `head_event_id` is the vault's current head. + #[serde(skip_serializing_if = "Option::is_none")] + #[ts(optional, as = "Option")] + pub end_event_id: Option, pub events: Vec, } diff --git a/sync-server/src/app_state/database/mutations.rs b/sync-server/src/app_state/database/mutations.rs index b0af9e4a..6585d102 100644 --- a/sync-server/src/app_state/database/mutations.rs +++ b/sync-server/src/app_state/database/mutations.rs @@ -10,14 +10,15 @@ impl Database { request_id: uuid::Uuid, fingerprint: &[u8], ) -> Result { - Ok( - sqlx::query("INSERT INTO events(request_id, request_fingerprint) VALUES (?, ?)") - .bind(request_id.to_string()) - .bind(fingerprint) - .execute(&mut **tx) - .await? - .last_insert_rowid(), + Ok(sqlx::query( + "INSERT INTO events(request_id, request_fingerprint, event_token) VALUES (?, ?, ?)", ) + .bind(request_id.to_string()) + .bind(fingerprint) + .bind(uuid::Uuid::new_v4().simple().to_string()) + .execute(&mut **tx) + .await? + .last_insert_rowid()) } pub async fn insert_document_version( diff --git a/sync-server/src/app_state/database/queries.rs b/sync-server/src/app_state/database/queries.rs index 5ba1a97d..6afec8d7 100644 --- a/sync-server/src/app_state/database/queries.rs +++ b/sync-server/src/app_state/database/queries.rs @@ -8,6 +8,22 @@ use super::{ use anyhow::{Context as _, Result, ensure}; impl Database { + pub async fn missing_document( + tx: &mut Transaction<'_>, + ids: &[DocumentId], + ) -> Result> { + // A single covering-index lookup per requested identity; never load a + // content blob or decode a version while validating manifest references. + Ok(sqlx::query_scalar( + "SELECT value FROM json_each(?) AS requested + WHERE NOT EXISTS (SELECT 1 FROM documents WHERE document_id = requested.value) + LIMIT 1", + ) + .bind(serde_json::to_string(ids)?) + .fetch_optional(&mut **tx) + .await?) + } + pub async fn get_request_event( tx: &mut Transaction<'_>, request_id: uuid::Uuid, @@ -168,24 +184,72 @@ impl Database { ); let rows: Vec<(VaultUpdateId, String)> = sqlx::query_as( - "SELECT event_id, request_id FROM events WHERE event_id > ? ORDER BY event_id", + "SELECT event_id, request_id FROM events WHERE event_id > ? ORDER BY event_id LIMIT 64", ) .bind(after) .fetch_all(&mut *tx) .await?; let mut events = Vec::with_capacity(rows.len()); + let mut bytes = 0; + let mut end_event_id = after; for (event_id, request_id) in rows { - events.push(EventRecord { + let event = EventRecord { event_id, request_id: request_id.parse()?, event: Self::event_by_id(&mut tx, event_id).await?, - }); + }; + let size = serde_json::to_vec(&event)?.len(); + // One manifest is atomic and may exceed this budget. Always allow + // one event so even such a page advances; never accumulate history. + if !events.is_empty() && bytes + size > 1024 * 1024 { + break; + } + bytes += size; + end_event_id = event_id; + events.push(event); } Ok(EventBatch { head_event_id, + end_event_id: Some(end_event_id), events, }) } } + +impl Database { + pub async fn history_checkpoint(&self, vault: &VaultId) -> Result { + let mut tx = self.create_readonly_transaction(vault).await?; + let row: Option<(i64, String)> = sqlx::query_as( + "SELECT event_id, event_token FROM events ORDER BY event_id DESC LIMIT 1", + ) + .fetch_optional(&mut *tx) + .await?; + Ok(row.map_or_else( + || "0:empty".to_owned(), + |(id, token)| format!("{id}:{token}"), + )) + } + + pub async fn contains_checkpoint(&self, vault: &VaultId, checkpoint: &str) -> Result { + if checkpoint == "0:empty" { + return Ok(true); + } + let Some((id, token)) = checkpoint.split_once(':') else { + return Ok(false); + }; + let Ok(id) = id.parse::() else { + return Ok(false); + }; + let mut tx = self.create_readonly_transaction(vault).await?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM events WHERE event_id = ? AND event_token = ?)", + ) + .bind(id) + .bind(token) + .fetch_one(&mut *tx) + .await?; + Ok(exists) + } +} diff --git a/sync-server/src/app_state/database/tests.rs b/sync-server/src/app_state/database/tests.rs index 9b5b1e62..b3578f46 100644 --- a/sync-server/src/app_state/database/tests.rs +++ b/sync-server/src/app_state/database/tests.rs @@ -94,6 +94,32 @@ async fn rejects_invalid_vault_names_at_the_database_boundary() { assert!(database.connection_pools.lock().await.is_empty()); } +#[tokio::test] +async fn migrations_can_be_reapplied_without_losing_requests_or_content() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + let vault = "migrations".to_owned(); + let version = insert_content(&database, &vault, uuid::Uuid::new_v4(), b"only copy").await; + let pool = database.get_connection_pool(&vault).await.unwrap(); + Database::run_migrations(&pool).await.unwrap(); + Database::run_migrations(&pool).await.unwrap(); + let reopened = Database::try_new(&config).await.unwrap(); + assert_eq!( + reopened + .get_document_version(&vault, version.vault_update_id, None) + .await + .unwrap() + .unwrap() + .content, + b"only copy" + ); + assert_eq!( + reopened.events_after(&vault, 0).await.unwrap().events.len(), + 1 + ); +} + #[tokio::test] async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { let directory = tempfile::tempdir().unwrap(); @@ -134,3 +160,246 @@ async fn snapshot_returns_latest_referenced_metadata_and_byte_lengths() { ); } } + +#[tokio::test] +async fn replay_pages_are_bounded_and_cover_every_event_in_order() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "pages".to_owned(); + let id = uuid::Uuid::new_v4(); + for _ in 0..150 { + insert_content(&database, &vault, id, b"edit").await; + } + let mut after = 0; + while after < 150 { + let batch = database.events_after(&vault, after).await.unwrap(); + assert!(batch.events.len() <= 64, "catchup must bound each page"); + assert!(!batch.events.is_empty()); + assert_eq!(batch.head_event_id, 150); + for event in batch.events { + assert_eq!(event.event_id, after + 1); + after = event.event_id; + } + } + assert!( + database + .events_after(&vault, after) + .await + .unwrap() + .events + .is_empty() + ); +} + +#[tokio::test] +async fn idle_cleanup_does_not_wait_for_an_active_vault() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + database.vault_snapshot(&"other".to_owned()).await.unwrap(); + let vault = "slow".to_owned(); + let transaction = database.create_write_transaction(&vault).await.unwrap(); + database + .connection_pools + .lock() + .await + .get_mut(&vault) + .unwrap() + .last_accessed = Instant::now() - Duration::from_secs(301); + tokio::time::timeout(Duration::from_millis(200), database.cleanup_idle_pools()) + .await + .expect("cleanup must leave an active pool alone"); + let other = tokio::time::timeout( + Duration::from_millis(200), + database.vault_snapshot(&"other".to_owned()), + ) + .await + .expect("other vaults must remain accessible") + .unwrap(); + assert_eq!(other.head_event_id, 0); + assert!( + !database + .get_connection_pool(&vault) + .await + .unwrap() + .is_closed() + ); + drop(transaction); +} + +#[tokio::test] +async fn manifest_existence_checks_are_independent_of_payload_decoding() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "existence".to_owned(); + let id = uuid::Uuid::new_v4(); + insert_content(&database, &vault, id, b"content").await; + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + // Make full-row deserialization fail: checking a foreign identity must not + // decode unrelated version fields or read its potentially enormous blob. + sqlx::query("UPDATE documents SET updated_date = 'unreadable timestamp'") + .execute(&mut *tx) + .await + .unwrap(); + assert_eq!( + Database::missing_document(&mut tx, &[id]).await.unwrap(), + None + ); + let missing = uuid::Uuid::new_v4(); + assert_eq!( + Database::missing_document(&mut tx, &[id, missing]) + .await + .unwrap(), + Some(missing.to_string()) + ); + assert_eq!( + Database::missing_document(&mut tx, &[]).await.unwrap(), + None + ); +} + +#[tokio::test] +async fn replay_byte_budget_prevents_accumulating_large_historical_manifests() { + let directory = tempfile::tempdir().unwrap(); + let database = Database::try_new(&config(&directory)).await.unwrap(); + let vault = "large-manifests".to_owned(); + let mut tx = database.create_write_transaction(&vault).await.unwrap(); + // Each event fits independently, but fewer than 64 fit the response budget. + let entries: std::collections::BTreeMap<_, _> = (0..1500) + .map(|index| { + ( + uuid::Uuid::new_v4(), + format!("{index}-{}.md", "x".repeat(180)), + ) + }) + .collect(); + for _ in 0..8 { + let file_manifest_id = Database::allocate_event(&mut tx, uuid::Uuid::new_v4(), b"test") + .await + .unwrap(); + Database::insert_file_manifest( + &mut tx, + &FileManifest { + file_manifest_id, + entries: entries.clone(), + }, + ) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + let batch = database.events_after(&vault, 0).await.unwrap(); + assert!(!batch.events.is_empty()); + assert!(batch.events.len() < 8); + assert!(serde_json::to_vec(&batch).unwrap().len() < 1024 * 1024 + 1024); + assert_eq!( + batch.end_event_id, + batch.events.last().map(|event| event.event_id) + ); +} + +#[tokio::test] +async fn opening_a_locked_vault_does_not_block_unrelated_vaults() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let database = Database::try_new(&config).await.unwrap(); + database.vault_snapshot(&"other".to_owned()).await.unwrap(); + let vault = "locked-opening".to_owned(); + // A second process holds the writer lock while this process lazily checks + // migrations. Even already-open, unrelated vaults must remain available. + let external = Database::open_database(&config, &Database::database_path(&config, &vault)) + .await + .unwrap(); + let transaction = external.begin_with("BEGIN IMMEDIATE").await.unwrap(); + let opening_database = database.clone(); + let opening = tokio::spawn(async move { opening_database.get_connection_pool(&vault).await }); + tokio::time::sleep(Duration::from_millis(100)).await; + let opening_is_blocked = !opening.is_finished(); + let other = tokio::time::timeout( + Duration::from_millis(300), + database.vault_snapshot(&"other".to_owned()), + ) + .await; + transaction.rollback().await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), opening) + .await + .unwrap() + .unwrap() + .unwrap(); + external.close().await; + assert!( + opening_is_blocked, + "the test must exercise an in-progress database open" + ); + assert_eq!( + other + .expect("a blocked migration must not lock every vault") + .unwrap() + .head_event_id, + 0 + ); +} + +#[tokio::test] +async fn concurrent_vault_opens_share_a_pool_and_cancelled_opens_can_retry() { + let directory = tempfile::tempdir().unwrap(); + let mut config = config(&directory); + config.max_connections_per_vault = 1; + let database = Database::try_new(&config).await.unwrap(); + let vault = "cancelled-opening".to_owned(); + let external = Database::open_database(&config, &Database::database_path(&config, &vault)) + .await + .unwrap(); + let transaction = external.begin_with("BEGIN IMMEDIATE").await.unwrap(); + let opener = database.clone(); + let name = vault.clone(); + let cancelled = tokio::spawn(async move { opener.get_connection_pool(&name).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!cancelled.is_finished()); + cancelled.abort(); + assert!(cancelled.await.unwrap_err().is_cancelled()); + transaction.rollback().await.unwrap(); + let mut tasks = Vec::new(); + for _ in 0..8 { + let database = database.clone(); + let vault = vault.clone(); + tasks.push(tokio::spawn(async move { + database.get_connection_pool(&vault).await.unwrap() + })); + } + let pools = tokio::time::timeout(Duration::from_secs(5), futures::future::join_all(tasks)) + .await + .unwrap(); + let pools: Vec<_> = pools.into_iter().map(Result::unwrap).collect(); + let checked_out = pools[0].acquire().await.unwrap(); + for pool in &pools[1..] { + assert!( + tokio::time::timeout(Duration::from_millis(25), pool.acquire()) + .await + .is_err(), + "all callers must share the single configured connection" + ); + } + drop(checked_out); + external.close().await; +} + +#[tokio::test] +async fn one_corrupt_vault_does_not_prevent_startup_or_healthy_vault_access() { + let directory = tempfile::tempdir().unwrap(); + let config = config(&directory); + let bad = Database::database_path(&config, "broken"); + tokio::fs::create_dir_all(bad.parent().unwrap()) + .await + .unwrap(); + tokio::fs::write(&bad, b"not a SQLite database") + .await + .unwrap(); + let database = Database::try_new(&config).await.unwrap(); + assert!(database.vault_snapshot(&"broken".to_owned()).await.is_err()); + let saved = insert_content(&database, "healthy", uuid::Uuid::new_v4(), b"safe").await; + assert_eq!(saved.content, b"safe"); + assert_eq!( + tokio::fs::read(bad).await.unwrap(), + b"not a SQLite database" + ); +} diff --git a/sync-server/src/app_state/websocket/broadcasts.rs b/sync-server/src/app_state/websocket/broadcasts.rs index 1b157b13..0c5a8f83 100644 --- a/sync-server/src/app_state/websocket/broadcasts.rs +++ b/sync-server/src/app_state/websocket/broadcasts.rs @@ -12,23 +12,39 @@ pub enum Notification { #[derive(Debug, Clone)] pub struct Broadcasts { capacity: usize, + admission: Arc>>>, senders: Arc>>>, } impl Broadcasts { pub fn new(config: &ServerConfig) -> Self { Self { - capacity: config.max_clients_per_vault.max(1), + capacity: config.max_clients_per_vault, + admission: Arc::default(), senders: Arc::default(), } } + pub fn try_admit(&self, vault: &VaultId) -> Option { + let mut limits = self.admission.lock().unwrap(); + limits.retain(|_, limit| limit.strong_count() > 0); + let limit = limits + .get(vault) + .and_then(std::sync::Weak::upgrade) + .unwrap_or_else(|| { + let limit = Arc::new(tokio::sync::Semaphore::new(self.capacity)); + limits.insert(vault.clone(), Arc::downgrade(&limit)); + limit + }); + limit.try_acquire_owned().ok() + } + async fn sender(&self, vault: VaultId) -> broadcast::Sender { self.senders .lock() .await .entry(vault) - .or_insert_with(|| broadcast::channel(self.capacity).0) + .or_insert_with(|| broadcast::channel(self.capacity.max(1)).0) .clone() } diff --git a/sync-server/src/app_state/websocket/utils.rs b/sync-server/src/app_state/websocket/utils.rs index e72a3775..a1423e09 100644 --- a/sync-server/src/app_state/websocket/utils.rs +++ b/sync-server/src/app_state/websocket/utils.rs @@ -49,9 +49,13 @@ pub async fn send_update_over_websocket( .context("Failed to serialize update") .map_err(server_error)?; - sender - .send(Message::Text(serialized_update)) - .await - .context("Failed to send message over websocket") - .map_err(server_error) + tokio::time::timeout( + std::time::Duration::from_secs(10), + sender.send(Message::Text(serialized_update)), + ) + .await + .context("WebSocket send deadline expired") + .map_err(server_error)? + .context("Failed to send message over websocket") + .map_err(server_error) } diff --git a/sync-server/src/config.rs b/sync-server/src/config.rs index 6a003d2e..c4246403 100644 --- a/sync-server/src/config.rs +++ b/sync-server/src/config.rs @@ -20,7 +20,6 @@ pub struct Config { pub database: DatabaseConfig, #[serde(default)] pub server: ServerConfig, - #[serde(default)] pub users: UserConfig, #[serde(default)] pub logging: LoggingConfig, @@ -28,23 +27,52 @@ pub struct Config { impl Config { pub async fn read_or_create(path: &Path) -> Result { - let config = if path.exists() { - info!( - "Loading configuration from `{}`", - path.canonicalize().unwrap().display() - ); - Self::load_from_file(path).await? - } else { - Self::default() - }; - - config.write(path).await?; - info!( - "Updated configuration at `{}`", - path.canonicalize().unwrap().display() - ); - - Ok(config) + match Self::load_from_file(path).await { + Ok(config) => return Ok(config), + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => {} + Err(error) => return Err(error), + } + let contents = serde_yaml::to_string(&Self::default())?; + let destination = path.to_owned(); + tokio::task::spawn_blocking(move || -> Result<()> { + use std::io::Write; + let path = destination; + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + let temporary = parent.join(format!(".vault-link-config-{}.tmp", uuid::Uuid::new_v4())); + let result = (|| -> Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + file.write_all(contents.as_bytes())?; + file.sync_all()?; + // Publish a complete, durable file without replacing a config + // another process may have created while we generated defaults. + match std::fs::hard_link(&temporary, &path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })(); + let _ = std::fs::remove_file(&temporary); + result + }) + .await??; + info!("Loading configuration from `{}`", path.display()); + Self::load_from_file(path).await } pub async fn load_from_file(path: &Path) -> Result { @@ -57,12 +85,54 @@ impl Config { serde_yaml::from_str(&contents).context("Failed to parse configuration") } +} - pub async fn write(&self, path: &Path) -> Result<()> { - let contents = serde_yaml::to_string(&self).context("Failed to serialize configuration")?; +#[cfg(test)] +mod tests { + use super::*; - fs::write(path, contents) - .await - .context("Failed to write configuration to disk") + #[tokio::test] + async fn startup_preserves_existing_configuration_bytes_and_permissions() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + let original = b"# administrator's comments and formatting\nserver: {port: 9123}\nusers: {user_configs: []}\n"; + fs::write(&path, original).await.unwrap(); + let mut permissions = fs::metadata(&path).await.unwrap().permissions(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).await.unwrap(); + let config = Config::read_or_create(&path).await.unwrap(); + assert_eq!(config.server.port, 9123); + assert_eq!(fs::read(&path).await.unwrap(), original); + } + + #[tokio::test] + async fn existing_configuration_cannot_generate_unpersisted_credentials() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + for contents in ["server: {port: 9123}\n", "users: {}\n"] { + fs::write(&path, contents).await.unwrap(); + for _ in 0..2 { + assert!( + Config::read_or_create(&path).await.is_err(), + "Existing credentials must be explicit; random defaults are only for initialization" + ); + } + assert_eq!(fs::read_to_string(&path).await.unwrap(), contents); + } + } + + #[tokio::test] + async fn concurrent_initialization_never_overwrites_configuration() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("config.yml"); + let results = + futures::future::join_all((0..16).map(|_| Config::read_or_create(&path))).await; + let config = Config::load_from_file(&path).await.unwrap(); + for result in results { + assert_eq!( + serde_yaml::to_string(&result.unwrap()).unwrap(), + serde_yaml::to_string(&config).unwrap() + ); + } } } diff --git a/sync-server/src/config/server_config.rs b/sync-server/src/config/server_config.rs index 59d1480f..fac5cf98 100644 --- a/sync-server/src/config/server_config.rs +++ b/sync-server/src/config/server_config.rs @@ -85,8 +85,9 @@ mod tests { let path = directory.path().join("config.yml"); let fresh = Config::read_or_create(&path).await.unwrap(); let reread = Config::load_from_file(&path).await.unwrap(); - let omitted: Config = serde_yaml::from_str("{}").unwrap(); - let explicit: Config = serde_yaml::from_str("server: {}").unwrap(); + let omitted: Config = serde_yaml::from_str("users: {user_configs: []}").unwrap(); + let explicit: Config = + serde_yaml::from_str("server: {}\nusers: {user_configs: []}").unwrap(); for config in [fresh, reread, omitted, explicit] { assert_eq!(config.server.host, DEFAULT_HOST); assert_eq!(config.server.port, DEFAULT_PORT); diff --git a/sync-server/src/config/user_config.rs b/sync-server/src/config/user_config.rs index 0bb3379b..985a5c2a 100644 --- a/sync-server/src/config/user_config.rs +++ b/sync-server/src/config/user_config.rs @@ -9,7 +9,7 @@ use crate::{ #[derive(Debug, Deserialize, Serialize, Clone)] pub struct UserConfig { - #[serde(default = "default_users", deserialize_with = "validate_users")] + #[serde(deserialize_with = "validate_users")] pub user_configs: Vec, } diff --git a/sync-server/src/server.rs b/sync-server/src/server.rs index 9eb66172..4df81e76 100644 --- a/sync-server/src/server.rs +++ b/sync-server/src/server.rs @@ -63,6 +63,11 @@ pub async fn create_server(config: Config) -> Result<()> { http::header::CONTENT_TYPE, http::header::AUTHORIZATION, DEVICE_ID_HEADER_NAME.clone(), + http::header::HeaderName::from_static("x-vault-link-history"), + ]) + .expose_headers([ + http::header::HeaderName::from_static("x-vault-link-history"), + http::header::HeaderName::from_static("x-vault-link-history-mismatch"), ]) .allow_methods([Method::GET, Method::PUT]), ) @@ -113,6 +118,10 @@ fn get_authed_routes(app_state: AppState) -> Router { get(endpoints::fetch_latest_document_version::fetch_latest_document_version) .put(endpoints::put_file_content::put_file_content), ) + .route( + "/vaults/:vault_id/documents/:document_id/metadata", + get(endpoints::fetch_latest_document_version::fetch_latest_document_metadata), + ) .route( "/vaults/:vault_id/documents/:document_id/versions/:vault_update_id/content", get(endpoints::fetch_document_version_content::fetch_document_version_content), diff --git a/sync-server/src/server/auth.rs b/sync-server/src/server/auth.rs index c601ef54..c7a49378 100644 --- a/sync-server/src/server/auth.rs +++ b/sync-server/src/server/auth.rs @@ -2,8 +2,9 @@ use std::collections::HashMap; use axum::{ extract::{Path, Request, State}, + http::{HeaderValue, StatusCode}, middleware::Next, - response::Response, + response::{IntoResponse, Response}, }; use axum_extra::{ TypedHeader, @@ -14,7 +15,9 @@ use log::info; use crate::{ app_state::{AppState, database::models::VaultId}, config::user_config::{AllowListedVaults, User, VaultAccess}, - errors::{SyncServerError, client_error, permission_denied_error, unauthenticated_error}, + errors::{ + SyncServerError, client_error, permission_denied_error, server_error, unauthenticated_error, + }, utils::normalize_vault_id::{normalize_string, validate_vault_id}, }; @@ -34,9 +37,42 @@ pub async fn auth_middleware( let user = auth(&state, token, &vault_id)?; + // A restore happens while the server is stopped. Validate before executing + // any old request, including retries whose numeric parent IDs were reused. + if let Some(checkpoint) = req.headers().get("x-vault-link-history") { + let checkpoint = checkpoint + .to_str() + .map_err(|error| client_error(error.into()))?; + if !state + .database + .contains_checkpoint(&vault_id, checkpoint) + .await + .map_err(server_error)? + { + let mut response = (StatusCode::CONFLICT, "Server history changed").into_response(); + response.headers_mut().insert( + "x-vault-link-history-mismatch", + HeaderValue::from_static("1"), + ); + return Ok(response); + } + } + req.extensions_mut().insert(user); - Ok(next.run(req).await) + let mut response = next.run(req).await; + if response.status().is_success() { + let checkpoint = state + .database + .history_checkpoint(&vault_id) + .await + .map_err(server_error)?; + response.headers_mut().insert( + "x-vault-link-history", + HeaderValue::from_str(&checkpoint).map_err(|error| server_error(error.into()))?, + ); + } + Ok(response) } pub fn auth(state: &AppState, token: &str, vault_id: &VaultId) -> Result { diff --git a/sync-server/src/server/endpoints/fetch_latest_document_version.rs b/sync-server/src/server/endpoints/fetch_latest_document_version.rs index 870ac669..3a51f182 100644 --- a/sync-server/src/server/endpoints/fetch_latest_document_version.rs +++ b/sync-server/src/server/endpoints/fetch_latest_document_version.rs @@ -9,7 +9,7 @@ use super::VaultPath; use crate::{ app_state::{ AppState, - database::models::{DocumentId, DocumentVersion}, + database::models::{DocumentId, DocumentVersion, DocumentVersionWithoutContent}, }, errors::{SyncServerError, not_found_error, server_error}, }; @@ -37,3 +37,21 @@ pub async fn fetch_latest_document_version( Ok(Json(latest_version.into())) } + +pub async fn fetch_latest_document_metadata( + Path((VaultPath(vault_id), document_id)): Path<(VaultPath, DocumentId)>, + State(state): State, +) -> Result, SyncServerError> { + let mut tx = state + .database + .create_readonly_transaction(&vault_id) + .await + .map_err(server_error)?; + let metadata = sqlx::query_as::<_, DocumentVersionWithoutContent>( + "SELECT vault_update_id, document_id, updated_date, user_id, device_id, length(content) AS content_size + FROM documents WHERE document_id = ? ORDER BY vault_update_id DESC LIMIT 1" + ).bind(document_id.hyphenated()).fetch_optional(&mut *tx).await + .map_err(|error| server_error(error.into()))? + .ok_or_else(|| not_found_error(anyhow!("Document does not exist")))?; + Ok(Json(metadata)) +} diff --git a/sync-server/src/server/endpoints/put_file_content.rs b/sync-server/src/server/endpoints/put_file_content.rs index 4af4504c..2796e71c 100644 --- a/sync-server/src/server/endpoints/put_file_content.rs +++ b/sync-server/src/server/endpoints/put_file_content.rs @@ -23,7 +23,7 @@ use axum::{ use axum_extra::TypedHeader; use base64::{Engine as _, engine::general_purpose::STANDARD}; use log::debug; -use reconcile_text::{BuiltinTokenizer, EditedText, NumberOrText}; +use reconcile_text::NumberOrText; use sha2::{Digest, Sha256}; #[axum::debug_handler] @@ -41,6 +41,30 @@ pub async fn put_file_content( .map_err(|error| server_error(error.into()))?, ); + // Build bytes against an immutable parent off the async executor and before + // taking the writer lock. The transaction below rechecks CAS and receipts. + let initial_parent = state + .database + .get_latest_document_version(&vault_id, &document_id, None) + .await + .map_err(server_error)?; + let prepared = if initial_parent.as_ref().map(|p| p.vault_update_id) == push.parent_version_id { + let limit = state + .config + .server + .max_body_size_mb + .saturating_mul(1024 * 1024); + Some( + tokio::task::spawn_blocking(move || { + decode_content(initial_parent, push.content, limit) + }) + .await + .map_err(|error| server_error(error.into()))?, + ) + } else { + None + }; + let mut tx = state .database .create_write_transaction(&vault_id) @@ -78,33 +102,13 @@ pub async fn put_file_content( ))); } - let content = match push.content { - PushContent::Snapshot(bytes) => STANDARD - .decode(bytes) - .map_err(|error| client_error(error.into()))?, - - PushContent::Diff(diff) => { - if diff - .iter() - .any(|n| matches!(n, NumberOrText::Number(i64::MIN))) - { - return Err(client_error(anyhow!("Invalid diff length"))); - } - - let parent = parent - .as_ref() - .ok_or_else(|| client_error(anyhow!("Diff requires a parent")))?; - - let text = - str::from_utf8(&parent.content).map_err(|error| client_error(error.into()))?; - - EditedText::from_diff(text, diff, &*BuiltinTokenizer::Word) - .map_err(|error| client_error(error.into()))? - .apply() - .text() - .into_bytes() - } - }; + let content = prepared + .ok_or_else(|| { + server_error(anyhow!( + "Document changed while preparing content; retry the request" + )) + })? + .map_err(client_error)?; let version = StoredDocumentVersion { vault_update_id: Database::allocate_event(&mut tx, push.request_id, &fingerprint) @@ -132,8 +136,65 @@ pub async fn put_file_content( Ok(Json(response)) } +const MAX_DIFF_ITEMS: usize = 10_000; + +fn decode_content( + parent: Option, + content: PushContent, + limit: usize, +) -> anyhow::Result> { + let bytes = match content { + PushContent::Snapshot(encoded) => STANDARD.decode(encoded)?, + PushContent::Diff(diff) => { + anyhow::ensure!( + diff.len() <= MAX_DIFF_ITEMS, + "Diff exceeds operation budget; send a snapshot" + ); + let parent = parent.ok_or_else(|| anyhow!("Diff requires a parent"))?; + let text = str::from_utf8(&parent.content)?; + apply_diff(text, diff, limit)? + } + }; + anyhow::ensure!(bytes.len() <= limit, "Content exceeds size limit"); + Ok(bytes) +} + +/// Each Unicode scalar in the parent is visited at most once. No tokenization +/// or repeated scans of a prefix are needed to apply the wire diff. +fn apply_diff(text: &str, diff: Vec, limit: usize) -> anyhow::Result> { + let mut source = text.chars(); + let mut result = String::new(); + for item in diff { + match item { + NumberOrText::Text(insert) => { + anyhow::ensure!( + insert.len() <= limit.saturating_sub(result.len()), + "Content exceeds size limit" + ); + result.push_str(&insert); + } + NumberOrText::Number(length) => { + for _ in 0..length.unsigned_abs() { + let ch = source + .next() + .ok_or_else(|| anyhow!("Diff exceeds parent length"))?; + if length > 0 { + anyhow::ensure!( + ch.len_utf8() <= limit.saturating_sub(result.len()), + "Content exceeds size limit" + ); + result.push(ch); + } + } + } + } + } + Ok(result.into_bytes()) +} + #[cfg(test)] mod tests { + use reconcile_text::EditedText; use std::path::PathBuf; use super::*; @@ -278,6 +339,29 @@ mod tests { } } + #[tokio::test] + async fn fragmented_diff_budget_rejects_without_committing_an_event() { + let vault = TestVault::new().await; + let document = uuid::Uuid::new_v4(); + let base = accepted( + vault + .send(snapshot(document, None, &"a".repeat(1024 * 1024))) + .await, + ); + let mut push = snapshot(document, Some(base.vault_update_id), ""); + let mut fragments = vec![NumberOrText::Number(0); 10_001]; + fragments.push(NumberOrText::Number(1024 * 1024)); + push.content = PushContent::Diff(fragments); + assert!(matches!( + push.send(&vault.state, &vault.vault).await, + Err(SyncServerError::ClientError(_)) + )); + assert_eq!( + vault.latest(document).await.vault_update_id, + base.vault_update_id + ); + } + #[tokio::test] async fn json_snapshot_preserves_bytes_and_rejects_invalid_base64_without_a_version() { let vault = TestVault::new().await; @@ -363,11 +447,8 @@ mod tests { outcomes => panic!("Expected one accepted push and one stale base: {outcomes:?}"), }; let latest = vault.latest(document).await; - assert_eq!(winner.vault_update_id, rejected.metadata.vault_update_id); - assert_eq!( - STANDARD.decode(rejected.content_base64).unwrap(), - latest.content - ); + assert_eq!(winner.vault_update_id, rejected.vault_update_id); + assert_eq!(rejected.content_size, latest.content.len()); assert!(latest.content == b"left" || latest.content == b"right"); let original = vault .state @@ -577,7 +658,7 @@ mod tests { assert!(matches!( vault.send(foreign_base).await, DocumentUpdateResponse::StaleBase(version) - if version.metadata.vault_update_id == updated.vault_update_id + if version.vault_update_id == updated.vault_update_id )); let binary_document = uuid::Uuid::new_v4(); diff --git a/sync-server/src/server/endpoints/put_file_manifest.rs b/sync-server/src/server/endpoints/put_file_manifest.rs index 98b559d2..05a26e11 100644 --- a/sync-server/src/server/endpoints/put_file_manifest.rs +++ b/sync-server/src/server/endpoints/put_file_manifest.rs @@ -62,18 +62,14 @@ pub async fn put_file_manifest( return Ok(Json(FileManifestUpdateResponse::StaleBase(latest))); } - for id in push.entries.keys() { - if state - .database - .get_latest_document_version(&vault_id, id, Some(&mut tx)) + if let Some(id) = + Database::missing_document(&mut tx, &push.entries.keys().copied().collect::>()) .await .map_err(server_error)? - .is_none() - { - return Err(client_error(anyhow!( - "File manifest references missing content: {id}" - ))); - } + { + return Err(client_error(anyhow!( + "File manifest references missing content: {id}" + ))); } let file_manifest = FileManifest { diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index ad1fe8c0..3a97911b 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -15,7 +15,8 @@ use axum::{ Path, State, ws::{Message, WebSocket, WebSocketUpgrade}, }, - response::Response, + http::StatusCode, + response::{IntoResponse, Response}, }; use futures::StreamExt; use log::{debug, info}; @@ -32,11 +33,22 @@ pub async fn websocket_handler( ) -> Result { debug!("Upgrading WebSocket connection for vault `{vault_id}`"); - Ok(ws.on_upgrade(move |socket| async move { - if let Err(error) = websocket(state, socket, vault_id).await { - debug!("WebSocket disconnected: {error}"); - } - })) + let Some(permit) = state.broadcasts.try_admit(&vault_id) else { + return Ok(( + StatusCode::TOO_MANY_REQUESTS, + "Vault connection limit reached", + ) + .into_response()); + }; + Ok(ws + .max_message_size(1024 * 1024) + .max_frame_size(1024 * 1024) + .on_upgrade(move |socket| async move { + let _permit = permit; + if let Err(error) = websocket(state, socket, vault_id).await { + debug!("WebSocket disconnected: {error}"); + } + })) } async fn websocket( @@ -48,10 +60,15 @@ async fn websocket( let authenticated = get_authenticated_handshake( &state, &vault, - receiver.next().await.transpose().unwrap_or_default(), + tokio::time::timeout(Duration::from_secs(5), receiver.next()) + .await + .map_err(|_| client_error(anyhow::anyhow!("WebSocket handshake deadline expired")))? + .transpose() + .unwrap_or_default(), )?; let device = authenticated.handshake.device_id; + let session = state.cursors.register_connection(&vault, &device).await; let mut after = authenticated .handshake .last_seen_vault_update_id @@ -69,7 +86,7 @@ async fn websocket( let batch = state.database.events_after(&vault, after).await.map_err(server_error)?; if !batch.events.is_empty() { - let head = batch.head_event_id; + let head = batch.end_event_id.unwrap_or(batch.head_event_id); send_update_over_websocket(&WebSocketServerMessage::VaultEvents(batch), &mut sender).await?; after = head; } @@ -92,7 +109,7 @@ async fn websocket( match message { WebSocketClientMessage::Handshake(_) => return Err(client_error(anyhow::anyhow!("Unexpected handshake"))), WebSocketClientMessage::CursorPositions(positions) => { - state.cursors.update_cursors(vault.clone(), authenticated.user.name.clone(), &device, positions.documents_with_cursors).await; + state.cursors.update_cursors(vault.clone(), authenticated.user.name.clone(), &device, session, positions.documents_with_cursors).await; let clients = state.cursors.get_cursors(&vault).await.into_iter().filter(|c| c.device_id != device).collect(); send_update_over_websocket(&WebSocketServerMessage::CursorPositions(CursorPositionFromServer { clients }), &mut sender).await?; } @@ -109,7 +126,7 @@ async fn websocket( state .cursors - .remove_cursors_of_device(&vault, &device) + .remove_cursors_of_device(&vault, &device, session) .await; result diff --git a/sync-server/src/server/responses.rs b/sync-server/src/server/responses.rs index 66c5b5cb..e8d85687 100644 --- a/sync-server/src/server/responses.rs +++ b/sync-server/src/server/responses.rs @@ -2,7 +2,7 @@ use serde::{self, Deserialize, Serialize}; use ts_rs::TS; use crate::app_state::database::models::{ - DocumentVersion, DocumentVersionWithoutContent, FileManifest, VaultUpdateId, + DocumentVersionWithoutContent, FileManifest, VaultUpdateId, }; /// Response to a ping request. @@ -34,7 +34,7 @@ pub enum DocumentUpdateResponse { Accepted(DocumentVersionWithoutContent), /// Nothing was written. The client must incorporate this version before retrying. - StaleBase(DocumentVersion), + StaleBase(DocumentVersionWithoutContent), } #[derive(TS, Debug, Clone, Serialize, Deserialize)] diff --git a/sync-server/src/utils/portable_path.rs b/sync-server/src/utils/portable_path.rs index 7c170848..3c4bf4f8 100644 --- a/sync-server/src/utils/portable_path.rs +++ b/sync-server/src/utils/portable_path.rs @@ -16,7 +16,9 @@ pub fn path_key(path: &str) -> String { } pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { - let mut nodes: BTreeMap = BTreeMap::new(); + // Store one component per node. Expanding every prefix makes a short, + // deeply nested path consume quadratic memory and CPU before any I/O. + let mut nodes: BTreeMap<(usize, String), (usize, &str, bool)> = BTreeMap::new(); for path in entries.values() { ensure!(!path.is_empty(), "Path must not be empty"); @@ -33,6 +35,7 @@ pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { "Reserved sync directory" ); + let mut parent = 0; for (index, part) in parts.iter().enumerate() { ensure!( !part.is_empty() && *part != "." && *part != "..", @@ -73,15 +76,18 @@ pub fn validate_file_manifest(entries: &FileManifestEntries) -> Result<()> { ); } - let prefix = parts[..=index].join("/"); let is_file = index == parts.len() - 1; + let key = (parent, path_key(part)); - if let Some((spelling, previous_is_file)) = nodes.get(&path_key(&prefix)) { - if *spelling != prefix || *previous_is_file || is_file { + if let Some((node, spelling, previous_is_file)) = nodes.get(&key) { + if spelling != part || *previous_is_file || is_file { bail!("Conflicting path: {path}"); } + parent = *node; } else { - nodes.insert(path_key(&prefix), (prefix, is_file)); + let node = nodes.len() + 1; + nodes.insert(key, (node, part, is_file)); + parent = node; } } } @@ -139,6 +145,18 @@ mod tests { } } + #[test] + fn deep_paths_do_not_expand_every_prefix() { + let path = vec!["a"; 10_000].join("/"); + let started = std::time::Instant::now(); + validate_file_manifest(&manifest(&[&path])).unwrap(); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "A 20 KB path exhausted the validation work budget: {:?}", + started.elapsed() + ); + } + #[test] fn rejects_invalid_components_characters_and_non_normalized_paths() { for path in [ -- 2.54.0 From 2fcdbfd27dd0731c4be102b002634846cebba892 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Mon, 21 Sep 2026 21:29:56 +0100 Subject: [PATCH 46/48] no power loss --- SYNC_ENGINE_HANDOVER.md | 5 + docs/architecture/sync-algorithm.md | 53 +- docs/guide/limitations.md | 17 +- frontend/deterministic-tests/README.md | 6 +- .../src/deterministic-agent.ts | 8 +- ...ote-quick-write-and-pending-rename.test.ts | 6 +- ...nding-rename-aliases-second-create.test.ts | 7 +- .../local-client-cli/src/node-filesystem.ts | 23 - frontend/package.json | 4 +- frontend/sync-client/README.md | 116 +- .../file-operations.recovery.test.ts | 428 ------- .../src/file-operations/file-operations.ts | 1123 +++++------------ .../file-operations/filesystem-operations.ts | 24 +- .../sync-client/src/persistence/database.ts | 54 +- .../src/persistence/persistence.ts | 6 +- frontend/sync-client/src/sync-client.ts | 30 +- .../src/sync-operations/content.ts | 3 +- .../sync-operations/cursor-tracker.test.ts | 1 - .../src/sync-operations/local-changes.ts | 8 +- .../sync-operations/outgoing-cursor.test.ts | 1 - .../sync-client/src/sync-operations/scan.ts | 2 +- .../src/sync-operations/syncer.test.ts | 30 +- .../sync-client/src/sync-operations/syncer.ts | 248 +--- .../src/utils/assert-set-contains-exactly.ts | 13 - .../utils/data-structures/min-covered.test.ts | 76 -- .../src/utils/data-structures/min-covered.ts | 61 - frontend/test-support/README.md | 191 +-- frontend/test-support/canonical.ts | 2 +- .../test-support/engine-regressions.test.ts | 393 ++---- frontend/test-support/event-delivery.test.ts | 44 +- .../test-support/file-application.test.ts | 219 ++++ .../journal-interleavings.test.ts | 407 ------ .../test-support/journal-protocol.test.ts | 612 --------- frontend/test-support/oracles.ts | 10 - frontend/test-support/polling.test.ts | 1 - frontend/test-support/protocol.test.ts | 1 - frontend/test-support/real-disk.test.ts | 9 +- frontend/test-support/real-disk.ts | 52 +- frontend/test-support/recovery.test.ts | 317 ++--- .../restart-reconciliation.test.ts | 231 ++++ frontend/test-support/storage.ts | 7 - frontend/test-support/sync-audit.test.ts | 206 ++- frontend/test-support/sync-fixture.ts | 1 - 43 files changed, 1377 insertions(+), 3679 deletions(-) delete mode 100644 frontend/sync-client/src/file-operations/file-operations.recovery.test.ts delete mode 100644 frontend/sync-client/src/utils/assert-set-contains-exactly.ts delete mode 100644 frontend/sync-client/src/utils/data-structures/min-covered.test.ts delete mode 100644 frontend/sync-client/src/utils/data-structures/min-covered.ts create mode 100644 frontend/test-support/file-application.test.ts delete mode 100644 frontend/test-support/journal-interleavings.test.ts delete mode 100644 frontend/test-support/journal-protocol.test.ts create mode 100644 frontend/test-support/restart-reconciliation.test.ts diff --git a/SYNC_ENGINE_HANDOVER.md b/SYNC_ENGINE_HANDOVER.md index aebacadc..9b5a6372 100644 --- a/SYNC_ENGINE_HANDOVER.md +++ b/SYNC_ENGINE_HANDOVER.md @@ -1,3 +1,8 @@ +> Historical handover: its client journaling, recovery-archive and filesystem +> durability descriptions are superseded by the direct-application model in +> [the sync client README](frontend/sync-client/README.md). The current client +> saves complete metadata and reconciles current disk contents after interruption. + # Sync engine v4 handover Prepared on 2026-09-19. This records the implementation delivered in the preceding diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index cfdaa5bc..229e9879 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -1,8 +1,7 @@ # Sync algorithm The current API v5 server stores immutable content versions and a separately -versioned file manifest. Clients reconcile snapshots and apply them through a -durable filesystem journal. The server does not merge competing edits. +versioned file manifest. Clients reconcile snapshots with the current local files. The server does not merge competing edits. ## Identity and ordering @@ -19,10 +18,11 @@ UUID is rejected. ## Reconciliation -A new client reads a coherent vault snapshot, applies it durably, then replays -subsequent events. Replayed pages are folded to their final document heads and -manifest before changing local files. This avoids applying intermediate remote -edits that were already reverted. +A new client reads a coherent vault snapshot, applies it, then replays +subsequent events over HTTP. Pages are folded in memory to their final document +heads and manifest before changing local files. Interrupted fetching restarts from +the last incorporated cursor without saved pagination state. This avoids applying +intermediate remote edits that were already reverted. For text, the client compares the common base, current local bytes and current remote bytes using `reconcile-text`. Configured extensions and valid UTF-8 without @@ -31,34 +31,36 @@ they do not authorize the server to merge a stale base. Large diff operation set are sent as snapshots. If only one side changed, that side is retained. Concurrent unmergeable changes -keep the server version while the journal archives displaced local bytes for -recovery. First contact with existing local text has no shared ancestor and uses +keep the server version. First contact with existing local text has no shared ancestor and uses an empty parent. Concurrent edits may still require human review. Paths use a separate three-way decision: keep a local change when the server path is unchanged, otherwise prefer the server path if both changed. Portable-name collisions receive deterministic conflict names. A manifest deletion is not -reversed merely because another client edited the document; displaced bytes are -retained locally. +reversed merely because another client edited the document. Local backup archives +are not created. ## Filesystem application -Before moving files, the client persists a journal describing sources, -destinations and replacement snapshots. It stages sources before installing any -destination, so swaps and rename cycles do not overwrite each other. Prepared -content is persisted before installation so a crash does not repeat a merge. +The client applies an in-memory plan directly to the visible namespace, then +saves one metadata snapshot. Swaps and cycles move occupied sources to temporary +conflict names that remain ordinary discoverable files if application stops. +There is no persisted filesystem plan, staged payload, recovery replay or fsync +protocol. An interruption may leave a subset of the operations visible, or partial +file content; the next scan reconciles whatever is present then. -External notifications retain logical document identities. Notifications received -during asynchronous filesystem reads or staging are incorporated before accepting -a source. An editor save to a staged document's old path remains part of that -document. Ignored and oversized physical occupants are protected during conflict -allocation. Adapters report atomic editor saves as content updates and suppress -notifications generated by the engine itself. +Create, delete and move notifications persist logical document identities; content +updates only wake the loop to reread current bytes. Namespace changes detected +before mutations invalidate the plan. Notifications during application are incorporated +between files using their document identities; the next scan rereads content. Content is reread +before replacement. Ignored and oversized occupants stay in place. Adapters +report atomic editor saves as updates and suppress engine-generated notifications. -The filesystem and metadata store have separate durability boundaries. Recovery -replays the journal before scanning, including when network sync is disabled. -Adapters must provide complete scans, coherent snapshots, exclusive vault access, -no-replace renames and durable writes. Recovery artifacts are excluded from sync. +Metadata persistence must replace a complete value atomically, leaving a readable +old/new value after interruption. Metadata and user files need not describe the +same moment: scans repair that disagreement. File creation and rename must reject +occupied destinations; scans must be complete and reads coherent. No guarantee is +made that user bytes or rename identities survive power loss. ## Server restores and failures @@ -68,8 +70,7 @@ The client persists and echoes it; the server rejects a checkpoint from a discar history before executing its request. Each new event gets a fresh token even if its numeric ID or request UUID was used on the discarded branch. -After a mismatch, the client durably records recovery intent, retains pending -content, clears old history-dependent state and bootstraps against the restored +After a mismatch, the client records reset intent, clears old history-dependent state and bootstraps against the restored server. Clean files reconcile against the restored snapshot. Unsent edits with an old base are preserved as separate local documents, with conflict names if needed. Interrupted recovery saves are replayable. Backups must be restored with the server diff --git a/docs/guide/limitations.md b/docs/guide/limitations.md index 87824d11..87c8e712 100644 --- a/docs/guide/limitations.md +++ b/docs/guide/limitations.md @@ -9,8 +9,7 @@ VaultLink works well for most Obsidian vaults, but has some constraints you shou Only **`.md`** and **`.txt`** files get automatic conflict-free merging. For other file types, a one-sided change is retained. Concurrent unmergeable -changes keep the server's version and archive displaced local bytes for recovery. -Review recovery artifacts when both devices edit a binary file. +changes keep the server's version. The client does not archive displaced bytes. ### Binary Detection @@ -60,7 +59,7 @@ Rough estimates (varies by vault size and activity): **Growth**: Version history grows with every change. A 10 MB vault with frequent edits might grow to 100+ MB over months. -Active journals, immutable merge bases and request receipts must not be independently deleted. There is no automatic safe history-pruning protocol. +Immutable merge bases and request receipts must not be independently deleted. There is no automatic safe history-pruning protocol. ### Implications @@ -143,6 +142,14 @@ the restored server; unsent edits are preserved as separate local documents. Clients and server must be upgraded together. This does not replace independent backups or protect against arbitrary physical-media corruption. +## Interrupted sync + +The client has no filesystem journal or power-loss durability protocol. Interrupted +writes may leave missing, partial or moved files. On restart, normal scans reconcile +the current files, including edits made while the client was stopped. Metadata +saves must remain complete and readable, so the client can continue syncing; exact +pre-interruption file contents and identities are not guaranteed. + ## Known Edge Cases ### Simultaneous Deletes and Edits @@ -150,10 +157,10 @@ backups or protect against arbitrary physical-media corruption. ``` User A deletes note.md User B edits note.md -Result: Manifest deletion removes the file; unsent edited bytes are retained for recovery. +Result: Manifest deletion removes the file, including unsent local edits. ``` -Content updates do not restore deleted membership. The client retains displaced bytes in its local recovery directory. +Content updates do not restore deleted membership. There is no local recovery archive. ### Large File Uploads diff --git a/frontend/deterministic-tests/README.md b/frontend/deterministic-tests/README.md index a5afbe4e..c137bc72 100644 --- a/frontend/deterministic-tests/README.md +++ b/frontend/deterministic-tests/README.md @@ -74,7 +74,7 @@ Network retry timers remain enabled. Separate fake-clock tests in A barrier checks byte-for-byte replica equality **and** the server's canonical UUID/path map, file bytes, independently computed base/observed hashes, content and manifest CAS parents, event watermark, -contiguous event log, and absence of pending requests/application journals. +contiguous event log, and absence of pending requests. Internal recovery files are excluded from the user-visible file list, but are present in failure disk images. Scenario assertions separately check intended content, document identity, or preservation markers; equality alone is not an @@ -106,8 +106,8 @@ distinct UUIDs at one path. - `wait-for-response-drop`: explicit checkpoint; every armed fault must fire. Every interrupted request must be retried with an identical request ID, target and body before a different CAS is sent. Fault kinds match exactly. -- `rename-next-write`: inject an editor rename after the engine's durable install - rename. The historical name is retained for fixtures; v4 never writes in place. +- `rename-next-write`: inject an editor rename after the engine's content write, + before its metadata save. - `pause-server`, `resume-server`: SIGSTOP/SIGCONT, not a crash. - `crash-server`, `restart-server`: SIGKILL and restart with the same database/port. Server-control scenarios have dedicated processes. diff --git a/frontend/deterministic-tests/src/deterministic-agent.ts b/frontend/deterministic-tests/src/deterministic-agent.ts index 36a029e5..4ef3a838 100644 --- a/frontend/deterministic-tests/src/deterministic-agent.ts +++ b/frontend/deterministic-tests/src/deterministic-agent.ts @@ -43,12 +43,8 @@ export class DeterministicAgent { }); this.disk.boundary = async (label) => { const rename = this.nextWriteRename; - // v4 installs through an exclusive rename, never write-in-place. - if ( - rename && - label.startsWith("durable:rename:") && - label.endsWith(`->${rename.oldPath}`) - ) { + // Inject after the content write, before the metadata save. + if (rename && label === `durable:write:${rename.oldPath}`) { this.nextWriteRename = undefined; await this.rename(rename.oldPath, rename.newPath); } diff --git a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts index 2a3b5de4..e78c2310 100644 --- a/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts +++ b/frontend/deterministic-tests/src/tests/same-doc-id-collapse-after-remote-quick-write-and-pending-rename.test.ts @@ -13,10 +13,8 @@ export const sameDocIdCollapseAfterRemoteQuickWriteAndPendingRenameTest: TestDef steps: [ { type: "enable-sync", client: 0 }, - // Create a deleted latest version before client 1 joins. - // Catch-up will advance MinCovered with a non-contiguous id, - // keeping client 1's create lastSeen low enough to exercise - // the server's same-doc merge path from the e2e failure. + // Seed a deleted document before client 1 bootstraps. Later creates + // at the reused path must still receive separate identities. { type: "create", client: 0, diff --git a/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts b/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts index ac8ed3ed..e3504098 100644 --- a/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts +++ b/frontend/deterministic-tests/src/tests/self-merge-pending-rename-aliases-second-create.test.ts @@ -33,11 +33,8 @@ export const selfMergePendingRenameAliasesSecondCreateTest: TestDefinition = { { type: "enable-sync", client: 1 }, { type: "barrier" }, - // Pause client 0's WS so its MinCovered watermark stays at 0 - // through the whole bug sequence. The merge condition the - // server is going to fire is `creation_vuid > last_seen`; with - // a non-zero gap the same-device second create gets merged - // into the same-device first create. + // Delay socket notifications while client 0 continues local work. + // HTTP catchup must preserve the two generations at the reused path. { type: "pause-websocket", client: 0 }, // Client 1 commits a doc to push the server's vuid above 0. diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index 521f76bd..5902db8a 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -134,7 +134,6 @@ export class NodeFileSystemOperations implements FileSystemOperations { await fs.unlink(fullPath).catch((error: unknown) => { if (!this.isMissing(error)) throw error; }); - await this.syncDirectory(path.dirname(fullPath)); } catch (error) { throw new Error( `Failed to delete file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` @@ -170,34 +169,13 @@ export class NodeFileSystemOperations implements FileSystemOperations { const tmpPath = path.join(tmpDir, `atomic-write-${randomUUID()}.tmp`); try { await fs.writeFile(tmpPath, content, encoding); - const fd = await fs.open(tmpPath, "r"); - try { - await fd.datasync(); - } finally { - await fd.close(); - } await fs.rename(tmpPath, fullPath); - await this.syncDirectory(path.dirname(fullPath)); } catch (error) { await fs.unlink(tmpPath).catch(() => undefined); throw error; } } - // Make the rename durable by fsync'ing the destination's parent directory. - // Skipped on Windows: fsync on a directory handle isn't supported there - private async syncDirectory(dir: string): Promise { - if (process.platform === "win32") { - return; - } - const fd = await fs.open(dir, "r"); - try { - await fd.sync(); - } finally { - await fd.close(); - } - } - private async deleteDirectoryTree(fullPath: string): Promise { const entry = await fs.lstat(fullPath).catch((error: unknown) => { if (this.isMissing(error)) return undefined; @@ -220,7 +198,6 @@ export class NodeFileSystemOperations implements FileSystemOperations { } await fs.rmdir(fullPath); - await this.syncDirectory(path.dirname(fullPath)); } private isMissing(error: unknown): boolean { diff --git a/frontend/package.json b/frontend/package.json index 0bbf94af..9e33d1b0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,8 +19,8 @@ "dev": "concurrently --kill-others \"npm run dev -w sync-client\" \"npm run dev -w obsidian-plugin\"", "test": "npm run test --workspaces", "check:harness": "tsc --noEmit -p test-support/tsconfig.json && tsc --noEmit -p deterministic-tests/tsconfig.json && tsc --noEmit -p test-client/tsconfig.json", - "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/journal-interleavings.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", - "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/journal-protocol.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts'", + "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/file-application.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", + "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/restart-reconciliation.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts'", "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md index 06d680ad..0a5bc956 100644 --- a/frontend/sync-client/README.md +++ b/frontend/sync-client/README.md @@ -4,28 +4,30 @@ The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). messages wake it; complete disk scans and the durable server event log are the sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a positive interval to repair missed notifications. Unset or `0` disables polling. -`syncLocallyCreatedFile`, `syncLocallyUpdatedFile`, and `syncLocallyDeletedFile` -durably record logical file changes before resolving, including while offline. -Shutdown waits for those saves. Notifications describe logical external changes: adapters suppress +Create, delete and move notifications persist logical identity changes before +resolving, including while offline. Shutdown waits for those saves. Content-update +notifications only wake reconciliation; scans read the latest bytes, including after +restart. Notifications describe logical external changes: adapters suppress self-generated create/delete/move notifications, and report atomic editor saves as content updates. This distinguishes a deliberate delete/recreate from an edit. Use `waitUntilFinished()` to await the active attempt; errors leave -pending work durable and are reported to the caller. Transient failures retry; +pending requests saved and are reported to the caller. Transient failures retry; authentication and protocol errors wait for an explicit wake or settings change. ## State and policy The persisted state contains the incorporated server file manifest, the current local identity map, and the exact pending submission. A received server file -manifest is not a merge base until its local application is durable. Successful +manifest becomes the merge base when its application metadata is saved. Successful submissions advance the base to the submitted content snapshot, preserving later local edits. Unknown outcomes are retried unchanged before incorporating newer -remote state. Each contiguous event batch is folded into its final manifest and -content heads before merging. Intermediate changes that the server has already -reverted must not overwrite local edits. +remote state. HTTP is the sole source of remote events; WebSocket messages only +wake the loop. Contiguous event pages are folded in memory into their final manifest +and content heads before merging. An interrupted fetch restarts from the last +incorporated cursor; pagination progress is not persisted. Intermediate changes +that the server has already reverted must not overwrite local edits. -A permanent upload rejection is recorded locally, with content retained for -recovery. An unchanged rejected payload is skipped so other files can sync; the +A permanent upload rejection is recorded in metadata. An unchanged rejected payload is skipped so other files can sync; the failure remains visible in history and `waitUntilFinished()`. Editing it, resetting the client, or changing settings permits a fresh attempt. Earlier attempts that committed despite a lost reply are incorporated through normal event replay. @@ -44,10 +46,9 @@ are implicit; empty directories are not synced. - Content is uploaded before new membership; the client chooses and persists UUIDs. - Text uses three-way `reconcile`. First contact at a matching path adopts the remote ID and reconciles differing text with an empty parent. -- Concurrent binary/unmergeable edits keep the server version and retain - displaced local bytes in recovery. If only one side changed, keep that side. -- Deletion follows the file manifest decision. Displaced edited bytes are retained - locally, without automatically restoring membership. +- Concurrent binary/unmergeable edits keep the server version. If only one side + changed, keep that side. No local backup archive is created. +- Deletion follows the file manifest decision, including when the file has local edits. - Notified moves preserve IDs. Offline inference requires a unique nonempty hash match on both sides; uncertain cases become delete/create. - Ignored and oversized files preserve their disk paths and remote membership. @@ -65,9 +66,8 @@ writes. WebSocket events wake HTTP replay, which validates that checkpoint. A restored database cannot silently reuse a version number: a missing or different event incarnation returns a history mismatch before executing the request. -On mismatch the client first recovers its filesystem journal and scans local -work. It durably records a reset before clearing the old checkpoint, archives -pending content requests, and clears old receipts, cursors, bases and diff-cache +On mismatch the client scans local work and records a reset before clearing the +old checkpoint. It clears old requests, receipts, cursors, bases and diff-cache entries. Clean files reconcile against the restored snapshot. Unsent edits with an old base become separate local documents; path conflicts preserve them under conflict names instead of applying a diff to an unrelated restored version. @@ -79,59 +79,37 @@ Their failures are reported, and remaining observers continue receiving updates. ## Required adapter contract -`FileSystemOperations` and `PersistenceProvider` define the adapter operations -needed by the recovery model. The Obsidian and CLI adapters have intentionally -not been updated in this rewrite. +`FileSystemOperations` and `PersistenceProvider` define the adapter operations. +The Obsidian and CLI adapters still need to adopt the current core interface. +The sync client does not coordinate separate processes accessing the same vault. -The sync client does not coordinate separate processes accessing the same local -vault. +The filesystem adapter must provide complete scans, coherent snapshots, recursive +directory creation, exclusive file creation, no-replace renames, and deletion of +individual regular files. Directory pruning uses only `rmdir`, never recursive +file removal. Reject links, special files and paths escaping the vault. An unreadable +file or directory must fail the scan rather than look like a deletion. Optional +editor cursor metadata is applied with file content. -The filesystem adapter must provide: +User-file writes have no power-loss durability requirement. There is no filesystem +journal, hidden staging area, replay, backup archive, or flush protocol. Swaps and +cycles temporarily move occupied files to visible conflict names; a fresh scan +can discover them if application stops partway through. Partial writes and edits +made before, during or after an interruption are treated as current disk content. +The next sync reconciles that content; it does not finish a saved filesystem plan. +Power loss may lose bytes or rename identity and may cause a merge to run again. -- Complete scans and coherent content snapshots. Missing and unreadable must - be distinguished; reject read races rather than return a partial snapshot. -- Vault-relative operations that reject symlinks, hardlinks, special files, and - ancestor symlink traversal. Staging and visible files share one filesystem. -- Atomic, complete, exclusive file creation (a crash leaves absent or complete - bytes, never a partial file) and atomic renames that **never replace** destinations. - Writes apply optional cursor metadata. -- `flushPaths` flushes existing files and ancestor directories, including parents - of absent paths. Recovery uses it to complete interrupted rename durability. -- Durable recursive directory creation, and directory pruning using only `rmdir` - operations (never unlink descendant files). All affected data/directory entries - must be flushed before a mutation resolves, including intermediate ancestors. +Metadata is the one atomic boundary: persistence `save` replaces a complete value +and an interrupted/failed save leaves an old or new complete value, never a torn +object. A failed save is reloaded before retrying. No ordering between metadata and +user-file durability is required. Settings and engine saves are serialized. +Keep metadata outside the user-file namespace or in `.vault-link-sync`, which is +excluded from scans. Changing a client's vault once it holds identities, offline +notifications, pending requests or bootstrap state requires a separate state store. -Keep the persistence store outside the user file namespace or inside the reserved -`.vault-link-sync` directory, so writing sync state cannot generate user edits. - -Persistence `save` atomically replaces the complete state and resolves only after -its data and directory entry are durable. A failed save may leave either complete -value; the engine reloads before retrying. Settings saves are assumed to succeed -and do not reload after errors. Settings and engine saves are serialized. -Changing a client's vault once it holds identities, offline notifications, -pending requests, or bootstrap/application state requires a separate state store -and client. Lifecycle operations are serialized. - -Before changing visible files, the engine persists an application journal. It -stages every affected source before installing destinations, so swaps and cycles -work. Recovery runs before scanning, including while the server is unavailable. -Prepared content is persisted before installation, preventing repeated merges -following a crash. Unexpected occupants are preserved and given conflict names. -Notifications received while staging rebind the actual source identity before -content is merged. Editor saves to a temporarily staged path continue the original -document's journal, including saves racing preparation and installation. Ignored -or oversized occupants are reserved before conflict allocation. - -`.vault-link-sync/transactions` contains only active recovery artifacts. Staged -sources and prepared output are removed after every install is durably recorded; -completed journals are not archived. `.vault-link-sync/recovery` retains displaced -original bytes and their document/path metadata before staging files are deleted. -`.vault-link-sync/requests` retains rejected submission snapshots for explicit -recovery. These directories are never scanned -as user files. Pending requests and active recovery journals must not be -independently discarded. - -These guarantees assume the adapters/storage honor their durability contracts. -Arbitrary external writers can change a file between observations; detected races -are preserved/reconciled, but unobserved writes and offline identity cannot be -reconstructed perfectly. There is no automated v2/v3 persisted-state migration. +Filesystem application saves metadata once when it finishes. Namespace changes detected +before mutations invalidate the plan. Changes during application are incorporated +between files using their document identities; the next scan rereads content. Content is reread before +replacement. Unexpected eligible occupants receive visible conflict names; +ignored and oversized occupants stay in place. External writers can still change +files between observations, so these checks do not promise transactional file +contents or perfect identity reconstruction. There is no automated v2/v3 state migration. diff --git a/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts b/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts deleted file mode 100644 index 22ed4c8e..00000000 --- a/frontend/sync-client/src/file-operations/file-operations.recovery.test.ts +++ /dev/null @@ -1,428 +0,0 @@ -import assert from "node:assert/strict"; -import { describe, it } from "node:test"; -import { - Database, - emptyState, - type StoredDatabase -} from "../persistence/database"; -import type { ServerConfig } from "../services/server-config"; -import { toStoredSnapshot } from "../sync-operations/content"; -import { isInternalPath } from "../utils/portable-path"; -import { Logger } from "../tracing/logger"; -import { FileOperations, type FileWrite } from "./file-operations"; -import type { - FileSnapshot, - FileSystemOperations -} from "./filesystem-operations"; - -const bytes = (text: string) => new TextEncoder().encode(text); -const text = (snapshot: FileSnapshot) => - new TextDecoder().decode(snapshot.content); -const parent = (path: string) => path.split("/").slice(0, -1).join("/"); - -/** Atomic mutations with an injectable interruption before or after durability. */ -class MemoryFiles implements FileSystemOperations { - public readonly files = new Map(); - public readonly directories = new Set([""]); - public boundary: (label: string) => void = () => {}; - - public change(label: string, action: () => void): void { - this.boundary(`before ${label}`); - action(); - this.boundary(`after ${label}`); - } - - public put(path: string, value: string): void { - for ( - let directory = parent(path); - directory; - directory = parent(directory) - ) - this.directories.add(directory); - this.files.set(path, { content: bytes(value) }); - } - - public async listFilesRecursively(root = ""): Promise { - assert(this.directories.has(root), `Cannot list non-directory ${root}`); - return [...this.files.keys()].filter( - (path) => !root || path.startsWith(`${root}/`) - ); - } - - public async readSnapshot(path: string): Promise { - assert(!this.directories.has(path), `Cannot read directory ${path}`); - return structuredClone(this.files.get(path)); - } - - public async stat(path: string) { - for (let ancestor = parent(path); ancestor; ancestor = parent(ancestor)) - assert( - !this.files.has(ancestor), - `Cannot traverse file ${ancestor}` - ); - const snapshot = this.files.get(path); - if (snapshot) - return { kind: "file" as const, size: snapshot.content.length }; - if (this.directories.has(path)) - return { kind: "directory" as const, size: 0 }; - return undefined; - } - - public async exists(path: string): Promise { - return this.files.has(path) || this.directories.has(path); - } - - public async createDirectory(path: string): Promise { - this.change(`mkdir ${path}`, () => { - for ( - let directory = path; - directory; - directory = parent(directory) - ) { - assert( - !this.files.has(directory), - `File blocks directory ${directory}` - ); - this.directories.add(directory); - } - }); - } - - public async write(path: string, snapshot: FileSnapshot): Promise { - this.change(`write ${path}`, () => { - assert( - !this.files.has(path) && !this.directories.has(path), - `Destination exists: ${path}` - ); - assert( - this.directories.has(parent(path)), - `Parent missing: ${path}` - ); - this.files.set(path, structuredClone(snapshot)); - }); - } - - public async rename(from: string, to: string): Promise { - this.change(`rename ${from} -> ${to}`, () => { - assert(this.files.has(from), `Source missing: ${from}`); - assert( - !this.files.has(to) && !this.directories.has(to), - `Destination exists: ${to}` - ); - assert(this.directories.has(parent(to)), `Parent missing: ${to}`); - this.files.set(to, this.files.get(from)!); - this.files.delete(from); - }); - } - - public async flushPaths(paths: readonly string[]): Promise { - this.change(`flush ${paths.join(",")}`, () => {}); - } - - public async delete(path: string): Promise { - this.change(`rmdir ${path}`, () => { - assert(!this.files.has(path)); - assert( - ![...this.files.keys()].some((file) => - file.startsWith(`${path}/`) - ) - ); - for (const directory of this.directories) - if (directory === path || directory.startsWith(`${path}/`)) - this.directories.delete(directory); - }); - } - - public async deleteFile(path: string): Promise { - this.change(`unlink ${path}`, () => { - assert(!this.directories.has(path)); - this.files.delete(path); - }); - } -} - -async function setup( - contents: Record, - local: Record -) { - const fs = new MemoryFiles(); - for (const [path, value] of Object.entries(contents)) fs.put(path, value); - let durable: StoredDatabase = emptyState("test"); - durable.local = local; - for (const [id, path] of Object.entries(local)) - durable.documents[id] = { - materialized: true, - observedHash: (await toStoredSnapshot(fs.files.get(path)!)).hash - }; - const config = { - getConfig: async () => ({ mergeableFileExtensions: ["md"] }) - } as ServerConfig; - const restart = () => { - const database = new Database( - new Logger(), - structuredClone(durable), - async (next) => { - fs.change("save", () => { - durable = structuredClone(next); - }); - }, - "test", - async () => structuredClone(durable) - ); - return { - database, - operations: new FileOperations(fs, database, config) - }; - }; - return { fs, restart, ...restart() }; -} - -async function scenario() { - const fixture = await setup( - { - "a.md": "A", - "b.md": "B", - "c.md": "C", - "d.md": "Unmanaged", - "e.md": "E", - "folder/keep.md": "Keep", - "deleted.md": "Deleted" - }, - { a: "a.md", b: "b.md", c: "c.md", e: "e.md", deleted: "deleted.md" } - ); - const next = structuredClone(fixture.database.state); - next.local = { - a: "b.md", - b: "a.md", - c: "d.md", - e: "folder", - added: "new.md" - }; - next.documents.added = { materialized: false }; - const writes: Record = { - a: { - expected: await fixture.operations.snapshot("a.md"), - replacement: await toStoredSnapshot({ content: bytes("A updated") }) - }, - added: { - replacement: await toStoredSnapshot({ - content: bytes("Downloaded") - }) - } - }; - return { ...fixture, next, writes }; -} - -async function verifyScenario(fs: MemoryFiles, database: Database) { - assert.equal(database.state.application, undefined); - for (const [id, expected] of Object.entries({ - a: "A updated", - b: "B", - c: "C", - e: "E", - added: "Downloaded" - })) { - const snapshot = fs.files.get(database.state.local[id])!; - assert.equal(text(snapshot), expected, id); - assert.equal(database.state.documents[id].materialized, true, id); - assert.equal( - database.state.documents[id].observedHash, - (await toStoredSnapshot(snapshot)).hash, - id - ); - } - assert.equal(database.state.local.a, "b.md"); - assert.equal(database.state.local.b, "a.md"); - assert.equal(database.state.local.c, "d.md"); - assert.notEqual(database.state.local.e, "folder"); - assert.equal(text(fs.files.get("folder/keep.md")!), "Keep"); - assert.equal(fs.files.has("deleted.md"), false); - assert.equal(database.state.local.deleted, undefined); - const visible = [...fs.files].filter(([path]) => !isInternalPath(path)); - assert.deepEqual( - visible.map(([, value]) => text(value)).sort(), - ["A updated", "B", "C", "Downloaded", "E", "Keep", "Unmanaged"].sort() - ); - assert.equal( - [...fs.files.keys()].some((path) => - path.startsWith(".vault-link-sync/transactions/") - ), - false - ); -} - -describe("filesystem journal recovery", () => { - it("stages swaps, preserves unexpected files/directories, and cleans recovery artifacts", async () => { - const { operations, database, fs, next, writes } = await scenario(); - await operations.apply(next, writes); - await verifyScenario(fs, database); - fs.boundary = () => { - throw new Error("Completed recovery must not write"); - }; - await operations.recover(); - }); - - it("resumes after interruption at every filesystem and metadata mutation boundary", async (t) => { - const baseline = await scenario(); - const boundaries: string[] = []; - baseline.fs.boundary = (label) => boundaries.push(label); - await baseline.operations.apply(baseline.next, baseline.writes); - for (const [index, label] of boundaries.entries()) { - await t.test(`${index}: ${label}`, async () => { - const fixture = await scenario(); - const interrupted = new Error("Interrupted"); - let count = 0; - fixture.fs.boundary = () => { - if (count++ === index) throw interrupted; - }; - await assert.rejects( - fixture.operations.apply(fixture.next, fixture.writes), - (error) => error === interrupted - ); - fixture.fs.boundary = () => {}; - const { database, operations } = fixture.restart(); - if ( - !database.state.application && - database.state.local.a === "a.md" - ) - await operations.apply( - structuredClone(fixture.next), - fixture.writes - ); - else await operations.recover(); - await verifyScenario(fixture.fs, database); - }); - } - }); - - it("merges an edit made after planning exactly once across an interrupted install", async () => { - const base = "First paragraph.\n\nLast paragraph.\n"; - const local = "Local first paragraph.\n\nLast paragraph.\n"; - const remote = "First paragraph.\n\nRemote last paragraph.\n"; - const fixture = await setup({ "note.md": base }, { note: "note.md" }); - const expected = await fixture.operations.snapshot("note.md"); - let edited = false; - const interrupted = new Error("Interrupted after install"); - fixture.fs.boundary = (label) => { - if (!edited && label === "after save") { - fixture.fs.put("note.md", local); - edited = true; - } - if ( - label.startsWith("after rename ") && - label.endsWith(" -> note.md") - ) - throw interrupted; - }; - await assert.rejects( - fixture.operations.apply(structuredClone(fixture.database.state), { - note: { - expected, - replacement: await toStoredSnapshot({ - content: bytes(remote) - }) - } - }), - (error) => error === interrupted - ); - fixture.fs.boundary = () => {}; - const { operations, database } = fixture.restart(); - await operations.recover(); - assert.equal( - text(fixture.fs.files.get("note.md")!), - "Local first paragraph.\n\nRemote last paragraph.\n" - ); - assert.equal( - database.state.documents.note.observedHash, - (await operations.snapshot("note.md"))!.hash - ); - assert.equal( - [...fixture.fs.files.keys()].some((path) => - path.startsWith(".vault-link-sync/transactions/") - ), - false - ); - }); - - it("preserves an external deletion between planning and staging", async () => { - const fixture = await setup( - { "note.md": "Local" }, - { note: "note.md" } - ); - const expected = await fixture.operations.snapshot("note.md"); - fixture.fs.boundary = (label) => { - if (label === "after save") fixture.fs.files.delete("note.md"); - }; - await fixture.operations.apply( - structuredClone(fixture.database.state), - { - note: { - expected, - replacement: await toStoredSnapshot({ - content: bytes("Remote") - }) - } - } - ); - assert.equal(fixture.database.state.local.note, undefined); - assert.equal(fixture.fs.files.has("note.md"), false); - }); - - it("preserves a file created after the destination check", async () => { - const fixture = await setup( - { "source.md": "Incoming" }, - { doc: "source.md" } - ); - const next = structuredClone(fixture.database.state); - next.local.doc = "target.md"; - fixture.fs.boundary = (label) => { - if ( - label.startsWith("before rename ") && - label.endsWith(" -> target.md") - ) - fixture.fs.put("target.md", "Concurrent"); - }; - await assert.rejects( - fixture.operations.apply(next), - /Destination exists/ - ); - fixture.fs.boundary = () => {}; - const { operations, database } = fixture.restart(); - await operations.recover(); - assert.equal(text(fixture.fs.files.get("target.md")!), "Incoming"); - const displaced = Object.entries(database.state.local).find( - ([id]) => id !== "doc" - )!; - assert.equal(text(fixture.fs.files.get(displaced[1])!), "Concurrent"); - }); - - it("preserves an unexpected file blocking a destination's ancestor", async () => { - const fixture = await setup( - { "source.md": "Incoming", target: "Occupant" }, - { doc: "source.md" } - ); - const next = structuredClone(fixture.database.state); - next.local.doc = "target/child.md"; - await fixture.operations.apply(next); - assert.equal( - text(fixture.fs.files.get("target/child.md")!), - "Incoming" - ); - const displaced = Object.entries(fixture.database.state.local).find( - ([id]) => id !== "doc" - )!; - assert.equal(text(fixture.fs.files.get(displaced[1])!), "Occupant"); - }); - - it("handles paths becoming directories and directories becoming files", async () => { - const fixture = await setup( - { a: "A", "b/deep/child": "B" }, - { a: "a", b: "b/deep/child" } - ); - const next = structuredClone(fixture.database.state); - next.local = { a: "b", b: "a/deep/child" }; - await fixture.operations.apply(next); - assert.equal(text(fixture.fs.files.get("b")!), "A"); - assert.equal(text(fixture.fs.files.get("a/deep/child")!), "B"); - }); -}); diff --git a/frontend/sync-client/src/file-operations/file-operations.ts b/frontend/sync-client/src/file-operations/file-operations.ts index 81d1a0f6..49c91dc4 100644 --- a/frontend/sync-client/src/file-operations/file-operations.ts +++ b/frontend/sync-client/src/file-operations/file-operations.ts @@ -1,55 +1,48 @@ import { v4 as uuid } from "uuid"; +import type { FileSystemOperations } from "./filesystem-operations"; import type { - FileSnapshot, - FileSystemOperations -} from "./filesystem-operations"; -import type { - ApplicationJournal, Database, EngineState, - FileStep, StoredSnapshot } from "../persistence/database"; -import { - allocatePortablePath, - INTERNAL_DIRECTORY, - isInternalPath -} from "../utils/portable-path"; +import { allocatePortablePath, isInternalPath } from "../utils/portable-path"; import { mergeContent, fromStoredSnapshot, toStoredSnapshot } from "../sync-operations/content"; import type { ServerConfig } from "../services/server-config"; -import { FileNotFoundError } from "../errors/errors"; import { applyLocalChange, unappliedChanges, type LocalChange } from "../sync-operations/local-changes"; -import { resolvePaths } from "../sync-operations/file-manifest"; +import { + FileNotFoundError, + LocalChangesDuringReconciliation +} from "../errors/errors"; export interface FileWrite { expected?: StoredSnapshot; replacement: StoredSnapshot; } -/** The persisted journal bridges the filesystem and the metadata store. Every - * source is staged before any destination is installed. A phase is saved BEFORE - * an ambiguous rename, so recovery never reapplies a content merge to its output. - */ +/** Files and metadata are independent observations, not a filesystem transaction. + * Interrupted applications are reconciled by the next ordinary scan. */ export class FileOperations { + private physical?: EngineState; + public constructor( public readonly fs: FileSystemOperations, private readonly database: Database, private readonly serverConfig: ServerConfig, private readonly filesApplied: ( paths: readonly string[] - ) => void = () => {}, + ) => void = () => undefined, private readonly localChanges: { entries: () => readonly LocalChange[]; flush: () => Promise; - } = { entries: () => [], flush: async () => {} }, + } = { entries: () => [], flush: async () => undefined }, private readonly protectedFile: ( path: string, size: number @@ -70,715 +63,309 @@ export class FileOperations { } public async snapshot(path: string): Promise { - const value = await this.fs.readSnapshot(path); - return value && toStoredSnapshot(value); - } - - /** Project journal phases onto the physical namespace before binding an - * editor notification. Content hashes must never choose a notified identity. */ - private async physicalState( - journal = this.database.state.application - ): Promise { - const state = structuredClone(journal?.next ?? this.database.state); - if (journal) { - for (const step of journal.steps) { - if (step.superseded) continue; - const origin = step.replacement ? step.output : step.staged; - const path = - step.phase === "installed" - ? step.to - : step.phase === "installing" && - !(await this.fs.exists(origin)) - ? step.to - : (await this.fs.exists(step.staged)) - ? step.staged - : step.from; - if (path) state.local[step.documentId] = path; - else delete state.local[step.documentId]; - } - } - return state; + const snapshot = await this.fs.readSnapshot(path); + return snapshot && toStoredSnapshot(snapshot); } public async captureChange( change: LocalChange, previous: readonly LocalChange[] ): Promise { - const state = await this.physicalState(); - // No notifications may bind provisional identities before the initial - // snapshot is available. Replay the whole ordered queue at bootstrap, - // including moves/deletions after creates. + const state = structuredClone(this.physical ?? this.database.state); if (!state.initialized) return; for (const pending of unappliedChanges(state, previous)) applyLocalChange(state, pending, () => false); - if ( - change.type === "update" && - !Object.values(state.local).includes(change.path) - ) { - // An editor may save to the logical path while its original is in - // staging. Namespace creates/deletes still bind to physical paths. - const step = this.database.state.application?.steps.find( - (step) => - step.from === change.path && step.phase !== "installed" - ); - if (step && isInternalPath(state.local[step.documentId] ?? "")) - state.local[step.documentId] = change.path; - } applyLocalChange(state, change, () => false); } - private async incorporateLocalChanges( - journal: ApplicationJournal - ): Promise { - await this.localChanges.flush(); - const changes = unappliedChanges( - journal.next, - this.localChanges.entries() - ); - if (!changes.length) return; - const physical = await this.physicalState(journal); - for (const change of changes) { - if (change.type === "update") { - const info = await this.fs.stat(change.path); - for (const id of Object.keys(change.identities ?? {})) { - // A save immediately before the rename has already moved - // to staging by the time its notification is incorporated. - const stagedPath = physical.local[id]; - const saved = - info ?? - (stagedPath && isInternalPath(stagedPath) - ? await this.fs.stat(stagedPath) - : undefined); - if ( - saved?.kind === "file" && - this.protectPath( - journal, - physical, - id, - change.path, - saved.size - ) - ) - journal.next.local[id] = change.path; - } - await this.incorporateContentSave(journal, physical, change); - journal.next.excluded = physical.excluded; - journal.next.lastAppliedLocalChangeId = change.changeId; - continue; - } - const before = { ...physical.local }; - const path = - change.type === "move" ? change.relativePath : change.path; - const info = await this.fs.stat(path); - if (info?.kind === "file" && this.protectedFile(path, info.size)) - journal.next.protectedPaths = [ - ...new Set([...(journal.next.protectedPaths ?? []), path]) - ]; - applyLocalChange( - physical, - change, - (path) => - isInternalPath(path) || - this.protectedFile(path, 0) || - journal.next.protectedPaths?.includes(path) === true || - Object.keys(journal.next.excluded ?? {}).some( - (id) => before[id] === path - ) - ); - for (const id of new Set([ - ...Object.keys(before), - ...Object.keys(physical.local) - ])) { - if (before[id] === physical.local[id]) continue; - if (physical.local[id] === undefined) - delete journal.next.local[id]; - else { - const path = physical.local[id]; - journal.next.local[id] = path; - const info = await this.fs.stat(path); - this.protectPath( - journal, - physical, - id, - path, - info?.size ?? 0 - ); - } - journal.next.documents[id] ??= physical.documents[id]; - } - journal.next.excluded = physical.excluded; - journal.next.lastAppliedLocalChangeId = change.changeId; - } - journal.next.local = resolvePaths( - journal.next.local, - journal.next.fileManifest.entries, - Object.keys(journal.next.excluded ?? {}), - journal.next.protectedPaths - ); - for (const step of journal.steps) { - if (step.superseded) continue; - // A notified move can exclude an input after this transaction was - // planned. Preserve its bytes as well as its chosen physical path. - if (journal.next.excluded?.[step.documentId] !== undefined) { - const origin = step.replacement ? step.output : step.staged; - if ( - step.phase === "installing" && - !(await this.fs.exists(origin)) - ) - step.phase = "installed"; - if (step.replacement && step.phase !== "installed") { - // contentWrite advanced the proposed merge context before - // applying bytes. Cancelling that application must retain - // the last incorporated base so reenabling the file still - // reconciles the deferred remote head. An installed output, - // including one whose acknowledgement was lost, keeps its - // new base for subsequent local edits. - const previous = - this.database.state.documents[step.documentId]; - Object.assign( - journal.next.documents[step.documentId], - structuredClone({ - base: previous?.base, - bootstrap: previous?.bootstrap, - recoveryBase: previous?.recoveryBase - }) - ); - } - step.replacement = undefined; - } - if (step.phase === "installed") { - step.to = physical.local[step.documentId]; - continue; - } - if (step.phase === "planned") { - const staged = await this.fs.exists(step.staged); - const from = physical.local[step.documentId]; - if (staged && from !== step.staged) { - // A notified move/delete and path reuse raced the source - // rename. Bind the staged object to the new occupant, then - // stage the original identity from its notified location. - const occupant = Object.keys(physical.local).find( - (id) => physical.local[id] === step.from - ); - const originalStaged = step.staged; - step.staged = `${step.staged}-${uuid()}`; - if (occupant) { - journal.steps.push({ - documentId: occupant, - from: step.from, - to: journal.next.local[occupant], - staged: originalStaged, - output: `${originalStaged}.output`, - phase: "staged" - }); - physical.local[occupant] = originalStaged; - } else { - // Retain an ambiguous object without merging it into a - // different document. Cleanup archives this source. - journal.steps.push({ - ...step, - staged: originalStaged, - to: undefined, - replacement: undefined, - phase: "installed", - superseded: true - }); - } - } - if ( - !staged || - (from !== undefined && !isInternalPath(from)) || - from === undefined - ) { - if (step.from && !from) step.replacement = undefined; - step.from = from; - } - } - step.to = journal.next.local[step.documentId]; - } - for (const [id, from] of Object.entries(physical.local)) { - const to = journal.next.local[id]; - if ( - from === to || - isInternalPath(from) || - journal.steps.some( - (step) => - step.documentId === id && step.phase !== "installed" - ) - ) - continue; - const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${id}-${uuid()}`; - journal.steps.push({ - documentId: id, - from, - to, - expected: await this.snapshot(from), - staged: `${prefix}.source`, - output: `${prefix}.output`, - phase: "planned" - }); - } - await this.persist(journal); - } - - private protectPath( - journal: ApplicationJournal, - physical: EngineState, - id: string, - path: string, - size: number - ): boolean { - if ( - !this.protectedFile(path, size) && - !journal.next.protectedPaths?.includes(path) - ) - return false; - physical.excluded ??= {}; - physical.excluded[id] ??= - this.database.state.fileManifest.entries[id] ?? null; - journal.next.protectedPaths = [ - ...new Set([...(journal.next.protectedPaths ?? []), path]) - ]; - return true; - } - - private async incorporateContentSave( - journal: ApplicationJournal, - physical: EngineState, - change: LocalChange & { type: "update" } - ): Promise { - for (const id of Object.keys(change.identities ?? {})) { - const step = journal.steps.find( - (step) => - step.documentId === id && - step.phase !== "installed" && - !step.superseded - ); - if ( - !step || - !(await this.fs.exists(step.staged)) || - (await this.fs.stat(change.path))?.kind !== "file" - ) - continue; - const moved = - step.phase === "installing" && - !(await this.fs.exists( - step.replacement ? step.output : step.staged - )); - if (moved && step.to === change.path) continue; - // Stage the editor's save as a continuation of the SAME identity. - // The previous source stays in the journal until durable cleanup. - const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${id}-${uuid()}`; - if (moved && step.to) { - // Installation won the race with a save to the old editor path. - // Stage its output before reinstalling this identity's merge. - journal.steps.push({ - documentId: id, - from: step.to, - staged: `${prefix}.previous`, - output: `${prefix}.unused`, - phase: "planned", - superseded: true - }); - } - journal.steps.push({ - documentId: id, - from: change.path, - to: step.to, - expected: step.expected, - replacement: step.replacement, - staged: `${prefix}.source`, - output: `${prefix}.output`, - phase: "planned" - }); - step.to = undefined; - step.replacement = undefined; - step.phase = "installed"; - step.superseded = true; - physical.local[id] = change.path; - } - } - - private directory(path: string): string { - return path.split("/").slice(0, -1).join("/"); - } - - private async writeOnce( - path: string, - snapshot: FileSnapshot - ): Promise { - const existing = await this.fs.readSnapshot(path); - if (existing) { - const actual = await toStoredSnapshot(existing); - const expected = await toStoredSnapshot(snapshot); - if (actual.contentBase64 !== expected.contentBase64) - throw new Error(`Recovery artifact changed: ${path}`); - await this.fs.flushPaths([path]); - return; - } - await this.ensureParents(path); - await this.fs.write(path, snapshot); - } - - private async writeJsonOnce(path: string, value: unknown): Promise { - await this.writeOnce(path, { - content: new TextEncoder().encode(JSON.stringify(value)) - }); - } - - private async persist(journal: ApplicationJournal): Promise { - await this.database.commit({ - ...this.database.state, - application: journal - }); - } - - public async retainSnapshot( - requestId: string, - documentId: string, - path: string | undefined, - snapshot: StoredSnapshot - ): Promise { - const prefix = `${INTERNAL_DIRECTORY}/requests/${requestId}`; - await this.writeOnce(`${prefix}.content`, fromStoredSnapshot(snapshot)); - const metadata = await this.fs.readSnapshot(`${prefix}.json`); - if (metadata) { - const saved: unknown = JSON.parse( - new TextDecoder().decode(metadata.content) - ); - if ( - typeof saved !== "object" || - saved === null || - !("documentId" in saved) || - saved.documentId !== documentId || - !("hash" in saved) || - saved.hash !== snapshot.hash - ) - throw new Error(`Recovery artifact changed: ${prefix}.json`); - await this.fs.flushPaths([`${prefix}.json`]); - return; // originalPath is a hint recorded once, not request identity. - } - await this.writeJsonOnce(`${prefix}.json`, { - documentId, - originalPath: path, - hash: snapshot.hash - }); - } - public async apply( next: EngineState, - writes: Record = {}, - validatePlan: () => void = () => {} + writes: Partial> = {}, + validatePlan: () => void = () => undefined ): Promise { - if (this.database.state.application) - throw new Error( - "Recover the existing filesystem transaction first" - ); - - const id = uuid(); - const steps: FileStep[] = []; - for (const documentId of new Set([ - ...Object.keys(this.database.state.local), - ...Object.keys(next.local) - ])) { - const oldPath = this.database.state.local[documentId]; - const newPath = next.local[documentId]; - const materialized = - this.database.state.documents[documentId]?.materialized; - const from = materialized ? oldPath : undefined; - const write = writes[documentId]; - if (!write && (from === undefined || from === newPath)) continue; - const expected = - write?.expected ?? - (from ? await this.snapshot(from) : undefined); - // An external deletion is a local change, not permission to resurrect - // the file using a stale network response. - if (from && !expected && !write) { - delete next.local[documentId]; - continue; - } - const prefix = `${INTERNAL_DIRECTORY}/transactions/${id}/${documentId}`; - steps.push({ - documentId, - from, - to: newPath, - expected, - replacement: write?.replacement, - staged: `${prefix}.source`, - output: `${prefix}.output`, - phase: "planned" - }); - } - validatePlan(); - if (steps.length === 0) { - const changedPaths = this.changedPaths( - this.database.state.local, - next.local - ); - await this.database.commit(next); - this.filesApplied(changedPaths); - return; - } - - const journal = { - id, - next: structuredClone(next), - steps, - extensions: (await this.serverConfig.getConfig()) - .mergeableFileExtensions + await this.localChanges.flush(); + const changes = this.localChanges.entries().length; + let applying = false; + const guard = (): void => { + // Once mutations begin, incorporate bound notifications between files + // and commit the batch before the next scan reads fresh content. + if (applying) return; + validatePlan(); + if (this.localChanges.entries().length > changes) + throw new LocalChangesDuringReconciliation(); }; - validatePlan(); - await this.persist(journal); - await this.recover(); - } - - public async recover(): Promise { - const saved = this.database.state.application; - if (!saved) return; - - const journal = structuredClone(saved); - await this.fs.createDirectory( - `${INTERNAL_DIRECTORY}/transactions/${journal.id}` - ); - - for (;;) { - await this.incorporateLocalChanges(journal); - // Stage every pending source before installing anything, including - // occupants discovered during an earlier installation attempt. - await this.stageSources(journal); - const step = journal.steps.find( - (step) => step.phase !== "installed" - ); - if (!step) break; - await this.prepareContent(journal, step); - await this.incorporateLocalChanges(journal); - if (step.superseded && step.phase === "installed") continue; - try { - await this.install(journal, step); - } catch (error) { - const code = - typeof error === "object" && - error !== null && - "code" in error && - typeof error.code === "string" - ? error.code + const beginMutation = (): void => { + guard(); + applying = true; + }; + guard(); + const physical = structuredClone(this.database.state); + physical.initialized = next.initialized; + this.physical = physical; + const paths = new Set(); + try { + for (const id of new Set([ + ...Object.keys(physical.local), + ...Object.keys(next.local) + ])) { + await this.incorporateChanges(next, physical, guard); + let from = + Object.hasOwn(physical.documents, id) && + physical.documents[id].materialized + ? physical.local[id] : undefined; - if ( - step.to === undefined || - ![ - "ENAMETOOLONG", - "EINVAL", - "ENOTSUP", - "EOPNOTSUPP", - "EACCES", - "EPERM" - ].includes(code ?? "") - ) - throw error; - const origin = step.replacement ? step.output : step.staged; - // A rename that took effect must be acknowledged at its original - // destination. Never infer failure from a failed durability flush. - if ( - step.phase === "installing" && - !(await this.fs.exists(origin)) - ) - throw error; - if (step.replanned === true) throw error; - const extension = - /\.[^.]*$/u.exec(step.to.split("/").at(-1) ?? "")?.[0] ?? - ""; - const wanted = `Recovered ${step.documentId}${extension}`; - step.to = await this.conflictPath( - journal, - step.documentId, - wanted - ); - step.replanned = true; - journal.next.local[step.documentId] = step.to; - // The payload has already been prepared; persist the new target - // before retrying so crashes cannot reapply its content merge. - step.phase = "prepared"; - await this.persist(journal); - } - } - - // Once every install intent is durable, the journal can be replayed - // without its staged inputs. Remove transient bytes before committing - // the next engine state so an interrupted cleanup is retried. - await this.cleanup(journal); - await this.database.commit(journal.next); - this.filesApplied( - [ - ...new Set( - journal.steps.flatMap((step) => [step.from, step.to]) - ) - ].filter((path): path is string => path !== undefined) - ); - } - - private async stageSources(journal: ApplicationJournal): Promise { - for (;;) { - await this.incorporateLocalChanges(journal); - const step = journal.steps.find((step) => step.phase === "planned"); - if (!step) break; - // An existing staging file proves the move completed before a crash. - if (step.from && !(await this.fs.exists(step.staged))) { - const { from } = step, - { staged } = step; - const present = await this.fs.exists(from); - await this.incorporateLocalChanges(journal); - if ( - step.from !== from || - step.staged !== staged || - step.phase !== "planned" - ) + let to = Object.hasOwn(next.local, id) + ? next.local[id] + : undefined; + const write = writes[id]; + if (write === undefined && (from === undefined || from === to)) continue; - if (present) { - await this.fs.rename(from, staged); - await this.incorporateLocalChanges(journal); - if ( - step.from !== from || - step.staged !== staged || - step.phase !== "planned" - ) - continue; - } else { - // Preserve an external deletion instead of resurrecting it. - delete journal.next.local[step.documentId]; - step.to = undefined; - step.replacement = undefined; - } - } - if (step.from) await this.fs.flushPaths([step.from, step.staged]); - step.phase = "staged"; - await this.persist(journal); - } - // Directory cleanup is also retried if staging completed before a crash. - for (const step of journal.steps) - await this.removeEmptyParents(step.from); - } - - private async prepareContent( - journal: ApplicationJournal, - step: FileStep - ): Promise { - if (step.phase !== "staged") return; - const source = await this.snapshot(step.staged); - if (source) { - if (step.replacement && step.expected) - step.replacement = await mergeContent( - step.to ?? step.from ?? "", - step.expected, - source, - step.replacement, - journal.extensions - ); - // The prepared replacement now includes this source's edits. A - // later save must be merged relative to this source, or superseded - // local text is reintroduced from the already merged replacement. - step.expected = source; - } - // Save the merged result before writing it, so recovery never merges twice. - step.phase = "prepared"; - await this.persist(journal); - } - - private async install( - journal: ApplicationJournal, - step: FileStep - ): Promise { - if (step.to) { - const origin = step.replacement ? step.output : step.staged; - const alreadyMoved = - step.phase === "installing" && !(await this.fs.exists(origin)); - if (alreadyMoved) { - if (!(await this.fs.exists(step.to))) { - // The installed file was externally deleted before - // recovery. Preserve that deletion instead of failing or - // resurrecting bytes from metadata. - delete journal.next.local[step.documentId]; - step.to = undefined; - step.replacement = undefined; - step.phase = "installed"; - await this.persist(journal); - return; - } - await this.fs.flushPaths([origin, step.to]); - } else { - if (step.replacement) - await this.writeOnce( - step.output, - fromStoredSnapshot(step.replacement) + if (from !== undefined) paths.add(from); + const info = + from !== undefined ? await this.fs.stat(from) : undefined; + guard(); + if ( + from !== undefined && + info?.kind === "file" && + this.protectedFile(from, info.size) + ) { + next.local[id] = from; + next.documents[id] = structuredClone( + physical.documents[id] ); - await this.incorporateLocalChanges(journal); - if (step.superseded) return; - if (!(await this.clearDestination(journal, step, step.to))) - return; - await this.ensureParents(step.to); - await this.incorporateLocalChanges(journal); - if (step.superseded) return; - // Persist intent first; the no-replace rename protects against - // another writer taking the destination after we checked it. - step.phase = "installing"; - await this.persist(journal); - await this.fs.rename(origin, step.to); - await this.incorporateLocalChanges(journal); - if (step.superseded) return; - } - const document = journal.next.documents[step.documentId]; - if (document) { - document.materialized = true; - document.observedHash = - step.replacement?.hash ?? - (await this.snapshot(step.to))?.hash; + (next.excluded ??= {})[id] = + physical.fileManifest.entries[id] ?? null; + (next.protectedPaths ??= []).push(from); + continue; + } + if (from !== undefined && info?.kind !== "file") { + Reflect.deleteProperty(next.local, id); + continue; + } + if (to === undefined) { + if (from !== undefined) { + beginMutation(); + await this.fs.deleteFile(from); + Reflect.deleteProperty(physical.local, id); + await this.removeEmptyParents(from); + } + continue; + } + // Move an occupant aside in the visible namespace. This breaks + // cycles without hiding source files in a recovery directory. + if (from !== to) { + try { + to = await this.clearDestination( + next, + physical, + id, + to, + guard, + beginMutation + ); + await this.ensureParents(to); + } catch (error) { + to = await this.fallbackPath(next, id, to, error); + await this.ensureParents(to); + } + next.local[id] = to; + // Clearing the destination may itself have moved this source + // (for example, a file becoming its own parent directory). + from = + Object.hasOwn(physical.documents, id) && + physical.documents[id].materialized + ? physical.local[id] + : undefined; + if (from !== undefined) { + guard(); + // Bind notifications delivered from inside rename to the + // destination identity. A failed rename aborts this plan. + beginMutation(); + physical.local[id] = to; + await this.fs.rename(from, to); + paths.add(to); + await this.removeEmptyParents(from); + from = to; + } + } + if (write !== undefined) { + const current = + from !== undefined + ? await this.snapshot(from) + : undefined; + guard(); + // A detected deletion is a local edit, not an invitation to + // recreate a previously materialized document. + if (from !== undefined && !current) { + Reflect.deleteProperty(next.local, id); + continue; + } + const replacement = + current && write.expected + ? await mergeContent( + to, + write.expected, + current, + write.replacement, + (await this.serverConfig.getConfig()) + .mergeableFileExtensions + ) + : write.replacement; + guard(); + if (current?.hash !== replacement.hash) { + beginMutation(); + if (current) await this.fs.deleteFile(to); + guard(); + physical.local[id] = to; + physical.documents[id] = { + ...next.documents[id], + materialized: true + }; + await this.fs.write( + to, + fromStoredSnapshot(replacement) + ); + } + next.documents[id].observedHash = replacement.hash; + paths.add(to); + } + next.documents[id].materialized = true; + physical.local[id] = to; + physical.documents[id] = structuredClone(next.documents[id]); } + await this.incorporateChanges(next, physical, guard); + guard(); + await this.database.commit(next); + } finally { + this.physical = undefined; + this.filesApplied([...paths]); + } + } + + private async incorporateChanges( + next: EngineState, + physical: EngineState, + guard: () => void + ): Promise { + await this.localChanges.flush(); + guard(); + for (const change of unappliedChanges( + next, + this.localChanges.entries() + )) { + const ignored = (path: string): boolean => + isInternalPath(path) || this.protectedFile(path, 0); + applyLocalChange(physical, change, ignored); + applyLocalChange(next, change, ignored); + physical.lastAppliedLocalChangeId = next.lastAppliedLocalChangeId = + change.changeId; } - // Deletions and completed renames share this path whether their - // acknowledgement was received or lost. Transient bytes are removed - // only after every step reaches this phase. - step.phase = "installed"; - await this.persist(journal); } private async clearDestination( - journal: ApplicationJournal, - step: FileStep, - path: string - ): Promise { - const obstruction = await this.obstruction(path); - if (!obstruction) return true; - if (obstruction.kind === "file") { + next: EngineState, + physical: EngineState, + id: string, + path: string, + guard: () => void, + beginMutation: () => void + ): Promise { + for (;;) { + const obstruction = await this.obstruction(path); + if (!obstruction) return path; + guard(); const info = await this.fs.stat(obstruction.path); if ( - journal.next.protectedPaths?.includes(obstruction.path) === - true || - (info !== undefined && - this.protectedFile(obstruction.path, info.size)) + obstruction.kind === "file" && + info && + next.protectedPaths?.includes(obstruction.path) !== true && + !this.protectedFile(obstruction.path, info.size) ) { - (journal.next.protectedPaths ??= []).push(obstruction.path); - step.to = await this.conflictPath( - journal, - step.documentId, - path + await this.moveAside( + next, + physical, + obstruction.path, + false, + beginMutation ); - journal.next.local[step.documentId] = step.to; - await this.persist(journal); - } else await this.displace(journal, obstruction.path); - } else if ( - (await this.fs.listFilesRecursively(obstruction.path)).length - ) { - // Preserve unmanaged descendants by changing the incoming file's name. - step.to = await this.conflictPath(journal, step.documentId, path); - journal.next.local[step.documentId] = step.to; - await this.persist(journal); - } else { - // The adapter only removes empty directories, including on a race. - await this.fs.delete(obstruction.path); - return true; + } else if (obstruction.kind === "directory") { + const children = await this.fs.listFilesRecursively( + obstruction.path + ); + let movable = true; + for (const child of children) { + const occupant = Object.keys(physical.local).find( + (other) => physical.local[other] === child + ); + const childInfo = await this.fs.stat(child); + if ( + occupant === undefined || + !Object.hasOwn(physical.documents, occupant) || + !physical.documents[occupant].materialized || + next.local[occupant] === child || + next.protectedPaths?.includes(child) === true || + childInfo === undefined || + this.protectedFile(child, childInfo.size) + ) { + movable = false; + break; + } + } + if (movable) { + // Only evacuate tracked sources already scheduled to leave. + // Root-level temporary names also break directory/file cycles. + for (const child of children) + await this.moveAside( + next, + physical, + child, + true, + beginMutation + ); + beginMutation(); + await this.fs.delete(obstruction.path); + } else path = await this.conflictPath(next, id, path); + } else { + path = await this.conflictPath(next, id, path); + } } - // The journal now contains a new source or destination. Retry it through - // the main loop so newly discovered sources are staged before installing. - return false; + } + + private async moveAside( + next: EngineState, + physical: EngineState, + path: string, + toRoot: boolean, + beginMutation: () => void + ): Promise { + const occupant = + Object.keys(physical.local).find( + (other) => + Object.hasOwn(physical.documents, other) && + physical.documents[other].materialized && + physical.local[other] === path + ) ?? uuid(); + const displaced = await this.conflictPath( + next, + occupant, + toRoot ? (path.split("/").at(-1) ?? path) : path + ); + await this.ensureParents(displaced); + beginMutation(); + physical.local[occupant] = displaced; + await this.fs.rename(path, displaced); + if (!Object.hasOwn(physical.documents, occupant)) { + const snapshot = await this.snapshot(displaced); + physical.documents[occupant] = { + materialized: true, + observedHash: snapshot?.hash + }; + next.documents[occupant] = structuredClone( + physical.documents[occupant] + ); + next.local[occupant] = displaced; + } else if (next.local[occupant] === path) + next.local[occupant] = displaced; } private async obstruction( @@ -791,126 +378,68 @@ export class FileOperations { if (info && (info.kind === "file" || i === parts.length - 1)) return { path: prefix, kind: info.kind }; } - return undefined; } private async conflictPath( - journal: ApplicationJournal, - documentId: string, + next: EngineState, + id: string, path: string ): Promise { - const occupied = { ...journal.next.local }; - delete occupied[documentId]; - for (const [index, diskPath] of ( + const occupied = { ...next.local }; + Reflect.deleteProperty(occupied, id); + for (const [i, diskPath] of ( await this.listFilesRecursively() ).entries()) - occupied[`disk-${index}`] = diskPath; - return allocatePortablePath(path, documentId, occupied); + occupied[`disk-${i}`] = diskPath; + return allocatePortablePath(path, id, occupied); } - private async displace( - journal: ApplicationJournal, - path: string - ): Promise { - const unexpected = await this.snapshot(path); - if (!unexpected) - throw new Error(`Destination changed during recovery: ${path}`); - const documentId = uuid(); - const to = await this.conflictPath(journal, documentId, path); - const prefix = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/${documentId}`; - journal.steps.push({ - documentId, - from: path, - to, - expected: unexpected, - staged: `${prefix}.source`, - output: `${prefix}.output`, - phase: "planned" - }); - journal.next.local[documentId] = to; - journal.next.documents[documentId] = { - materialized: true, - observedHash: unexpected.hash - }; - await this.persist(journal); + private async fallbackPath( + next: EngineState, + id: string, + path: string, + error: unknown + ): Promise { + const code = + typeof error === "object" && error !== null && "code" in error + ? error.code + : undefined; + if ( + ![ + "ENAMETOOLONG", + "EINVAL", + "ENOTSUP", + "EOPNOTSUPP", + "EACCES", + "EPERM" + ].includes(String(code)) + ) + throw error; + const extension = + /\.[^.]*$/u.exec(path.split("/").at(-1) ?? "")?.[0] ?? ""; + return this.conflictPath(next, id, `Recovered ${id}${extension}`); } private async ensureParents(path: string): Promise { - const parent = this.directory(path); + const parent = path.split("/").slice(0, -1).join("/"); if (parent) await this.fs.createDirectory(parent); } - private async removeEmptyParents(path?: string): Promise { - if (!path) return; - let parent = this.directory(path); + private async removeEmptyParents(path: string): Promise { + let parent = path.split("/").slice(0, -1).join("/"); while (parent && !isInternalPath(parent)) { - // An earlier installation may have turned an ancestor into a file. const info = await this.obstruction(parent); - if (!info) { - parent = this.directory(parent); - continue; - } - if (info.kind !== "directory") break; - if ((await this.fs.listFilesRecursively(parent)).length) break; - // delete() must reject non-empty directories, including unmanaged entries. + if ( + info?.kind !== "directory" || + (await this.fs.listFilesRecursively(parent)).length + ) + break; try { await this.fs.delete(parent); } catch { break; } - parent = this.directory(parent); + parent = parent.split("/").slice(0, -1).join("/"); } } - - private async cleanup(journal: ApplicationJournal): Promise { - for (const step of journal.steps) { - const source = await this.snapshot(step.staged); - if ( - source && - (!step.to || - (step.replacement && source.hash !== step.replacement.hash)) - ) { - const prefix = `${INTERNAL_DIRECTORY}/recovery/${journal.id}-${step.staged.split("/").at(-1)}`; - await this.writeOnce( - `${prefix}.content`, - fromStoredSnapshot(source) - ); - await this.writeJsonOnce(`${prefix}.json`, { - documentId: step.documentId, - originalPath: step.from, - hash: source.hash - }); - } - for (const path of [ - step.staged, - step.output, - `${step.staged}.recovery`, - `${step.staged}.json` - ]) { - if (await this.fs.exists(path)) await this.fs.deleteFile(path); - } - } - const completed = `${INTERNAL_DIRECTORY}/transactions/${journal.id}/completed.json`; - if (await this.fs.exists(completed)) - await this.fs.deleteFile(completed); - await this.fs.delete( - `${INTERNAL_DIRECTORY}/transactions/${journal.id}` - ); - } - - private changedPaths( - before: Readonly>, - after: Readonly> - ): string[] { - const result = new Set(); - for (const id of new Set([ - ...Object.keys(before), - ...Object.keys(after) - ])) { - if (before[id] === after[id]) continue; - if (before[id] !== undefined) result.add(before[id]); - if (after[id] !== undefined) result.add(after[id]); - } - return [...result]; - } } diff --git a/frontend/sync-client/src/file-operations/filesystem-operations.ts b/frontend/sync-client/src/file-operations/filesystem-operations.ts index f4e79397..d8fcf0ab 100644 --- a/frontend/sync-client/src/file-operations/filesystem-operations.ts +++ b/frontend/sync-client/src/file-operations/filesystem-operations.ts @@ -6,10 +6,9 @@ export interface FileSnapshot { cursors?: CursorPosition[]; } -/** Required adapter contract for API v4; there is deliberately no weak fallback. +/** Filesystem operations act on the current user-owned namespace. * Paths are vault-relative. Never follow symlinks (including ancestor symlinks). * Only independent regular files are supported; reject hardlinks/special files. - * Every mutation must flush file data and affected directories before resolving. * A successful scan is complete: unreadable directories must reject, not vanish. */ export interface FileSystemOperations { @@ -22,24 +21,15 @@ export interface FileSystemOperations { ) => Promise<{ kind: "file" | "directory"; size: number } | undefined>; /** Includes directories. */ exists: (path: string) => Promise; - /** Recursive, idempotent, durable directory creation. */ + /** Recursive, idempotent directory creation. */ createDirectory: (path: string) => Promise; - /** Atomically create a complete file: never replace an existing object. - * A crash leaves the path absent or complete, never a partial file. - * Flush data and directory entries before resolving. */ + /** Create a file exclusively; never overwrite a raced destination. + * Writes apply optional editor cursor metadata. No durability guarantee. */ write: (path: string, snapshot: FileSnapshot) => Promise; - /** Atomic, durable rename on the same filesystem; fail if destination exists. - * Flush the source's data and both affected directories before resolving. */ + /** Move a file without replacing an existing destination. */ rename: (from: string, to: string) => Promise; - /** Flush existing files and all affected ancestor directories, even when a - * named file is absent. Recovery uses this after a mutation became visible - * before its caller learned that it was durable. */ - flushPaths: (paths: readonly string[]) => Promise; - /** Idempotently unlink one regular file and flush its parent directory. - * The sync engine uses this only for its own internal recovery artifacts. */ + /** Idempotently unlink one regular file. */ deleteFile: (path: string) => Promise; - /** Prune an implicit directory tree using rmdir only, durably. Reject - * regular files and never unlink descendant files; reject if files remain. - * Partial removal of empty child directories on failure is harmless. */ + /** Prune directories using rmdir only. Reject files and nonempty directories. */ delete: (path: string) => Promise; } diff --git a/frontend/sync-client/src/persistence/database.ts b/frontend/sync-client/src/persistence/database.ts index 750eb667..ec47430d 100644 --- a/frontend/sync-client/src/persistence/database.ts +++ b/frontend/sync-client/src/persistence/database.ts @@ -1,5 +1,4 @@ import type { CursorPosition } from "reconcile-text"; -import type { Logger } from "../tracing/logger"; import type { DocumentUpdateResponse } from "../services/types/DocumentUpdateResponse"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; import type { FileManifest } from "../services/types/FileManifest"; @@ -7,7 +6,6 @@ import type { FileManifestUpdateResponse } from "../services/types/FileManifestU import type { PushFileManifest } from "../services/types/PushFileManifest"; import type { PutFileContent } from "../services/types/PutFileContent"; import type { VaultSnapshot } from "../services/types/VaultSnapshot"; -import type { EventRecord } from "../services/types/EventRecord"; import type { FileManifestEntries } from "../types/file-manifest-entries"; export type VaultUpdateId = number; @@ -23,7 +21,6 @@ export interface StoredSnapshot { export interface DocumentMetadata { parentVersionId: number; hash: string; - remoteRelativePath?: string; } export interface DocumentState { @@ -54,7 +51,7 @@ export type PendingRequest = { rejection?: string } & ( ); export interface EngineState { - /** Durable reset intent; clear the transport checkpoint before bootstrap. */ + /** Persisted reset intent; clear the transport checkpoint before bootstrap. */ historyRecovery?: boolean; vaultKey: string; initialized: boolean; @@ -64,17 +61,10 @@ export interface EngineState { lastSeenUpdateId: number; remoteHeads: Record; bootstrap?: VaultSnapshot; - /** Fold pages durably without applying intermediate remote snapshots. */ - eventReplay?: { - after: number; - manifest: FileManifest; - heads: Record; - receipts: EventRecord[]; - }; pending?: PendingRequest; /** A rejected retry does not prove an earlier attempt failed to commit. */ unconfirmed?: PendingRequest[]; - /** Commits consumption of the durable notification queue with the identity map. */ + /** Commits consumption of the saved notification queue with the identity map. */ lastAppliedLocalChangeId?: string; /** Physical exclusions, including files with no sync identity. */ protectedPaths?: string[]; @@ -83,38 +73,13 @@ export interface EngineState { rejectedManifest?: { entries: FileManifestEntries; message: string }; } -export interface FileStep { - documentId: string; - from?: string; - to?: string; - expected?: StoredSnapshot; - replacement?: StoredSnapshot; - staged: string; - output: string; - /** A permanent destination error has already selected a root fallback. */ - replanned?: boolean; - /** Retained input of a later step for this identity; never projected as live. */ - superseded?: boolean; - phase: "planned" | "staged" | "prepared" | "installing" | "installed"; -} - -export interface ApplicationJournal { - id: string; - extensions: string[]; - next: EngineState; - steps: FileStep[]; -} - -export interface StoredDatabase extends EngineState { - application?: ApplicationJournal; -} +export type StoredDatabase = EngineState; // Read-only view used by cursor tracking and status reporting. export interface DocumentRecord { documentId: string; relativePath: string; metadata?: DocumentMetadata; - isDeleted: boolean; } export function emptyState(vaultKey: string): StoredDatabase { @@ -134,9 +99,7 @@ function canRebindEmpty(state: Partial | undefined): boolean { state !== undefined && !state.initialized && !state.pending && - !state.application && !state.bootstrap && - !state.eventReplay && (state.unconfirmed?.length ?? 0) === 0 && Object.keys(state.local ?? {}).length === 0 ); @@ -146,7 +109,6 @@ export class Database { public state: StoredDatabase; private needsReload = false; public constructor( - private readonly logger: Logger, initial: Partial | undefined, private readonly saveData: (data: StoredDatabase) => Promise, vaultKey: string, @@ -181,7 +143,7 @@ export class Database { const snapshot = structuredClone(next); if (this.needsReload) throw new Error( - "Reload durable state before retrying an uncertain save" + "Reload saved state before retrying an uncertain save" ); this.needsReload = true; await this.saveData(snapshot); @@ -204,10 +166,6 @@ export class Database { return Object.keys(this.state.local).length; } - public getLastSeenUpdateId(): number { - return this.state.lastSeenUpdateId; - } - public getDocumentByDocumentId(id: string): DocumentRecord | undefined { const relativePath = this.state.local[id]; if (relativePath === undefined) return undefined; @@ -215,11 +173,9 @@ export class Database { return { documentId: id, relativePath, - isDeleted: false, metadata: base && { parentVersionId: base.vaultUpdateId, - hash: base.hash, - remoteRelativePath: this.state.fileManifest.entries[id] + hash: base.hash } }; } diff --git a/frontend/sync-client/src/persistence/persistence.ts b/frontend/sync-client/src/persistence/persistence.ts index f879d06b..ac6f724a 100644 --- a/frontend/sync-client/src/persistence/persistence.ts +++ b/frontend/sync-client/src/persistence/persistence.ts @@ -1,6 +1,6 @@ -/** save() atomically replaces the entire value and resolves only after both the - * data and its directory entry are durable across power loss. Failed saves must - * leave either complete value readable; callers retain their recovery journal. +/** Replace the complete metadata value atomically. Interrupted or failed saves + * must leave an old/new complete value (or no value for the first save), never + * partially parsed state. No ordering or durability of user files is required. */ export interface PersistenceProvider { load: () => Promise; diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 71dabeaa..51b92dfa 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -25,12 +25,18 @@ import { isInternalPath } from "./utils/portable-path"; import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; import { globsToRegexes } from "./utils/globs-to-regexes"; import { Lock } from "./utils/data-structures/locks"; -import type { LocalChange } from "./sync-operations/local-changes"; +import { + unappliedChanges, + type LocalChange +} from "./sync-operations/local-changes"; type StoredClient = Partial<{ settings: Partial; database: Partial; - localChanges: LocalChange[]; + localChanges: ( + | LocalChange + | { type: "update"; path: string; changeId?: string } + )[]; localChangesVaultKey: string; historyCheckpoint: { vaultKey: string; checkpoint?: string }; }>; @@ -96,7 +102,7 @@ export class SyncClient { let stored = (await persistence.load()) ?? {}; let saving: Promise = Promise.resolve(); let reloadBeforeSave = false; - // Settings and engine state share a store. Serialize complete durable + // Settings and engine state share a store. Serialize complete metadata // replacements so one cannot overwrite a newer save from the other. const save = async (update: StoredClient): Promise => { const operation = saving.then(async () => { @@ -122,7 +128,6 @@ export class SyncClient { save({ settings: data }) ); const database = new Database( - logger, stored.database, async (data) => save({ database: data }), vaultKey(settings.getSettings()), @@ -132,8 +137,15 @@ export class SyncClient { return stored.database as StoredDatabase | undefined; } ); + // Older stores also queued content updates. Trim the acknowledged prefix + // before discarding them, since its marker may belong to an update. + const localChanges = unappliedChanges( + database.state, + stored.localChanges ?? [] + // eslint-disable-next-line no-restricted-syntax -- Migration needs a narrowed copy, not removal of a specific object. + ).filter((change) => change.type !== "update"); if ( - stored.localChanges?.length && + localChanges.length && stored.localChangesVaultKey !== vaultKey(settings.getSettings()) ) { throw new Error( @@ -168,7 +180,6 @@ export class SyncClient { ); const serverConfig = new ServerConfig(service); const notifier = new FileChangeNotifier(); - const localChanges = stored.localChanges ?? []; let syncer: Syncer; const files = new FileOperations( fs, @@ -283,7 +294,7 @@ export class SyncClient { this.check(); if (this.started) throw new Error("SyncClient has already been started"); - await this.syncer.recoverLocalState(); + await this.syncer.reloadLocalState(); await this.database.bindVault( vaultKey(this.settings.getSettings()) ); @@ -310,7 +321,7 @@ export class SyncClient { await stopped; } - /** Restart transports; never discard pending requests or filesystem journals. */ + /** Restart transports; never discard pending requests. */ public async reset(): Promise { await this.lifecycle.withLock(async () => { this.check(); @@ -335,12 +346,11 @@ export class SyncClient { this.database.state.initialized || this.database.state.bootstrap || this.database.state.pending || - this.database.state.application || this.syncer.hasLocalChanges || Object.keys(this.database.state.local).length) ) throw new Error( - "Changing vaults requires a separate state store and client; pending recovery data must not be reset" + "Changing vaults requires a separate state store and client; existing sync state must not be reset" ); try { diff --git a/frontend/sync-client/src/sync-operations/content.ts b/frontend/sync-client/src/sync-operations/content.ts index a70896bb..8a5aafaf 100644 --- a/frontend/sync-client/src/sync-operations/content.ts +++ b/frontend/sync-client/src/sync-operations/content.ts @@ -47,8 +47,7 @@ export async function mergeContent( remote.contentBase64 ].map(base64ToBytes); - // Concurrent unmergeable edits keep the server's content. The filesystem - // journal retains displaced local bytes for recovery. + // Concurrent unmergeable edits keep the server's content. if (!isFileTypeMergable(path, extensions) || inputs.some(isBinary)) return remote; diff --git a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts index 8828d887..9b1c80da 100644 --- a/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts +++ b/frontend/sync-client/src/sync-operations/cursor-tracker.test.ts @@ -18,7 +18,6 @@ it("retains future cursors and publishes them when content catches up", async () getDocumentByDocumentId: () => ({ documentId: "document", relativePath: "note.md", - isDeleted: false, metadata: { parentVersionId: 2, hash: contentHash } }) } as unknown as Database; diff --git a/frontend/sync-client/src/sync-operations/local-changes.ts b/frontend/sync-client/src/sync-operations/local-changes.ts index 6022f992..209c6853 100644 --- a/frontend/sync-client/src/sync-operations/local-changes.ts +++ b/frontend/sync-client/src/sync-operations/local-changes.ts @@ -11,13 +11,12 @@ export type LocalChange = { | { type: "move"; oldPath: string; relativePath: string } | { type: "create"; path: string; documentId?: string } | { type: "delete"; path: string } - | { type: "update"; path: string } ); -export function unappliedChanges( +export function unappliedChanges( state: EngineState, - changes: readonly LocalChange[] -): LocalChange[] { + changes: readonly T[] +): T[] { for (const change of changes) change.changeId ??= uuid(); const applied = changes.findIndex( (change) => change.changeId === state.lastAppliedLocalChangeId @@ -63,7 +62,6 @@ export function applyLocalChange( (path === from || path.startsWith(from + "/")) ) ); - if (change.type === "update") return; for (const [id, path] of Object.entries(change.identities)) { if (change.type === "delete") { if (ignored(path)) { diff --git a/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts index b50d2b8a..5ccfa7a8 100644 --- a/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts +++ b/frontend/sync-client/src/sync-operations/outgoing-cursor.test.ts @@ -17,7 +17,6 @@ for (const change of ["version", "identity", "path", "delete"] as const) let current: DocumentRecord | undefined = { documentId: "a", relativePath: "a.md", - isDeleted: false, metadata: { parentVersionId: 1, hash: digest } }; const sent: { documentsWithCursors: DocumentWithCursors[] }[] = []; diff --git a/frontend/sync-client/src/sync-operations/scan.ts b/frontend/sync-client/src/sync-operations/scan.ts index 3e743231..90aa9fdb 100644 --- a/frontend/sync-client/src/sync-operations/scan.ts +++ b/frontend/sync-client/src/sync-operations/scan.ts @@ -38,7 +38,7 @@ export async function scanLocalFiles( }: ScanOptions, initial?: VaultSnapshot ): Promise> { - // Leave the prefix queued until its resulting state is durably committed. + // Leave the prefix queued until its resulting state is saved. // A persisted change ID makes replay safe after an uncertain save. const changes = [...queuedChanges]; const unsyncablePaths = new Set(); diff --git a/frontend/sync-client/src/sync-operations/syncer.test.ts b/frontend/sync-client/src/sync-operations/syncer.test.ts index e3c11360..862c56e1 100644 --- a/frontend/sync-client/src/sync-operations/syncer.test.ts +++ b/frontend/sync-client/src/sync-operations/syncer.test.ts @@ -153,8 +153,7 @@ describe("Syncer per-file state and event delivery", () => { path === "note.md" ? { documentId: "document", - relativePath: path, - isDeleted: false + relativePath: path } : undefined } as unknown as Database; @@ -166,10 +165,8 @@ describe("Syncer per-file state and event delivery", () => { (batch: EventBatch) => Promise >() } as unknown as WebSocketManager; - let httpEventsCalls = 0; const service = { events: async () => { - httpEventsCalls++; return { headEventId: 0, events: [] }; } } as unknown as SyncService; @@ -191,8 +188,7 @@ describe("Syncer per-file state and event delivery", () => { return { database, websocket, - syncer, - httpEventsCalls: () => httpEventsCalls + syncer }; } @@ -211,28 +207,6 @@ describe("Syncer per-file state and event delivery", () => { assert.strictEqual(syncer.isDocumentUpToDate("note.md"), false); }); - it("consumes a complete WebSocket event batch without fetching it again", async () => { - const { websocket, syncer, httpEventsCalls } = fixture(); - const batch: EventBatch = { - headEventId: 1, - events: [ - { - eventId: 1, - requestId: "00000000-0000-0000-0000-000000000001", - type: "fileManifest", - fileManifest: { fileManifestId: 1, entries: {} } - } - ] - }; - await websocket.onRemoteVaultUpdateReceived.triggerAsync(batch); - - const selected = await ( - syncer as unknown as { nextEventBatch: () => Promise } - ).nextEventBatch(); - assert.deepEqual(selected, batch); - assert.strictEqual(httpEventsCalls(), 0); - }); - it("projects content responses to metadata before persistence", () => { const { syncer } = fixture(); const projected = ( diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 3a9bc83a..9753c7c5 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -20,6 +20,7 @@ import type { WebSocketManager } from "../services/websocket-manager"; import type { DocumentVersion } from "../services/types/DocumentVersion"; import type { DocumentVersionWithoutContent } from "../services/types/DocumentVersionWithoutContent"; import type { EventBatch } from "../services/types/EventBatch"; +import type { EventRecord } from "../services/types/EventRecord"; import type { FileManifest } from "../services/types/FileManifest"; import type { VaultSnapshot } from "../services/types/VaultSnapshot"; import type { PushContent } from "../services/types/PushContent"; @@ -66,7 +67,6 @@ export class Syncer { private notificationSaveFailed = false; private readonly pendingPaths = new Set(); private unsyncablePaths = new Set(); - private eventBatches: EventBatch[] = []; private hasScanned = false; public constructor( @@ -99,10 +99,7 @@ export class Syncer { // including when the optional polling interval is disabled. this.wake(); }); - websocket.onRemoteVaultUpdateReceived.add(async (batch) => { - this.eventBatches.push(batch); - // Notifications are only hints; HTTP replay recovers discarded pages. - if (this.eventBatches.length > 4) this.eventBatches.shift(); + websocket.onRemoteVaultUpdateReceived.add(async () => { this.wake(); }); } @@ -120,7 +117,6 @@ export class Syncer { !!this.running || this.dirty || !!this.database.state.pending || - !!this.database.state.application || !!this.failure || !!this.rejectionMessage ); @@ -169,11 +165,7 @@ export class Syncer { return false; } - return !this.database.state.application?.steps.some( - (step) => - step.documentId === record.documentId && - step.phase !== "installed" - ); + return true; } public start(): void { @@ -273,7 +265,7 @@ export class Syncer { oldPath?: string; relativePath: string; }): Promise { - // Content snapshots are re-read during journal staging. Only namespace + // Content snapshots are re-read before writing. Only namespace // notifications invalidate an identity plan; keystrokes must not starve // a complete scan of unrelated documents. if (oldPath !== undefined && oldPath !== "") this.localGeneration++; @@ -289,23 +281,14 @@ export class Syncer { oldPath, relativePath }); - } else if (!isInternalPath(relativePath)) { - await this.recordLocalChange({ - type: "update", - path: relativePath - }); } this.wake(); } - /** Local recovery must run even when network sync is disabled. Caller stops - * the engine first and holds the client's lifecycle lock. */ - public async recoverLocalState(): Promise { + /** Reload an uncertain metadata save before starting another attempt. */ + public async reloadLocalState(): Promise { await this.flushLocalChanges(); await this.database.recoverPersistence(); - const { application } = this.database.state; - await this.files.recover(); - if (application) this.recordRecoveredApplication(application.steps); } private async recordLocalChange(change: LocalChange): Promise { @@ -339,7 +322,7 @@ export class Syncer { this.onRemainingOperationsCountChanged.trigger(1); this.failure = undefined; try { - await this.recoverLocalState(); + await this.reloadLocalState(); if ( this.database.state.historyRecovery || (this.database.state.initialized && @@ -360,15 +343,13 @@ export class Syncer { await this.finishPending(); } await this.scan(); - const batch = await this.nextEventBatch(); + const batch = await this.service.events( + this.database.state.lastSeenUpdateId + ); await this.incorporateEventBatch(batch); - if (this.database.state.eventReplay) { - this.dirty = true; - continue; - } await this.scan(); // Previously excluded content becomes eligible after a setting - // change. Its durable obligation survives advancing the cursor. + // change. Its saved obligation survives advancing the cursor. if (Object.keys(this.database.state.excluded ?? {}).length) await this.incorporateFileManifest( this.database.state.fileManifest @@ -392,14 +373,6 @@ export class Syncer { await this.finishPending(); this.dirty = true; } - if ( - this.eventBatches.some( - (batch) => - batch.headEventId > - this.database.state.lastSeenUpdateId - ) - ) - this.dirty = true; } while (this.dirty && !this.stopped); if (!this.dirty && !this.database.state.pending) this.pendingPaths.clear(); @@ -428,24 +401,11 @@ export class Syncer { private async recoverServerHistory(): Promise { this.onServerHistoryChanged.trigger(); this.contentCache.reset(); - this.eventBatches = []; this.hasScanned = false; if (!this.database.state.historyRecovery) { await this.scan(); const guard = this.localGuard(); const previous = this.database.state; - for (const pending of [ - previous.pending, - ...(previous.unconfirmed ?? []) - ]) { - if (pending?.type === "content") - await this.files.retainSnapshot( - pending.request.requestId, - pending.documentId, - previous.local[pending.documentId], - pending.snapshot - ); - } const next = emptyState(previous.vaultKey); next.historyRecovery = true; next.lastAppliedLocalChangeId = previous.lastAppliedLocalChangeId; @@ -561,10 +521,7 @@ export class Syncer { } private next(): EngineState { - const { application: _, ...next } = structuredClone( - this.database.state - ); - return next; + return structuredClone(this.database.state); } private async bootstrap(): Promise { @@ -604,56 +561,53 @@ export class Syncer { } private async incorporateEventBatch(batch: EventBatch): Promise { - const replay = this.database.state.eventReplay; - const after = replay?.after ?? this.database.state.lastSeenUpdateId; - let cursor = after; - let manifest = replay?.manifest ?? this.database.state.fileManifest; - const heads = { ...(replay?.heads ?? this.database.state.remoteHeads) }; - const receipts = [...(replay?.receipts ?? [])]; - for (const event of batch.events) { - if (event.eventId <= cursor) continue; - if (event.eventId !== cursor + 1) - throw new PermanentSyncError( - "Non-contiguous event history; refusing to skip changes" - ); - cursor = event.eventId; - if ( - this.database.state.unconfirmed?.some( - (pending) => pending.request.requestId === event.requestId - ) === true - ) - receipts.push(event); - if (event.type === "fileManifest") { - if (event.fileManifest.fileManifestId > manifest.fileManifestId) - manifest = event.fileManifest; - } else if ( - !heads[event.document.documentId] || - heads[event.document.documentId].vaultUpdateId < - event.document.vaultUpdateId - ) { - heads[event.document.documentId] = this.documentHead( - event.document - ); - } - } - const end = batch.endEventId ?? batch.headEventId; - if ( - end !== cursor || - end > batch.headEventId || - (end === after && end < batch.headEventId) - ) - throw new PermanentSyncError("Incomplete event batch"); - if (end < batch.headEventId) { - await this.database.commit({ - ...this.next(), - eventReplay: { - after: end, - manifest, - heads, - receipts + let cursor = this.database.state.lastSeenUpdateId; + let manifest = this.database.state.fileManifest; + const heads = { ...this.database.state.remoteHeads }; + const receipts: EventRecord[] = []; + // Fold every page before touching local files. If fetching is interrupted, + // discard this work and retry from the last committed cursor. + for (;;) { + const after = cursor; + for (const event of batch.events) { + if (event.eventId <= cursor) continue; + if (event.eventId !== cursor + 1) + throw new PermanentSyncError( + "Non-contiguous event history; refusing to skip changes" + ); + cursor = event.eventId; + if ( + this.database.state.unconfirmed?.some( + (pending) => + pending.request.requestId === event.requestId + ) === true + ) + receipts.push(event); + if (event.type === "fileManifest") { + if ( + event.fileManifest.fileManifestId > + manifest.fileManifestId + ) + manifest = event.fileManifest; + } else if ( + !heads[event.document.documentId] || + heads[event.document.documentId].vaultUpdateId < + event.document.vaultUpdateId + ) { + heads[event.document.documentId] = this.documentHead( + event.document + ); } - }); - return; + } + const end = batch.endEventId ?? batch.headEventId; + if ( + end !== cursor || + end > batch.headEventId || + (end === after && end < batch.headEventId) + ) + throw new PermanentSyncError("Incomplete event batch"); + if (end === batch.headEventId) break; + batch = await this.service.events(cursor); } if (cursor === this.database.state.lastSeenUpdateId) return; // Recover our submitted bases before folding later remote edits. A @@ -707,24 +661,6 @@ export class Syncer { ); } - private async nextEventBatch(): Promise { - const after = - this.database.state.eventReplay?.after ?? - this.database.state.lastSeenUpdateId; - if (this.service.verifiesHistory) { - this.eventBatches = []; - return this.service.events(after); - } - this.eventBatches = this.eventBatches.filter( - (batch) => (batch.endEventId ?? batch.headEventId) > after - ); - const queued = this.eventBatches.findIndex((batch) => - batch.events.some((event) => event.eventId === after + 1) - ); - if (queued !== -1) return this.eventBatches.splice(queued, 1)[0]; - return this.service.events(after); - } - private documentHead( version: DocumentVersionWithoutContent ): DocumentVersionWithoutContent { @@ -738,42 +674,6 @@ export class Syncer { }; } - private recordRecoveredApplication( - steps: readonly { - documentId: string; - from?: string; - to?: string; - replacement?: unknown; - }[] - ): void { - for (const step of steps) { - const to = this.database.state.local[step.documentId]; - if (!step.from && !to) continue; - this.history.addHistoryEntry({ - status: SyncStatus.SUCCESS, - message: "Filesystem application recovered", - details: - step.from && to && step.from !== to - ? { - type: SyncType.MOVE, - relativePath: to, - movedFrom: step.from - } - : !step.from && to - ? { type: SyncType.CREATE, relativePath: to } - : step.from && !to - ? { - type: SyncType.DELETE, - relativePath: step.from - } - : { - type: SyncType.UPDATE, - relativePath: to ?? step.from! - } - }); - } - } - private recordManifestApplication( before: Readonly>, writes: Readonly> @@ -948,7 +848,6 @@ export class Syncer { const next = this.next(); if (cursor !== undefined) { next.lastSeenUpdateId = cursor; - delete next.eventReplay; } if (clearPending) { delete next.pending; @@ -1037,7 +936,7 @@ export class Syncer { } private async preparePush(): Promise { - if (this.stopped || this.database.state.eventReplay) { + if (this.stopped) { return false; } @@ -1214,18 +1113,15 @@ export class Syncer { await this.database.recoverPersistence(); const next = structuredClone(this.database.state); let changed = false; - for (const state of [next, next.application?.next]) { - if (!state) continue; - if (state.rejectedManifest) { - delete state.rejectedManifest; + if (next.rejectedManifest) { + delete next.rejectedManifest; + changed = true; + } + for (const doc of Object.values(next.documents)) { + if (doc.rejected) { + delete doc.rejected; changed = true; } - for (const doc of Object.values(state.documents)) { - if (doc.rejected) { - delete doc.rejected; - changed = true; - } - } } if (changed) await this.database.commit(next); } @@ -1260,12 +1156,6 @@ export class Syncer { if (pending.rejection && !pending.response) { const next = this.next(); if (pending.type === "content") { - await this.files.retainSnapshot( - pending.request.requestId, - pending.documentId, - next.local[pending.documentId], - pending.snapshot - ); next.documents[pending.documentId].rejected = { hash: pending.snapshot.hash, message: pending.rejection @@ -1346,12 +1236,6 @@ export class Syncer { } else { const response = pending.response!; if (response.type === "StaleBase") { - await this.files.retainSnapshot( - pending.request.requestId, - pending.documentId, - this.database.state.local[pending.documentId], - pending.snapshot - ); await this.incorporateContent( response, undefined, diff --git a/frontend/sync-client/src/utils/assert-set-contains-exactly.ts b/frontend/sync-client/src/utils/assert-set-contains-exactly.ts deleted file mode 100644 index 5baf965e..00000000 --- a/frontend/sync-client/src/utils/assert-set-contains-exactly.ts +++ /dev/null @@ -1,13 +0,0 @@ -import assert from "node:assert"; - -export function assertSetContainsExactly(set: Set, ...values: T[]): void { - assert.ok( - set.size === values.length && - Array.from(set).every((value) => values.includes(value)), - `Expected set to contain only ${values.map((v) => '"' + v + '"').join(", ")}, but it contained ${Array.from( - set - ) - .map((v) => '"' + v + '"') - .join(", ")}` - ); -} diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts b/frontend/sync-client/src/utils/data-structures/min-covered.test.ts deleted file mode 100644 index 7b7271d7..00000000 --- a/frontend/sync-client/src/utils/data-structures/min-covered.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { describe, it } from "node:test"; -import assert from "node:assert"; -import { CoveredValues } from "./min-covered"; - -describe("CoveredValues", () => { - it("should initialize with the given min value", () => { - const covered = new CoveredValues(5); - assert.strictEqual(covered.min, 5); - }); - - it("should add values greater than min", () => { - const covered = new CoveredValues(0); - covered.add(3); - assert.strictEqual(covered.min, 0); - covered.add(1); - assert.strictEqual(covered.min, 1); - covered.add(4); - assert.strictEqual(covered.min, 1); - covered.add(2); - assert.strictEqual(covered.min, 4); - }); - - it("should ignore duplicate values", () => { - const covered = new CoveredValues(0); - covered.add(3); - covered.add(3); - covered.add(3); - assert.strictEqual(covered.min, 0); - covered.add(1); - covered.add(2); - assert.strictEqual(covered.min, 3); - }); - - it("should handle multiple consecutive values", () => { - const covered = new CoveredValues(132); - for (let i = 250; i > 132; i--) { - assert.strictEqual(covered.min, 132); - covered.add(i); - } - assert.strictEqual(covered.min, 250); - }); - - it("should handle adding values lower than current min", () => { - const covered = new CoveredValues(5); - covered.add(3); - assert.strictEqual(covered.min, 5); - covered.add(6); - assert.strictEqual(covered.min, 6); - }); - - it("should auto-advance when setting min value", () => { - const covered = new CoveredValues(5); - covered.add(7); - covered.add(8); - covered.add(9); - assert.strictEqual(covered.min, 5); - // Setting min to 6 should auto-advance through 7, 8, 9 - covered.min = 6; - assert.strictEqual(covered.min, 9); - covered.add(10); - assert.strictEqual(covered.min, 10); - }); - - it("should handle setting min value with no consecutive values", () => { - const covered = new CoveredValues(5); - covered.add(10); - covered.add(15); - assert.strictEqual(covered.min, 5); - // Setting min to 8 should not auto-advance (no consecutive values) - covered.min = 8; - assert.strictEqual(covered.min, 8); - // Add 9 to trigger auto-advance to 10 - covered.add(9); - assert.strictEqual(covered.min, 10); - }); -}); diff --git a/frontend/sync-client/src/utils/data-structures/min-covered.ts b/frontend/sync-client/src/utils/data-structures/min-covered.ts deleted file mode 100644 index 8b38822f..00000000 --- a/frontend/sync-client/src/utils/data-structures/min-covered.ts +++ /dev/null @@ -1,61 +0,0 @@ -/** - * A class that tracks the minimum covered value in a sequence of numbers. - * It keeps track of a minimum value based on the seen values. - * - * It expects integers slightly out of order and makes sure that the value of `min` is - * always the minimum of the seen values. This is done with bounded memory usage. - * - * @example - * ```typescript - * const covered = new CoveredValues(0); - * covered.add(2); // seenValues = [2], min = 0 - * covered.add(1); // seenValues = [], min = 2 - * covered.min; // returns 2 - * ``` - */ -export class CoveredValues { - private seenValues: number[] = []; - - public constructor(private minValue: number) {} - - public get min(): number { - return this.minValue; - } - - public set min(value: number) { - this.minValue = Math.max(value, this.minValue); - this.seenValues = this.seenValues.filter((v) => v > this.minValue); - this.advanceMinWhilePossible(); - } - - public add(value: number | undefined): void { - if (value === undefined || value < this.minValue) { - return; - } - - let i = 0; - while (i < this.seenValues.length && this.seenValues[i] < value) { - i++; - } - - if (i === this.seenValues.length) { - this.seenValues.push(value); - } else if (this.seenValues[i] === value) { - return; - } else { - this.seenValues.splice(i, 0, value); - } - - this.advanceMinWhilePossible(); - } - - private advanceMinWhilePossible(): void { - while ( - this.seenValues.length > 0 && - this.seenValues[0] === this.minValue + 1 - ) { - this.seenValues.shift(); - this.minValue++; - } - } -} diff --git a/frontend/test-support/README.md b/frontend/test-support/README.md index e3601ba5..824714ab 100644 --- a/frontend/test-support/README.md +++ b/frontend/test-support/README.md @@ -1,163 +1,52 @@ # Shared sync harness support -This is test-only code, shared by `deterministic-tests` and `test-client` rather -than exported as debugging machinery by the production sync client. +Test-only code shared by the deterministic and seeded clients. -The journal suites also cover repeated saves replacing earlier unsynced text, -tracked file/directory moves into ignored paths, and saves crossing the size -limit. They check exact bytes and identities through recovery, including real -server reads. Client and Rust unit suites bound deep-path validation resources. +| Suite | Coverage | +| --- | --- | +| `harness.test.ts`, `oracle-regressions.test.ts` | Independent content/identity oracles, exercised faults, exclusive operations, persistence ownership, cleanup and seeds | +| `file-application.test.ts` | Swaps, file/directory transitions, visible conflict files, live edits, exclusions, and one metadata save per application | +| `recovery.test.ts` | Interrupt every application filesystem/save boundary, reopen metadata, modify disk, and verify stable ordinary scans; manifest policy and notification identity | +| `engine-regressions.test.ts`, `sync-audit.test.ts` | Offline identities, unknown network outcomes, rejection/retry, exclusions, interrupted in-memory pagination, namespace races and lifecycle changes | +| `event-delivery.test.ts`, `polling.test.ts` | WebSocket hints with HTTP-only event reads, missed notifications, opt-in polling, shutdown and retry | +| `protocol.test.ts` | Abrupt worker termination after accepted requests, exact retries, real server SIGKILL, CAS and event ordering | +| `restart-reconciliation.test.ts` | Interrupted file application followed by user edits, two-client/server convergence and a second restart | +| `server-restore.test.ts` | History checkpoints, server backup restores and interrupted metadata resets | +| `content-limits.test.ts`, `protocol-audit.test.ts` | Content validation and API invariants | +| `real-disk.test.ts` | Host filesystem exclusive create/rename, unsafe links, and rename cycles | -| Layer | What it establishes | -| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `harness.test.ts` | Divergence/self-comparison detection, binary-safe comparisons, marker loss/duplication, cleanup failure propagation, deep persistence ownership, exclusive operations, exercised faults, seed reproducibility | -| `oracle-regressions.test.ts` | Negative checks for regenerated retries, wrong fault kinds, duplicate clients, corrupt hashes, BOMs, path byte limits, idempotent unlink, directory durability, and complete identity/content expectations | -| `recovery.test.ts` | Real engine journal recovery at every enumerated apply and replay mutation/save boundary; two interruptions followed by another power loss; identity, retained bytes and merge-policy regressions | -| `engine-regressions.test.ts` | Recovery archives, offline notifications, rejection/retry, exclusions, reverted remote updates, lifecycle races and editor changes during staging/install | -| `event-delivery.test.ts` | Real sync loop with checkpoint-controlled WS bursts before an iteration and during event application; no polling or extra wakeups to hide queued events | -| `polling.test.ts` | Real sync loop with fake clocks: missed local/remote notifications, disabled controls, repeated polls, shutdown and network retries independent of polling | -| `protocol.test.ts` | Terminated worker runtimes after accepted creates, existing-document Snapshot/Diff updates, and manifest requests, exact durable retries after newer remote heads and real server SIGKILL; concurrent CAS and ordered WS catchup | -| `real-disk.test.ts` | Host filesystem exclusive create/rename, symlink/hardlink rejection and a real engine rename-cycle application | -| Scripted suite | Existing multi-client regression scenarios plus v4 binary, identity, path-conflict, retry, and server-restart cases | -| Seeded workloads | Concrete replay traces, offline and rapid online moves/edits, concurrent creates, exact deletion/overwrite exemptions, nested Unicode paths, delayed notifications and before/after network failures | +The filesystem interruption tests assert the supported contract: complete metadata +remains usable, scans observe the current files, and a second scan is stable. +They do not demand the pre-interruption file contents, identities or a replay of +unfinished operations. Normal-operation tests still check exact bytes and identities. +There is no filesystem journal or hidden recovery archive. -`scripts/e2e-runner.test.mjs` also guards clean-install workspace/lockfile -consistency, CI dependency installation and artifact upload wiring, subprocess -exit codes, log failures, spawn failures, and timeouts. +`MemoryDisk` exposes `before:`, `visible:` and `durable:` boundaries. A process +interruption retains visible changes; a simulated power loss rolls unflushed +changes back. This provides different partial-application states for testing; it +is not an APFS/ext4 writeback model or a production durability requirement. +`MemoryPersistence` atomically replaces complete objects and can reject before or +after replacement. Abandoned filesystem sessions are fenced against later writes. -`MemoryDisk` distinguishes visible namespace/data from durable entries. Each -mutation has `before:`, `visible:`, and `durable:` boundaries. `crash(false)` -models process death with dirty kernel state retained; `crash(true)` discards -unflushed state. Operations that return successfully have flushed their affected -entries. Idempotent unlink flushes absent entries too, and durable directory -removal makes all former descendants unreachable. This is a deliberately small atomic-operation model, not an emulation -of APFS/ext4 writeback. A crashed session is fenced against subsequent writes. +`crash-worker.ts` runs a complete client in a worker, which the parent terminates +at accepted-response checkpoints without graceful shutdown. A new runtime receives +the saved disk and metadata. The separate real-disk fixture uses a test-only native +exclusive-rename helper and ordinary writes without flushes. It is not a production +adapter or a sandbox against hostile concurrent changes to ancestor directories. -`MemoryPersistence` atomically saves complete, cloned objects. Failure before a -save leaves the old object; failure after durable replacement leaves the new one. -The journal sweep recreates the production database/file-operations objects after -each fault. From each interrupted state it traces recovery and injects a second -process death or power loss at every replay mutation/save boundary. Final recovery -is replayed again and followed by another power loss. Cases cover swaps with new -output, cross-directory moves with a surviving sibling, concurrent text merges, -binary replacement, deletion and recovery archive cleanup. Exact visible bytes, -UUID/path mappings, retained bytes and absence of transaction files are checked. An intent is resubmitted only if nothing was durably saved; -resubmitting a partially recovered operation would hide journal defects. +Scripted and seeded suites run multiple clients against real servers. They check +visible bytes, UUID mappings, canonical server state, content heads and event cursors. +Polling is disabled except in tests specifically exercising it. Faults must fire; +missed injections fail cleanup rather than silently passing. Workload traces, +per-worker results and logs are saved by `scripts/e2e.sh` for reproduction. -`crash-worker.ts` runs a complete SyncClient in a separate worker. The parent -terminates it at an explicit accepted-response checkpoint without `destroy()`; -no old timers, websocket handlers, or promises survive. Update cases bootstrap an existing UUID and assert the non-null parent and exact -Snapshot/Diff request type before termination. Its saved memory-disk image and persisted database seed a fresh runtime. The separate real-disk fixture -uses a test-only native exclusive-rename primitive. It is **not** a -production adapter or an adversarial `openat` sandbox: concurrent hostile changes -to ancestor directories and actual machine power cuts are outside its coverage. - -## Regression coverage - -The previously exposed WS queue, notified-rename UUID, uncertain-save replay, -and cross-directory durability failures are fixed and remain asserted by the -default harness. Scripted, seeded, event-delivery and engine regression clients -explicitly disable polling so it cannot hide a missing wakeup. Crash workers also -assert that this setting survives restart. Polling is enabled only in separate -tests that intentionally omit notifications and advance a fake clock. Identity -assertions distinguish documents even when bytes -are equal. The engine regressions additionally exercise retained recovery bytes, -durable offline notifications, permanently rejected requests, ignored namespace -conflicts, reverted remote updates, and concurrent lifecycle changes. - -Tests assert intended invariants, including bytes and UUID mappings, rather than -accepting the observed output as an oracle. The memory fault model and real-disk -checks complement one another; neither proves behavior under every real power cut. - -## September 2026 audit regressions - -`scripts/e2e.sh` now runs the Rust and client unit suites as well as the harness, -real-server protocol checks, scripted scenarios, and seeded workloads. A failing -unit suite makes the complete run fail. The audit tests are separate, individually -quoted arguments in the default scripts, with a runner regression guarding that wiring. - -The audit cases were first run with assertions for the required behavior and -observed failing before the corresponding fixes. Coverage is organized by the -original finding numbers: - -| Findings | Permanent coverage and resulting guarantee | -| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | Engine and real-server lost-reply/rejected-retry tests preserve later content and rename undos; interrupted receipt saves and paged replay are tested after process replacement. | -| 2, 6 | Ignored/oversized unmanaged occupants keep their paths, including arrivals after scanning; submitted manifests reserve excluded server paths and remain valid. | -| 3, 12 | Oversized remote heads defer uploads; oversized local files are rejected before snapshot reads; scans retain hashes rather than encoded vault contents; content notifications cannot starve namespace scans. | -| 4 | A subprocess deadline detects synchronous conflict-allocation hangs caused by long extensions. | -| 5 | Persistent destination errors select a durable root fallback. The recovery sweep keeps that error active through every first and second crash boundary, checking exact bytes, identities and retained originals. | -| 7–9 | Bootstrap notifications adopt remote identities; settings updates resume enabled engines, disabling sync persists across reset, and initial vault bindings survive restart; disabled startup recovers local journals. | -| 10–11 | Fake-clock silent-socket recovery and real-server connection admission, slot release and handshake expiry. | -| 13 | Rust tests bound event count and aggregate bytes per page; client tests fold remote reversions before application and preserve the replay position across restart. | -| 14–15 | Metadata-only manifest reference validation and active-pool cleanup tests ensure unrelated payload decoding and checked-out transactions cannot block these operations. | -| 16 | Dirty cursor updates invalidate precise positions; disconnect/reset clear presented cursors. | -| 17 | Read-only configuration preservation and concurrent initialization verify that existing credentials are never rewritten or replaced. | - -The follow-up audit added these independently reproduced failures. Each behavior -was asserted before its fix; the default test commands include every regression. - -| Additional failure | Regression coverage | -| ------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Delayed content receipts crash after local deletion | `sync-audit.test.ts` verifies receipt consumption, preserved deletion, and cursor advancement without a local path. | -| Cursor reads restore presence after disconnect/reset | Cursor unit tests hold an active read and queue another update, retire the connection, then release both. | -| Failed WebSocket upgrades strand clients with polling disabled | `polling.test.ts` requires exact remote content and rename catchup over HTTP after a disconnect signal. | -| WebSocket construction errors escape startup/retry callbacks | Transport unit tests fail the first or second construction and require a successful subsequent connection. | -| Opening a locked vault blocks unrelated vaults | Rust tests hold a real SQLite writer lock during lazy opening; another vault must remain readable. Concurrent openings must share the configured pool, including after cancellation. | -| Failure clearing rejected requests leaves an enabled client stopped | Settings tests fail the second save, before and after commit, and require later notifications to wake synchronization. | -| Caller mutations bypass the checked settings snapshot | A settings test changes the caller's vault name and exclusion array while transports pause; persisted settings must retain the original input. | -| Cursor freshness uses metadata captured before an asynchronous read | A cursor test advances the document while the read is suspended and requires the old cursor to be marked outdated. | -| Throwing/silent socket closes leave live handlers after shutdown | Transport tests require disconnected status and no message delivery after a failed or timed-out close. | -| Handshake send errors escape the socket-open callback | A transport test drops the send during opening and requires safe retirement and retry. | -| Old connection cleanup deletes its replacement's cursors | A real-server overlapping reconnect test preserves the replacement; Rust tests also reject delayed updates from the old session. | - -These cases exercise transitions the earlier tests did not combine: a deletion -between submission and acknowledgement, disconnects during reads, failures after -the settings save, and two connections sharing a device identity. Happy-path -reconnect and single-save fault tests do not establish those guarantees. - -Replay pages contain at most 64 events and approximately 1 MiB of aggregate event -JSON. One atomic manifest can exceed that budget; it is sent alone so replay still -advances. `endEventId` identifies the page boundary while `headEventId` identifies -the vault head. Deploy the updated client and server together: older clients -require each batch to reach the vault head. Local partial replay folds retain only -latest document heads, the latest manifest, and matching outstanding receipts; -local files are reconciled only when catchup reaches the head. - -Permanent destination failures use `Recovered ` in the -vault root, resolving any occupied name. The journal retains the original bytes -when content is replaced. Permission or storage failures that also prevent that -fallback remain explicit recovery failures with the journal intact. - -A rejected retry retains an unconfirmed receipt until its request appears in the -event log. Absence from one catchup is not proof that an earlier in-flight attempt -cannot still commit. Receipt/artifact pruning remains an explicit operational concern; these tests do not claim -formal completeness or simulate arbitrary physical-media corruption. - -Reproduce the complete validation run from the repository root: +Run from the repository root: ```sh -E2E_ITERATIONS=100 E2E_SEED=8201 E2E_WORKERS=2 scripts/e2e.sh +E2E_WORKERS=32 E2E_SEED=12 E2E_TIMEOUT_SECONDS=9000 E2E_ITERATIONS=100 scripts/e2e.sh ``` -## Sync regression coverage added after the September audit - -The normal test commands include these regressions: - -| Failure | Maintained coverage | -| ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Ignored occupant renamed into scope; incoming download discarded by unrelated notifications | `journal-protocol.test.ts` checks visible files, original IDs and actual server content. | -| Source path reused or moved during staging; editor save becomes a duplicate | `journal-interleavings.test.ts` injects moves, delete/recreate, move/recreate and content saves at every enumerated read, mutation and state-save boundary. Assertions specify bytes and document identities independently. | -| Journal/editor interaction followed by a crash | `journal-protocol.test.ts` crosses three editor operations with failures before/after state persistence, process replacement and simulated power loss, then checks another reset. | -| Restored server history falsely appears current | `server-restore.test.ts` restores actual SQLite backups and restarts Rust, covering equal/lower counters, dirty files, lost acknowledgements, interrupted recovery saves and old client state without checkpoints. | -| Oversized remote fallback and stale responses | `content-limits.test.ts` measures payload downloads, checks metadata-only responses and verifies highly fragmented local edits fall back to snapshots. | -| Adversarial diff work | Rust endpoint tests reject excessive operations without appending events and exercise Unicode reconstruction. | -| Observer exception kills sync | `observer.test.ts` runs throwing/rejecting subscribers in a child process and verifies a second sync; listener unit tests verify continued delivery. | -| Outgoing cursor version race | `outgoing-cursor.test.ts` changes version, identity, path and existence during a read, deliberately including identical bytes. | -| Unpersisted credentials | Rust config tests reject both missing users and missing user lists while preserving existing file bytes; initialization tests verify stable saved defaults. | -| Extensionless filenames enable merging | Extension unit tests include root/nested `md` and `txt` and dots in ancestor directories. | -| Corrupt vault blocks startup | Rust database regression opens a healthy vault alongside an invalid SQLite file and verifies the invalid file remains untouched. | - -The filesystem sweep reports the number of boundaries and injected interleavings. -A passing sweep covers this adapter model and these operations; arbitrary physical -media corruption and every possible editor/OS behavior remain outside its scope. +The command rebuilds peers, runs unit/harness/protocol/scripted suites, and then +runs seeds 12 through 43. `E2E_ARTIFACTS` optionally selects the artifact directory. +The runner tests also verify subprocess exit codes, log errors, spawn failures, +timeouts, workspace dependency consistency and CI artifact wiring. diff --git a/frontend/test-support/canonical.ts b/frontend/test-support/canonical.ts index 0f30ffa4..296413a2 100644 --- a/frontend/test-support/canonical.ts +++ b/frontend/test-support/canonical.ts @@ -26,7 +26,7 @@ export async function getJson(url: string, token: string): Promise { } /** Equality alone can pass when every replica loses the same bytes. Check the - * canonical UUIDs, content heads, receipts, journals and event cursor as well. + * canonical UUIDs, content heads, receipts and event cursor as well. * Scenario-specific content/identity oracles still run separately. */ export async function assertCanonical( diff --git a/frontend/test-support/engine-regressions.test.ts b/frontend/test-support/engine-regressions.test.ts index 517d1360..81995ee6 100644 --- a/frontend/test-support/engine-regressions.test.ts +++ b/frontend/test-support/engine-regressions.test.ts @@ -1,170 +1,144 @@ import assert from "node:assert/strict"; import { PermanentSyncError } from "../sync-client/src/errors/errors"; import { test } from "node:test"; -import { - Database, - emptyState, - type StoredDatabase -} from "../sync-client/src/persistence/database"; -import { Settings } from "../sync-client/src/persistence/settings"; -import { FileOperations } from "../sync-client/src/file-operations/file-operations"; +import { Database, emptyState } from "../sync-client/src/persistence/database"; import { Syncer } from "../sync-client/src/sync-operations/syncer"; import { SyncClient } from "../sync-client/src/sync-client"; import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; -import { Logger } from "../sync-client/src/tracing/logger"; -import { SyncHistory } from "../sync-client/src/tracing/sync-history"; -import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; -import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; import { allocatePortablePath } from "../sync-client/src/utils/portable-path"; -import type { ServerConfig } from "../sync-client/src/services/server-config"; -import type { SyncService } from "../sync-client/src/services/sync-service"; -import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; import { MemoryDisk, MemoryPersistence } from "./storage"; -import type { LocalChange } from "../sync-client/src/sync-operations/local-changes"; -const bytes = (text: string) => new TextEncoder().encode(text); -const head = (id: string, version: number, content: string) => ({ - documentId: id, - vaultUpdateId: version, - contentSize: bytes(content).length, - updatedDate: "2026-09-19T00:00:00Z", - userId: "user", - deviceId: "device" +import { bytes, head, fixture } from "./sync-fixture"; + +test("offline content notifications need no saves and restart reads the current bytes", async () => { + const disk = new MemoryDisk(); + await disk.userWrite("a.md", bytes("original")); + const initial = emptyState(JSON.stringify(["http://offline.test", "test"])); + initial.initialized = true; + initial.local = { a: "a.md" }; + initial.documents.a = { + materialized: true, + observedHash: (await toStoredSnapshot({ content: bytes("original") })) + .hash + }; + const store = new MemoryPersistence({ + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" } + }); + let client = await SyncClient.create({ fs: disk, persistence: store }); + try { + const saved = store.snapshot(); + store.boundary = () => { + throw new Error("content notifications must not save"); + }; + for (let edit = 0; edit < 20; edit++) { + await disk.userWrite("a.md", bytes(`edit ${edit}`)); + await client.syncLocallyUpdatedFile({ relativePath: "a.md" }); + } + await client.destroy(); + assert.deepEqual(store.snapshot(), saved); + store.boundary = () => {}; + // A final edit has no notification at all. + await disk.userWrite("a.md", bytes("latest offline bytes")); + client = await SyncClient.create({ fs: disk, persistence: store }); + const internals = client as unknown as { + database: Database; + syncer: { + stopped: boolean; + scan(): Promise; + preparePush(): Promise; + }; + }; + await internals.syncer.scan(); + internals.syncer.stopped = false; + assert.equal(await internals.syncer.preparePush(), true); + const pending = internals.database.state.pending; + assert(pending?.type === "content"); + assert.equal(pending.documentId, "a"); + assert.equal( + Buffer.from(pending.snapshot.contentBase64, "base64").toString(), + "latest offline bytes" + ); + } finally { + store.boundary = () => {}; + await client.destroy(); + } }); -async function fixture( - contents: Record = { "a.md": "local" }, - service: Partial = {} -) { +test("legacy content notifications preserve the acknowledged namespace prefix on restart", async () => { const disk = new MemoryDisk(); - const initial = emptyState("test"); + await disk.userWrite("d.md", bytes("A")); + await disk.userWrite("b.md", bytes("B")); + const initial = emptyState(JSON.stringify(["http://offline.test", "test"])); initial.initialized = true; - for (const [path, content] of Object.entries(contents)) { - await disk.userWrite(path, bytes(content)); - const id = path.split(".")[0]; - initial.local[id] = path; + initial.local = { a: "c.md", b: "b.md" }; + initial.fileManifest.entries = { ...initial.local }; + for (const [id, content] of [ + ["a", "A"], + ["b", "B"] + ]) initial.documents[id] = { materialized: true, observedHash: (await toStoredSnapshot({ content: bytes(content) })) .hash }; + initial.lastAppliedLocalChangeId = "consumed-update"; + const saved = { + database: initial, + settings: { remoteUri: "http://offline.test", vaultName: "test" }, + localChangesVaultKey: initial.vaultKey, + localChanges: [ + { + type: "move", + oldPath: "a.md", + relativePath: "b.md", + identities: { a: "a.md" }, + changeId: "consumed-move" + }, + { type: "update", path: "c.md", changeId: "consumed-update" }, + { type: "update", path: "b.md", changeId: "new-update" }, + { + type: "move", + oldPath: "c.md", + relativePath: "d.md", + identities: { a: "c.md" }, + changeId: "new-move" + } + ] + }; + const store = new MemoryPersistence(saved); + const client = await SyncClient.create({ fs: disk, persistence: store }); + try { + const internals = client as unknown as { + database: Database; + syncer: { scan(): Promise }; + }; + await internals.syncer.scan(); + assert.deepEqual(internals.database.state.local, { + a: "d.md", + b: "b.md" + }); + assert.equal( + internals.database.state.lastAppliedLocalChangeId, + "new-move" + ); + } finally { + await client.destroy(); } - initial.fileManifest = { fileManifestId: 1, entries: { ...initial.local } }; - const persistence = new MemoryPersistence({ database: initial }); - const logger = new Logger(); - const database = new Database( - logger, - initial, - async (next) => persistence.save({ database: next }), - "test", - async () => (await persistence.load()).database as StoredDatabase - ); - const config = { - initialize: async () => {}, - getConfig: async () => ({ mergeableFileExtensions: ["md"] }), - reset: () => {} - } as unknown as ServerConfig; - const settings = new Settings( - logger, - { syncIntervalMs: 0 }, - async () => {} - ); - const changes: LocalChange[] = []; - const files = new FileOperations( - disk.session(), - database, - config, - undefined, - { entries: () => changes, flush: async () => {} } - ); - const websocket = { - onWebSocketStatusChanged: new EventListeners(), - onRemoteVaultUpdateReceived: new EventListeners() - } as unknown as WebSocketManager; - const syncer = new Syncer( - "device", - logger, - database, - settings, - service as SyncService, - websocket, - files, - config, - new SyncHistory(logger), - new FixedSizeDocumentCache(1024), - { entries: changes, save: async () => {} } - ); - const internals = syncer as unknown as { - finishPending(): Promise; - preparePush(): Promise; - scan(): Promise; - incorporateEventBatch( - batch: import("../sync-client/src/services/types/EventBatch").EventBatch - ): Promise; - stopped: boolean; - incorporateFileManifest(remote: { - fileManifestId: number; - entries: Record; - }): Promise; - incorporateContent(remote: ReturnType): Promise; - }; - return { - disk, - database, - files, - settings, - syncer, - internals, - persistence, - changes - }; -} - -test("remote binary replacement retains displaced bytes outside the visible namespace", async () => { - const f = await fixture({ "a.bin": "private edit" }); - const next = structuredClone(f.database.state); - await f.files.apply(next, { - a: { - expected: await f.files.snapshot("a.bin"), - replacement: await toStoredSnapshot({ content: bytes("remote") }) - } - }); - assert.deepEqual(f.disk.userFiles(), new Map([["a.bin", bytes("remote")]])); - const retained = (await f.disk.listFilesRecursively()).filter((path) => - path.startsWith(".vault-link-sync/recovery/") - ); - assert( - ( - await Promise.all(retained.map((path) => f.disk.readSnapshot(path))) - ).some( - (value) => - value && - Buffer.from(value.content).toString() === "private edit" - ) - ); - await f.files.recover(); - assert.deepEqual( - await f.disk.listFilesRecursively(), - ["a.bin", ...retained].sort() - ); }); -test("remote deletion retains unsent bytes for recovery", async () => { - const f = await fixture(); +test("remote binary replacement and deletion do not create backup artifacts", async () => { + const f = await fixture({ "a.bin": "private edit" }); + await f.files.apply(structuredClone(f.database.state), { + a: { replacement: await toStoredSnapshot({ content: bytes("remote") }) } + }); + assert.deepEqual(await f.disk.listFilesRecursively(), ["a.bin"]); + assert.deepEqual(f.disk.userFiles(), new Map([["a.bin", bytes("remote")]])); await f.internals.incorporateFileManifest({ fileManifestId: 2, entries: {} }); - assert.equal(f.disk.userFiles().size, 0); - const retained = await f.disk.listFilesRecursively(); - assert( - ( - await Promise.all(retained.map((path) => f.disk.readSnapshot(path))) - ).some( - (value) => - value && Buffer.from(value.content).toString() === "local" - ) - ); + assert.deepEqual(await f.disk.listFilesRecursively(), []); }); test("rejected content recovery remains idempotent across a rename and network failure", async () => { @@ -281,45 +255,6 @@ test("unfinished bootstrap cannot be rebound to another vault", async () => { } }); -test("local swap arriving after a remote journal is saved preserves both identities and contents", async () => { - const f = await fixture({ "a.md": "A", "b.md": "B" }); - let fired = false; - f.persistence.boundary = async (label) => { - if ( - fired || - label !== "durable:save" || - !f.persistence.snapshot().database?.application - ) - return; - fired = true; - for (const [oldPath, relativePath] of [ - ["a.md", "tmp.md"], - ["b.md", "a.md"], - ["tmp.md", "b.md"] - ]) { - await f.disk.userRename(oldPath, relativePath); - await f.syncer.syncLocallyUpdatedFile({ oldPath, relativePath }); - } - }; - const next = structuredClone(f.database.state); - next.local = { a: "remote-a.md", b: "remote-b.md" }; - await f.files.apply(next); - assert(fired); - assert.equal( - Buffer.from( - f.disk.userFiles().get(f.database.state.local.a)! - ).toString(), - "A" - ); - assert.equal( - Buffer.from( - f.disk.userFiles().get(f.database.state.local.b)! - ).toString(), - "B" - ); - assert.deepEqual(f.database.state.local, { a: "b.md", b: "a.md" }); -}); - test("a rejected upload is retained locally and does not block another file or a corrected snapshot", async () => { let reject = true; const sent: string[] = []; @@ -375,7 +310,7 @@ test("a transient upload failure retries the exact request and recovers its ackn assert.equal(f.database.state.pending, undefined); }); -test("a persisted upload rejection finishes locally after a recovery write fails", async () => { +test("a persisted upload rejection finishes without filesystem writes", async () => { let uploads = 0; const f = await fixture(undefined, { putFileContent: async () => { @@ -385,29 +320,14 @@ test("a persisted upload rejection finishes locally after a recovery write fails }); f.internals.stopped = false; await f.internals.preparePush(); - const { pending } = f.database.state; - assert(pending); - const { requestId } = pending.request; - const retained = `.vault-link-sync/requests/${requestId}.content`; - const { boundary } = f.disk; - f.disk.boundary = (label): void => { - if (label === `before:write:${retained}`) - throw new Error("recovery write failed"); + f.disk.boundary = (label) => { + if (!label.startsWith("read:")) + throw new Error("Unexpected filesystem mutation"); }; - await assert.rejects(f.internals.finishPending(), /recovery write failed/); - assert.equal( - f.persistence.snapshot().database?.pending?.rejection, - "HTTP 413" - ); - f.disk.boundary = boundary; await f.internals.finishPending(); assert.equal(uploads, 1); assert.equal(f.database.state.pending, undefined); assert.equal(f.database.state.documents.a.rejected?.message, "HTTP 413"); - assert.deepEqual( - (await f.disk.readSnapshot(retained))?.content, - bytes("local") - ); assert.deepEqual(f.disk.userFiles(), new Map([["a.md", bytes("local")]])); }); @@ -749,79 +669,6 @@ test("an incoming ignored document cannot force an existing ignored file to move ); }); -for (const boundary of ["visible", "durable"] as const) { - for (const action of [ - "move-installed", - "recreate-staged-source" - ] as const) { - test(`${action} at a ${boundary} filesystem boundary keeps original identities and all bytes`, async () => { - const f = await fixture({ "a.md": "A", "b.md": "B" }); - let changed = false; - f.disk.boundary = async (label) => { - if (changed || !label.startsWith(`${boundary}:rename:`)) return; - if (action === "move-installed" && label.endsWith("->b.md")) { - changed = true; - await f.disk.userRename("b.md", "later.md"); - await f.syncer.syncLocallyUpdatedFile({ - oldPath: "b.md", - relativePath: "later.md" - }); - } else if ( - action === "recreate-staged-source" && - label.includes("rename:a.md->") - ) { - changed = true; - await f.disk.userWrite("a.md", bytes("new")); - await f.syncer.syncLocallyCreatedFile("a.md"); - } - }; - const next = structuredClone(f.database.state); - next.local = { a: "b.md", b: "a.md" }; - next.fileManifest = { - fileManifestId: 2, - entries: { ...next.local } - }; - await f.files.apply(next); - assert(changed); - assert.equal( - Buffer.from( - (await f.disk.readSnapshot(f.database.state.local.a))! - .content - ).toString(), - "A" - ); - assert.equal( - Buffer.from( - (await f.disk.readSnapshot(f.database.state.local.b))! - .content - ).toString(), - "B" - ); - assert.deepEqual( - [...f.disk.userFiles().values()] - .map((value) => Buffer.from(value).toString()) - .sort(), - action === "move-installed" ? ["A", "B"] : ["A", "B", "new"] - ); - f.disk.crash(true); - assert.equal( - Buffer.from( - (await f.disk.readSnapshot(f.database.state.local.a))! - .content - ).toString(), - "A" - ); - assert.equal( - Buffer.from( - (await f.disk.readSnapshot(f.database.state.local.b))! - .content - ).toString(), - "B" - ); - }); - } -} - test("rejected content is reported to callers and an explicit retry clears the rejection", async () => { const f = await fixture(undefined, { putFileContent: async () => { diff --git a/frontend/test-support/event-delivery.test.ts b/frontend/test-support/event-delivery.test.ts index a31d58e9..012e24fd 100644 --- a/frontend/test-support/event-delivery.test.ts +++ b/frontend/test-support/event-delivery.test.ts @@ -21,11 +21,11 @@ import { MemoryDisk, MemoryPersistence } from "./storage"; for (const [count, delivery] of [ [1, "before the loop"], - [3, "before the loop"], + [9, "before the loop"], [3, "during the first batch"] ] as const) { test( - `${count} WS batches delivered ${delivery} drain without polling or another notification`, + `${count} WS hints delivered ${delivery} catch up through HTTP without polling`, { timeout: 5_000 }, async () => { const logger = new Logger(); @@ -33,7 +33,6 @@ for (const [count, delivery] of [ initial.initialized = true; const persistence = new MemoryPersistence({ database: initial }); const database = new Database( - logger, initial, async (next) => persistence.save({ database: next }), initial.vaultKey, @@ -85,13 +84,20 @@ for (const [count, delivery] of [ fileManifest: { fileManifestId: index + 1, entries: {} } }) ); + let available: number = delivery === "before the loop" ? count : 1; + const fetchedAfter: number[] = []; const service = { - // A correct client may drain the queue or catch up over HTTP. Either - // must reach the same head without the test manually waking it. - events: async (after: number): Promise => ({ - headEventId: count, - events: events.filter((event) => event.eventId > after) - }) + events: async (after: number): Promise => { + fetchedAfter.push(after); + return { + headEventId: available, + events: events.filter( + (event) => + event.eventId > after && + event.eventId <= available + ) + }; + } } as unknown as SyncService; const websocket = { onWebSocketStatusChanged: new EventListeners< @@ -116,7 +122,15 @@ for (const [count, delivery] of [ const deliver = async (event: EventRecord) => { await websocket.onRemoteVaultUpdateReceived.triggerAsync({ headEventId: event.eventId, - events: [event] + // Payloads are deliberately inconsistent with HTTP. The socket + // only wakes the loop; it cannot supply reconciliation state. + events: [ + { + ...event, + type: "fileManifest", + fileManifest: { fileManifestId: 999, entries: {} } + } + ] }); }; try { @@ -129,6 +143,7 @@ for (const [count, delivery] of [ await deliver(events[0]); initialized.resolve(); await firstBatch.promise; + available = count; for (const event of events.slice(1)) await deliver(event); continueBatch.resolve(); } @@ -140,11 +155,14 @@ for (const [count, delivery] of [ ); assert.deepEqual( applied, - events.map((event) => event.eventId), - "Every event must be committed in order" + delivery === "before the loop" ? [count] : [1, count], + "Each HTTP catchup commits its final cursor" + ); + assert.deepEqual( + fetchedAfter, + delivery === "before the loop" ? [0] : [0, 1] ); assert.equal(database.state.fileManifest.fileManifestId, count); - assert.equal(database.state.application, undefined); assert.equal(database.state.pending, undefined); assert.equal(syncer.isBusy, false); } finally { diff --git a/frontend/test-support/file-application.test.ts b/frontend/test-support/file-application.test.ts new file mode 100644 index 00000000..957fe62f --- /dev/null +++ b/frontend/test-support/file-application.test.ts @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes } from "./sync-fixture"; +import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; +import { LocalChangesDuringReconciliation } from "../sync-client/src/errors/errors"; + +for (const equal of [false, true]) + test(`swaps preserve identities with ${equal ? "equal" : "distinct"} content and save once`, async () => { + const f = await fixture({ "a.md": "A", "b.md": equal ? "A" : "B" }); + let saves = 0; + f.persistence.boundary = (label) => { + if (label === "before:save") saves++; + }; + const next = structuredClone(f.database.state); + next.local = { a: "b.md", b: "a.md" }; + await f.files.apply(next); + assert.deepEqual(f.database.state.local, next.local); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["a.md", bytes(equal ? "A" : "B")], + ["b.md", bytes("A")] + ]) + ); + assert.equal(saves, 1); + assert.deepEqual(await f.disk.listFilesRecursively(), ["a.md", "b.md"]); + }); + +for (const direction of ["file-to-directory", "directory-to-file"]) + test(direction, async () => { + const f = await fixture( + direction === "file-to-directory" + ? { folder: "A", "b.md": "B" } + : { "a.md": "A", "folder/b.md": "B" } + ); + const ids = Object.keys(f.database.state.local); + const next = structuredClone(f.database.state); + next.local = + direction === "file-to-directory" + ? { [ids[0]]: "folder/a.md", [ids[1]]: "b.md" } + : { [ids[0]]: "folder", [ids[1]]: "other.md" }; + const expected = structuredClone(next.local); + await f.files.apply(next); + assert.deepEqual(f.database.state.local, expected); + assert.deepEqual( + new Set(f.disk.userFiles().keys()), + new Set(Object.values(f.database.state.local)) + ); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((value) => Buffer.from(value).toString()) + .sort(), + ["A", "B"] + ); + }); + +test("an unexpected occupant is preserved under a visible conflict name", async () => { + const f = await fixture(); + await f.disk.userWrite("remote.md", bytes("unexpected")); + const next = structuredClone(f.database.state); + next.local.a = "remote.md"; + await f.files.apply(next); + assert.equal(f.database.state.local.a, "remote.md"); + assert.equal(f.disk.userFiles().size, 2); + assert.deepEqual( + [...f.disk.userFiles().values()] + .map((value) => Buffer.from(value).toString()) + .sort(), + ["local", "unexpected"] + ); + assert.equal(Object.keys(f.database.state.local).length, 2); +}); + +test("content edited after planning is merged with the incoming snapshot", async () => { + const f = await fixture({ "a.md": "base\n" }); + const expected = (await f.files.snapshot("a.md"))!; + await f.disk.userWrite("a.md", bytes("base\nlocal\n")); + await f.files.apply(structuredClone(f.database.state), { + a: { + expected, + replacement: await toStoredSnapshot({ + content: bytes("remote\nbase\n") + }) + } + }); + assert.deepEqual( + f.disk.userFiles().get("a.md"), + bytes("remote\nbase\nlocal\n") + ); +}); + +test("deleting the source after planning does not resurrect it", async () => { + const f = await fixture(); + const next = structuredClone(f.database.state); + next.local.a = "remote.md"; + await f.disk.userDelete("a.md"); + await f.files.apply(next, { + a: { replacement: await toStoredSnapshot({ content: bytes("remote") }) } + }); + assert.deepEqual(f.disk.userFiles(), new Map()); + assert.deepEqual(f.database.state.local, {}); +}); + +for (const protect of ["ignore", "size"] as const) + test(`protect ${protect} at application time`, async () => { + const f = await fixture(); + const next = structuredClone(f.database.state); + next.local.a = "remote.md"; + if (protect === "ignore") + await f.settings.setSetting("ignorePatterns", ["a.md"]); + else await f.settings.setSetting("maxFileSizeMB", 0); + await f.files.apply(next, { + a: { + replacement: await toStoredSnapshot({ + content: bytes("remote") + }) + } + }); + assert.deepEqual( + f.disk.userFiles(), + new Map([["a.md", bytes("local")]]) + ); + assert.equal(f.database.state.local.a, "a.md"); + assert.equal(f.database.state.excluded?.a, "a.md"); + }); + +for (const when of ["planning", "installed"]) + test(`notified move during ${when} is attributed to the right identity on rescan`, async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + let changed = false; + f.disk.boundary = async (label) => { + if ( + changed || + label !== + (when === "planning" + ? "read:stat:a.md" + : "visible:rename:a.md->b.md") + ) + return; + changed = true; + const oldPath = when === "planning" ? "a.md" : "b.md"; + await f.disk.userRename(oldPath, "editor.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath, + relativePath: "editor.md" + }); + }; + const next = structuredClone(f.database.state); + next.local = { a: "b.md", b: "a.md" }; + if (when === "planning") + await assert.rejects( + f.files.apply(next), + LocalChangesDuringReconciliation + ); + else await f.files.apply(next); + assert(changed); + await f.internals.scan(); + assert.equal(f.database.state.local.a, "editor.md"); + assert.deepEqual( + f.disk.userFiles().get(f.database.state.local.a), + bytes("A") + ); + assert.deepEqual( + f.disk.userFiles().get(f.database.state.local.b), + bytes("B") + ); + }); + +for (const action of ["move", "delete"] as const) + test(`notified ${action}/recreate between file operations does not repurpose the replacement file`, async () => { + const f = await fixture({ "a.md": "A", "b.md": "B" }); + await f.settings.setSetting("ignorePatterns", ["private.md"]); + let changed = false; + f.disk.boundary = async (label) => { + if (changed || label !== "visible:rename:a.md->remote-a.md") return; + changed = true; + if (action === "move") { + await f.disk.userRename("b.md", "private.md"); + await f.syncer.syncLocallyUpdatedFile({ + oldPath: "b.md", + relativePath: "private.md" + }); + } else { + await f.disk.userDelete("b.md"); + await f.syncer.syncLocallyDeletedFile("b.md"); + } + await f.disk.userWrite("b.md", bytes("replacement")); + await f.syncer.syncLocallyCreatedFile("b.md"); + }; + const next = structuredClone(f.database.state); + next.local = { a: "remote-a.md", b: "remote-b.md" }; + await f.files.apply(next); + assert(changed); + await f.internals.scan(); + assert.deepEqual(f.disk.userFiles().get("b.md"), bytes("replacement")); + assert.equal(f.disk.userFiles().has("remote-b.md"), false); + assert.equal( + f.database.state.local.b, + action === "move" ? "private.md" : undefined + ); + if (action === "move") + assert.deepEqual(f.disk.userFiles().get("private.md"), bytes("B")); + }); + +test("a file and nested directory exchange their exact paths", async () => { + const f = await fixture({ a: "A", "b/deep/child": "B" }); + const [a, b] = Object.keys(f.database.state.local); + const next = structuredClone(f.database.state); + next.local = { [a]: "b", [b]: "a/deep/child" }; + await f.files.apply(next); + assert.deepEqual(f.database.state.local, { [a]: "b", [b]: "a/deep/child" }); + assert.deepEqual( + f.disk.userFiles(), + new Map([ + ["b", bytes("A")], + ["a/deep/child", bytes("B")] + ]) + ); +}); diff --git a/frontend/test-support/journal-interleavings.test.ts b/frontend/test-support/journal-interleavings.test.ts deleted file mode 100644 index 93f0bbb0..00000000 --- a/frontend/test-support/journal-interleavings.test.ts +++ /dev/null @@ -1,407 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { fixture, bytes, head } from "./sync-fixture"; -import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; - -for (const origin of ["bootstrap", "history recovery"]) - test(`cancelling a replacement retains the ${origin} merge context`, async () => { - const f = await fixture( - { "a.md": "BASE\n" }, - { getDocumentVersionContent: async () => bytes("REMOTE\nBASE\n") } - ); - const recoveryBase = await toStoredSnapshot({ - content: bytes("BASE\n") - }); - if (origin === "bootstrap") - f.database.state.documents.a.bootstrap = true; - else f.database.state.documents.a.recoveryBase = recoveryBase; - await f.database.save(); - await f.settings.setSetting("ignorePatterns", ["private.md"]); - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith("before:mkdir:.vault-link-sync/transactions/") - ) - return; - injected = true; - await f.disk.userRename("a.md", "private.md"); - await f.disk.userWrite("private.md", bytes("BASE\nLOCAL\n")); - await f.syncer.syncLocallyUpdatedFile({ - oldPath: "a.md", - relativePath: "private.md" - }); - }; - await f.internals.incorporateContent(head("a", 2, "REMOTE\nBASE\n")); - assert(injected); - const deferred = f.database.state.documents.a; - assert.equal(deferred.base, undefined); - if (origin === "bootstrap") assert.equal(deferred.bootstrap, true); - else assert.deepEqual(deferred.recoveryBase, recoveryBase); - f.disk.boundary = () => {}; - await f.settings.setSetting("ignorePatterns", []); - await f.internals.incorporateFileManifest( - f.database.state.fileManifest - ); - const content = Buffer.from( - f.disk.userFiles().get("private.md")! - ).toString(); - assert.equal(content.match(/REMOTE/g)?.length, 1); - assert.equal(content.match(/LOCAL/g)?.length, 1); - assert.equal(f.database.state.documents.a.base?.vaultUpdateId, 2); - assert.equal(f.database.state.documents.a.recoveryBase, undefined); - }); - -for (const timing of ["before", "durable"]) - test(`a save that exceeds the size limit ${timing} staging keeps its exact bytes`, async () => { - const f = await fixture({ "a.bin": "PUBLIC" }); - await f.settings.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); - const large = "PRIVATE".repeat(100); - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith( - `${timing}:rename:a.bin->.vault-link-sync/transactions/` - ) - ) - return; - injected = true; - await f.disk.userWrite("a.bin", bytes(large)); - await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.bin" }); - }; - await f.files.apply(structuredClone(f.database.state), { - a: { - expected: await toStoredSnapshot({ content: bytes("PUBLIC") }), - replacement: await toStoredSnapshot({ - content: bytes("REMOTE") - }) - } - }); - assert(injected); - assert.equal( - Buffer.from(f.disk.userFiles().get("a.bin")!).toString(), - large - ); - assert.equal(f.database.state.excluded?.a, "a.bin"); - assert.deepEqual(f.database.state.local, { a: "a.bin" }); - }); - -test("an excluded move after installation survives an uncertain journal acknowledgement", async () => { - const f = await fixture({ "a.md": "PUBLIC" }); - await f.settings.setSetting("ignorePatterns", ["private.md"]); - let moved = false, - crashed = false; - f.disk.boundary = async (label) => { - if ( - moved || - !label.startsWith("durable:rename:") || - !label.endsWith(".output->a.md") - ) - return; - moved = true; - await f.disk.userRename("a.md", "private.md"); - await f.disk.userWrite("private.md", bytes("SECRET")); - await f.syncer.syncLocallyUpdatedFile({ - oldPath: "a.md", - relativePath: "private.md" - }); - }; - f.persistence.boundary = (label) => { - if (!moved || crashed || label !== "durable:save") return; - crashed = true; - throw new Error("lost journal acknowledgement"); - }; - await assert.rejects( - f.files.apply(structuredClone(f.database.state), { - a: { - expected: await toStoredSnapshot({ content: bytes("PUBLIC") }), - replacement: await toStoredSnapshot({ - content: bytes("PUBLIC REMOTE") - }) - } - }), - /lost journal acknowledgement/ - ); - assert(moved && crashed); - f.disk.boundary = () => {}; - f.persistence.boundary = () => {}; - await f.database.recoverPersistence(); - await f.files.recover(); - assert.deepEqual(f.database.state.local, { a: "private.md" }); - assert.equal( - Buffer.from(f.disk.userFiles().get("private.md")!).toString(), - "SECRET" - ); -}); - -test("moving a tracked file into an ignored path during a journal leaves it protected", async () => { - const f = await fixture({ "a.md": "PUBLIC", "b.md": "OTHER" }); - await f.settings.setSetting("ignorePatterns", ["private.md"]); - const next = structuredClone(f.database.state); - next.local.b = "remote-b.md"; - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith("before:mkdir:.vault-link-sync/transactions/") - ) - return; - injected = true; - await f.disk.userRename("a.md", "private.md"); - await f.disk.userWrite("private.md", bytes("SECRET")); - await f.syncer.syncLocallyUpdatedFile({ - oldPath: "a.md", - relativePath: "private.md" - }); - }; - await f.files.apply(next); - assert(injected); - assert.equal( - Buffer.from(f.disk.userFiles().get("private.md") ?? []).toString(), - "SECRET" - ); - assert.equal(f.database.state.local.a, "private.md"); - assert.equal(f.database.state.excluded?.a, "a.md"); -}); - -test("a directory move protects each ignored descendant during a journal", async () => { - const f = await fixture({ "folder/a.md": "PUBLIC", "b.md": "OTHER" }); - await f.settings.setSetting("ignorePatterns", ["private/**"]); - const next = structuredClone(f.database.state); - next.local.b = "remote-b.md"; - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith("before:mkdir:.vault-link-sync/transactions/") - ) - return; - injected = true; - await f.disk.userRename("folder/a.md", "private/a.md"); - await f.disk.userWrite("private/a.md", bytes("SECRET")); - await f.syncer.syncLocallyUpdatedFile({ - oldPath: "folder", - relativePath: "private" - }); - }; - await f.files.apply(next); - assert(injected); - assert.equal( - Buffer.from(f.disk.userFiles().get("private/a.md") ?? []).toString(), - "SECRET" - ); - assert.equal(f.database.state.local["folder/a"], "private/a.md"); - assert.equal(f.database.state.excluded?.["folder/a"], "folder/a.md"); -}); - -test("a later editor save replaces earlier unsynced text after a merge was prepared", async () => { - const f = await fixture({ "a.md": "base" }); - const next = structuredClone(f.database.state); - let saves = 0; - f.disk.boundary = async (label) => { - if (saves === 0 && label.startsWith("before:rename:a.md->")) { - saves++; - await f.disk.userWrite("a.md", bytes("base ONE")); - await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); - } else if ( - saves === 1 && - label.startsWith("before:write:") && - label.endsWith(".output") - ) { - saves++; - await f.disk.userWrite("a.md", bytes("base TWO")); - await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); - } - }; - await f.files.apply(next, { - a: { - expected: await toStoredSnapshot({ content: bytes("base") }), - replacement: await toStoredSnapshot({ - content: bytes("base REMOTE") - }) - } - }); - assert.equal(saves, 2); - assert.equal( - Buffer.from(f.disk.userFiles().get("a.md")!).toString(), - "base TWO REMOTE" - ); - assert.deepEqual(f.database.state.local, { a: "a.md" }); -}); - -test("a content save during remote application remains on the same document", async () => { - const f = await fixture({ "a.md": "base" }); - f.database.state.documents.a.base = { - ...head("a", 1, "base"), - hash: f.database.state.documents.a.observedHash! - }; - const next = structuredClone(f.database.state); - next.documents.a.base = { - ...head("a", 2, "base REMOTE"), - hash: (await toStoredSnapshot({ content: bytes("base REMOTE") })).hash - }; - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith( - "durable:rename:a.md->.vault-link-sync/transactions/" - ) - ) - return; - injected = true; - await f.disk.userWrite("a.md", bytes("base LOCAL")); - await f.syncer.syncLocallyUpdatedFile({ relativePath: "a.md" }); - }; - await f.files.apply(next, { - a: { - expected: await toStoredSnapshot({ content: bytes("base") }), - replacement: await toStoredSnapshot({ - content: bytes("base REMOTE") - }) - } - }); - - assert.equal(f.disk.userFiles().size, 1); - const result = Buffer.from( - (await f.disk.readSnapshot("a.md"))!.content - ).toString(); - assert(result.includes("LOCAL") && result.includes("REMOTE")); -}); - -async function setup() { - const f = await fixture({ "a.md": "ORIGINAL", "b.md": "OTHER" }); - f.database.state.documents.a.base = { - ...head("a", 1, "ORIGINAL"), - hash: f.database.state.documents.a.observedHash! - }; - const next = structuredClone(f.database.state); - next.local = { a: "remote.md", b: "remote-b.md" }; - next.fileManifest = { fileManifestId: 3, entries: { ...next.local } }; - const expected = await toStoredSnapshot({ content: bytes("ORIGINAL") }); - const replacement = await toStoredSnapshot({ - content: bytes("ORIGINAL REMOTE") - }); - next.documents.a.base = { - ...head("a", 2, "ORIGINAL REMOTE"), - hash: replacement.hash - }; - return { f, next, writes: { a: { expected, replacement } } }; -} -test("editor changes preserve exact bytes and identities at every filesystem read, mutation and persistence boundary", async (t) => { - const { f, next, writes } = await setup(); - const labels: string[] = []; - const trace = (l: string) => { - labels.push(l); - }; - f.disk.boundary = trace; - f.persistence.boundary = trace; - await f.files.apply(next, writes); - const failures = []; - let exercised = 0; - for (const kind of ["move", "move-recreate", "delete-recreate", "save"]) - for (let target = 0; target < labels.length; target++) { - const { f, next, writes } = await setup(); - let count = 0, - injected = false, - from: string | undefined; - const boundary = async (l: string) => { - if (injected) return; - if (count++ !== target) return; - const files = f.disk.userFiles(); - from = files.has("a.md") - ? "a.md" - : files.has("remote.md") - ? "remote.md" - : undefined; - if (kind === "save" && !from) from = "a.md"; - if (!from) return; - injected = true; - exercised++; - if (kind === "save") { - await f.disk.userWrite( - from, - bytes( - Buffer.from( - files.get(from) ?? bytes("ORIGINAL") - ).toString() + " LOCAL" - ) - ); - await f.syncer.syncLocallyUpdatedFile({ - relativePath: from - }); - } else if (kind === "delete-recreate") { - await f.disk.userDelete(from); - await f.syncer.syncLocallyDeletedFile(from); - } else { - await f.disk.userRename(from, "mine.md"); - await f.syncer.syncLocallyUpdatedFile({ - oldPath: from, - relativePath: "mine.md" - }); - } - if (kind !== "move" && kind !== "save") { - await f.disk.userWrite(from, bytes("REPLACEMENT")); - await f.syncer.syncLocallyCreatedFile(from); - } - }; - f.disk.boundary = boundary; - f.persistence.boundary = boundary; - let error; - try { - await f.files.apply(next, writes); - } catch (e) { - error = String(e); - } - f.disk.boundary = () => {}; - f.persistence.boundary = () => {}; - try { - await f.database.recoverPersistence(); - await f.files.recover(); - await f.internals.scan(); - } catch (e) { - error += ";retry=" + String(e); - } - if (!injected) continue; - const fs = Object.fromEntries( - [...f.disk.userFiles()].map(([p, b]) => [ - p, - Buffer.from(b).toString() - ]) - ); - const bad = - kind === "save" - ? ![ - "ORIGINAL LOCAL REMOTE", - "ORIGINAL REMOTE LOCAL" - ].includes(fs[f.database.state.local.a]) || - Object.keys(f.database.state.local).length !== 2 - : kind === "delete-recreate" - ? f.database.state.local.a !== undefined - : f.database.state.local.a !== "mine.md" || - fs["mine.md"] !== "ORIGINAL REMOTE"; - if ( - bad || - (kind !== "move" && - kind !== "save" && - fs[from!] !== "REPLACEMENT") - ) - failures.push({ - kind, - target, - label: labels[target], - error, - ids: f.database.state.local, - files: fs - }); - } - assert( - exercised > 50, - "The sweep must actually exercise the injected actions" - ); - t.diagnostic( - `${labels.length} boundaries; ${exercised} editor interleavings` - ); - assert.deepEqual(failures, []); -}); diff --git a/frontend/test-support/journal-protocol.test.ts b/frontend/test-support/journal-protocol.test.ts deleted file mode 100644 index 568b9688..00000000 --- a/frontend/test-support/journal-protocol.test.ts +++ /dev/null @@ -1,612 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; -import { fixture, bytes, randomUUID } from "./sync-server-fixture"; - -for (const exclusion of ["size", "ignore"] as const) - for (const phase of [ - "planned", - ...(exclusion === "size" ? ["staged", "prepared"] : []), - "installed" - ]) - for (const recovery of [ - "none", - "restart", - "before:save", - "durable:save" - ] as const) - test(`reenabling ${exclusion} exclusion at ${phase} preserves the content base through ${recovery}`, async () => { - const f = await fixture(); - try { - const installed = phase === "installed"; - const id = randomUUID(); - const base = await f.put( - "/documents/" + id, - f.content("BASE\n") - ); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "a.md" } - }); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.client.setSettings({ - maxFileSizeMB: - exclusion === "size" ? 100 / (1024 * 1024) : 1, - ignorePatterns: ["private.md"] - }); - const remote = await f.put( - "/documents/" + id, - f.content("REMOTE\nBASE\n", base.vaultUpdateId) - ); - const local = "BASE\n" + "LOCAL\n".repeat(30); - const path = exclusion === "size" ? "a.md" : "private.md"; - let injected = false, - armed = false, - crashed = false; - f.disk.boundary = async (label) => { - const atBoundary = - phase === "installed" - ? label.startsWith("durable:rename:") && - label.endsWith(".output->a.md") - : phase === "staged" - ? label.startsWith("durable:rename:a.md->") - : phase === "prepared" - ? label.startsWith("before:write:") && - label.endsWith(".output") - : label.startsWith( - "before:mkdir:.vault-link-sync/transactions/" - ); - if (injected || !atBoundary) return; - injected = true; - if (exclusion === "ignore") - await f.disk.userRename("a.md", path); - // When installation already completed, deleting REMOTE - // is a subsequent local edit, which must not be undone. - await f.disk.userWrite(path, bytes(local)); - await f.client.syncLocallyUpdatedFile({ - relativePath: path, - ...(exclusion === "ignore" - ? { oldPath: "a.md" } - : {}) - }); - armed = true; - }; - f.store.boundary = (label) => { - if (!armed || crashed || label !== recovery) return; - crashed = true; - f.disk.crash(true); - throw new Error("interrupted exclusion save"); - }; - if (recovery.endsWith(":save")) { - await assert.rejects( - f.client.setSetting("isSyncEnabled", true), - /interrupted exclusion save/ - ); - assert(crashed); - } else await f.client.setSetting("isSyncEnabled", true); - assert(injected); - f.disk.boundary = f.store.boundary = () => {}; - if (recovery !== "none") { - await f.reopenClient(); - await f.client.start(); - } - - assert.deepEqual( - (await f.disk.readSnapshot(path))?.content, - bytes(local) - ); - const stored = f.store.snapshot().database!; - assert.equal( - stored.documents![id].base?.vaultUpdateId, - installed ? remote.vaultUpdateId : base.vaultUpdateId - ); - assert.equal( - stored.remoteHeads![id].vaultUpdateId, - remote.vaultUpdateId - ); - assert.equal( - (await f.get("/documents/" + id)).contentBase64, - Buffer.from("REMOTE\nBASE\n").toString("base64") - ); - await f.client.setSettings({ - maxFileSizeMB: 1, - ignorePatterns: [] - }); - const expected = installed ? local : "REMOTE\n" + local; - const verify = async () => { - assert.deepEqual( - (await f.get("/file-manifest")).entries, - { [id]: path } - ); - assert.deepEqual( - (await f.disk.readSnapshot(path))?.content, - bytes(expected) - ); - assert.equal( - (await f.get("/documents/" + id)).contentBase64, - Buffer.from(expected).toString("base64") - ); - }; - await verify(); - await f.client.reset(); - await verify(); - } finally { - await f.dispose(); - } - }); - -for (const at of [undefined, "before:save", "durable:save"]) - for (const powerLoss of at ? [false, true] : [false]) - test(`repeated saves retain only the latest local edit through ${at ?? "completion"} and powerLoss=${powerLoss}`, async () => { - const f = await fixture(); - try { - const id = randomUUID(); - const base = await f.put("/documents/" + id, f.content("base")); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "a.md" } - }); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.put( - "/documents/" + id, - f.content("base REMOTE", base.vaultUpdateId) - ); - let saves = 0, - armed = false, - crashed = false; - f.disk.boundary = async (label) => { - if ( - saves === 0 && - label.startsWith("before:rename:a.md->") - ) { - saves++; - await f.disk.userWrite("a.md", bytes("base ONE")); - await f.client.syncLocallyUpdatedFile({ - relativePath: "a.md" - }); - } else if ( - saves === 1 && - label.startsWith("before:write:") && - label.endsWith(".output") - ) { - saves++; - await f.disk.userWrite("a.md", bytes("base TWO")); - await f.client.syncLocallyUpdatedFile({ - relativePath: "a.md" - }); - armed = true; - } - }; - f.store.boundary = (label) => { - if (!armed || crashed || label !== at) return; - crashed = true; - f.disk.crash(powerLoss); - throw new Error("injected repeated-save crash"); - }; - if (at) { - await assert.rejects( - f.client.setSetting("isSyncEnabled", true), - /injected repeated-save crash/ - ); - assert(crashed); - f.disk.boundary = () => {}; - f.store.boundary = () => {}; - await f.reopenClient(); - await f.client.start(); - } else await f.client.setSetting("isSyncEnabled", true); - assert.equal(saves, 2); - const verify = async () => { - assert.deepEqual((await f.get("/file-manifest")).entries, { - [id]: "a.md" - }); - assert.deepEqual( - (await f.disk.readSnapshot("a.md"))?.content, - bytes("base TWO REMOTE") - ); - assert.equal( - Buffer.from( - (await f.get("/documents/" + id)).contentBase64, - "base64" - ).toString(), - "base TWO REMOTE" - ); - }; - await verify(); - await f.client.reset(); - await verify(); - } finally { - await f.dispose(); - } - }); - -for (const timing of ["before", "durable"]) - for (const crash of [false, true]) - test(`oversized save ${timing} staging stays local across crash=${crash}`, async () => { - const f = await fixture(); - try { - const id = randomUUID(), - large = "PRIVATE".repeat(100); - const base = await f.put( - "/documents/" + id, - f.content("PUBLIC") - ); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "a.bin" } - }); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.client.setSetting("maxFileSizeMB", 100 / (1024 * 1024)); - await f.put( - "/documents/" + id, - f.content("REMOTE", base.vaultUpdateId) - ); - let injected = false, - armed = false, - crashed = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith( - `${timing}:rename:a.bin->.vault-link-sync/transactions/` - ) - ) - return; - injected = true; - await f.disk.userWrite("a.bin", bytes(large)); - await f.client.syncLocallyUpdatedFile({ - relativePath: "a.bin" - }); - armed = true; - }; - f.store.boundary = (label) => { - if (!crash || !armed || crashed || label !== "durable:save") - return; - crashed = true; - f.disk.crash(true); - throw new Error("injected oversized-save crash"); - }; - if (crash) { - await assert.rejects( - f.client.setSetting("isSyncEnabled", true), - /injected oversized-save crash/ - ); - assert(crashed); - f.disk.boundary = () => {}; - f.store.boundary = () => {}; - await f.reopenClient(); - await f.client.start(); - } else await f.client.setSetting("isSyncEnabled", true); - assert(injected); - const verify = async () => { - assert.deepEqual( - (await f.disk.readSnapshot("a.bin"))?.content, - bytes(large) - ); - assert.deepEqual((await f.get("/file-manifest")).entries, { - [id]: "a.bin" - }); - assert.equal( - Buffer.from( - (await f.get("/documents/" + id)).contentBase64, - "base64" - ).toString(), - "REMOTE" - ); - }; - await verify(); - await f.client.reset(); - await verify(); - } finally { - await f.dispose(); - } - }); - -test("a tracked file moved into an ignored path during remote application never uploads its new bytes", async () => { - const f = await fixture(); - try { - const a = randomUUID(), - b = randomUUID(); - await f.put("/documents/" + a, f.content("PUBLIC")); - await f.put("/documents/" + b, f.content("OTHER")); - const m = await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [a]: "a.md", [b]: "b.md" } - }); - await f.client.setSetting("ignorePatterns", ["private.md"]); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: m.fileManifestId, - entries: { [a]: "a.md", [b]: "remote-b.md" } - }); - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith("before:mkdir:.vault-link-sync/transactions/") - ) - return; - injected = true; - await f.disk.userRename("a.md", "private.md"); - await f.disk.userWrite("private.md", bytes("SECRET")); - await f.client.syncLocallyUpdatedFile({ - oldPath: "a.md", - relativePath: "private.md" - }); - }; - await f.client.setSetting("isSyncEnabled", true); - assert(injected); - const manifest = await f.get("/file-manifest"); - const contents = await Promise.all( - Object.keys(manifest.entries).map((id) => f.get("/documents/" + id)) - ); - assert( - !contents.some( - (c) => - Buffer.from(c.contentBase64, "base64").toString() === - "SECRET" - ) - ); - assert.deepEqual( - (await f.disk.readSnapshot("private.md"))?.content, - bytes("SECRET") - ); - assert.equal(manifest.entries[a], "a.md"); - } finally { - await f.dispose(); - } -}); -test("remote creation plus unrelated notification must not delete remote membership on retry", async () => { - const f = await fixture(); - try { - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - const id = randomUUID(); - await f.put("/documents/" + id, f.content("REMOTE")); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "remote.md" } - }); - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith("before:mkdir:.vault-link-sync/transactions/") - ) - return; - injected = true; - await f.disk.userWrite("local.md", bytes("LOCAL")); - await f.client.syncLocallyCreatedFile("local.md"); - }; - await f.client.setSetting("isSyncEnabled", true); - assert(injected); - f.disk.boundary = () => {}; - await f.client.reset(); - const manifest = await f.get("/file-manifest"); - - assert.equal(manifest.entries[id], "remote.md"); - } finally { - await f.dispose(); - } -}); -test("notified ignored occupant during remote rename must never upload its bytes", async () => { - const f = await fixture(); - try { - const id = randomUUID(); - await f.put("/documents/" + id, f.content("REMOTE")); - const m = await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "a.md" } - }); - await f.client.setSetting("ignorePatterns", ["private"]); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: m.fileManifestId, - entries: { [id]: "private/note.md" } - }); - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith("before:mkdir:.vault-link-sync/transactions/") - ) - return; - injected = true; - await f.disk.userWrite("private", bytes("SECRET")); - await f.client.syncLocallyCreatedFile("private"); - }; - await f.client.setSetting("isSyncEnabled", true); - f.disk.boundary = () => {}; - const manifest = await f.get("/file-manifest"); - const contents = await Promise.all( - Object.keys(manifest.entries).map((id) => f.get("/documents/" + id)) - ); - - assert(injected); - assert.deepEqual( - (await f.disk.readSnapshot("private"))?.content, - bytes("SECRET") - ); - assert( - !contents.some( - (c) => - Buffer.from(c.contentBase64, "base64").toString() === - "SECRET" - ) - ); - } finally { - await f.dispose(); - } -}); -test("notified delete/recreate before staging retains replacement file", async () => { - const f = await fixture(); - try { - const id = randomUUID(); - const h = await f.put("/documents/" + id, f.content("ORIGINAL")); - const m = await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "a.md" } - }); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.put( - "/documents/" + id, - f.content("ORIGINAL REMOTE", h.vaultUpdateId) - ); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: m.fileManifestId, - entries: { [id]: "remote.md" } - }); - let injected = false; - f.disk.boundary = async (label) => { - if ( - injected || - !label.startsWith( - "before:rename:a.md->.vault-link-sync/transactions/" - ) - ) - return; - injected = true; - await f.disk.userDelete("a.md"); - await f.client.syncLocallyDeletedFile("a.md"); - await f.disk.userWrite("a.md", bytes("REPLACEMENT")); - await f.client.syncLocallyCreatedFile("a.md"); - }; - await f.client.setSetting("isSyncEnabled", true); - f.disk.boundary = () => {}; - const manifest = await f.get("/file-manifest"); - - assert.deepEqual( - (await f.disk.readSnapshot("a.md"))?.content, - bytes("REPLACEMENT") - ); - } finally { - await f.dispose(); - } -}); - -for (const mode of ["delete", "move", "save"] as const) - for (const powerLoss of [false, true]) - for (const at of ["before:save", "durable:save"]) - test(`${mode} during staging survives ${at} and ${powerLoss ? "power loss" : "process replacement"}`, async () => { - const f = await fixture(); - try { - const id = randomUUID(); - const h = await f.put( - "/documents/" + id, - f.content("ORIGINAL") - ); - await f.put("/file-manifest", { - requestId: randomUUID(), - parentFileManifestId: 0, - entries: { [id]: "a.md" } - }); - await f.client.start(); - await f.client.setSetting("isSyncEnabled", false); - await f.put( - "/documents/" + id, - f.content("ORIGINAL REMOTE", h.vaultUpdateId) - ); - let injected = false, - armed = false, - crashed = false; - f.disk.boundary = async (label) => { - const point = mode === "save" ? "durable" : "before"; - if ( - injected || - !label.startsWith( - `${point}:rename:a.md->.vault-link-sync/transactions/` - ) - ) - return; - injected = true; - if (mode === "save") { - await f.disk.userWrite( - "a.md", - bytes("ORIGINAL LOCAL") - ); - await f.client.syncLocallyUpdatedFile({ - relativePath: "a.md" - }); - } else { - if (mode === "delete") { - await f.disk.userDelete("a.md"); - await f.client.syncLocallyDeletedFile("a.md"); - } else { - await f.disk.userRename("a.md", "mine.md"); - await f.client.syncLocallyUpdatedFile({ - oldPath: "a.md", - relativePath: "mine.md" - }); - } - await f.disk.userWrite( - "a.md", - bytes("REPLACEMENT") - ); - await f.client.syncLocallyCreatedFile("a.md"); - } - armed = true; - }; - f.store.boundary = (label) => { - if (!armed || crashed || label !== at) return; - crashed = true; - f.disk.crash(powerLoss); - throw new Error("injected journal crash"); - }; - await assert.rejects( - f.client.setSetting("isSyncEnabled", true), - /injected journal crash/ - ); - assert(injected && crashed); - f.disk.boundary = () => {}; - f.store.boundary = () => {}; - await f.reopenClient(); - await f.client.start(); - const manifest = await f.get("/file-manifest"); - if (mode === "save") { - assert.deepEqual(manifest.entries, { [id]: "a.md" }); - const actual = Buffer.from( - (await f.disk.readSnapshot("a.md"))!.content - ).toString(); - assert.equal(actual, "ORIGINAL LOCAL REMOTE"); - } else { - assert.deepEqual( - (await f.disk.readSnapshot("a.md"))?.content, - bytes("REPLACEMENT") - ); - assert.notEqual( - Object.keys(manifest.entries).find( - (key) => manifest.entries[key] === "a.md" - ), - id - ); - if (mode === "move") { - assert.equal(manifest.entries[id], "mine.md"); - assert.deepEqual( - (await f.disk.readSnapshot("mine.md"))?.content, - bytes("ORIGINAL REMOTE") - ); - } else assert.equal(manifest.entries[id], undefined); - } - await f.client.reset(); - assert.deepEqual( - (await f.get("/file-manifest")).entries, - manifest.entries - ); - } finally { - await f.dispose(); - } - }); diff --git a/frontend/test-support/oracles.ts b/frontend/test-support/oracles.ts index f50141ef..bc59b971 100644 --- a/frontend/test-support/oracles.ts +++ b/frontend/test-support/oracles.ts @@ -111,16 +111,6 @@ export function assertQuiescent( undefined, "Pending CAS request at convergence" ); - assert.equal( - state.eventReplay, - undefined, - "Unfinished event replay at convergence" - ); - assert.equal( - state.application, - undefined, - "Unfinished filesystem journal at convergence" - ); assert.deepEqual( state.local, entries, diff --git a/frontend/test-support/polling.test.ts b/frontend/test-support/polling.test.ts index 37627b5e..18187dae 100644 --- a/frontend/test-support/polling.test.ts +++ b/frontend/test-support/polling.test.ts @@ -101,7 +101,6 @@ async function fixture(syncIntervalMs: number | undefined) { const persistence = new MemoryPersistence({ database: initial }); const logger = new Logger(); const database = new Database( - logger, initial, async (next) => persistence.save({ database: next }), initial.vaultKey, diff --git a/frontend/test-support/protocol.test.ts b/frontend/test-support/protocol.test.ts index d0652d79..50b0a38d 100644 --- a/frontend/test-support/protocol.test.ts +++ b/frontend/test-support/protocol.test.ts @@ -248,7 +248,6 @@ for (const [kind, format, originalPath] of [ "Restart changed the persisted request" ); assert.equal(restarted.stored.database!.pending, undefined); - assert.equal(restarted.stored.database!.application, undefined); const recovered = new MemoryDisk(); recovered.restore(restarted.disk); assert.deepEqual( diff --git a/frontend/test-support/real-disk.test.ts b/frontend/test-support/real-disk.test.ts index 3b917c81..1819f632 100644 --- a/frontend/test-support/real-disk.test.ts +++ b/frontend/test-support/real-disk.test.ts @@ -8,12 +8,11 @@ import { promisify } from "node:util"; import { RealDisk } from "./real-disk"; import { Database, emptyState } from "../sync-client/src/persistence/database"; import { FileOperations } from "../sync-client/src/file-operations/file-operations"; -import { Logger } from "../sync-client/src/tracing/logger"; import type { ServerConfig } from "../sync-client/src/services/server-config"; import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; test( - "real filesystem: exclusive operations, unsafe entries, and journal swaps", + "real filesystem: exclusive operations, unsafe entries, and file swaps", { timeout: 20_000 }, async () => { const directory = await fs.mkdtemp( @@ -53,11 +52,9 @@ test( ); await disk.write("a.md", snapshot("A")); await disk.write("b.md", snapshot("B")); - // Recovery must flush the old source's ancestors even if another - // writer has replaced one of those directories with a file. + // A file in an ancestor position makes descendants absent. assert.equal(await disk.stat("a.md/absent.md"), undefined); assert.equal(await disk.readSnapshot("a.md/absent.md"), undefined); - await disk.flushPaths(["a.md/absent.md"]); await assert.rejects(disk.rename("a.md", "b.md")); await assert.rejects(disk.delete("a.md"), /regular file/); await fs.symlink("a.md", path.join(root, "symlink")); @@ -105,7 +102,6 @@ test( ).hash }; const database = new Database( - new Logger(), initial, async () => {}, "real", @@ -129,7 +125,6 @@ test( ).toString(), "A" ); - assert.equal(database.state.application, undefined); } finally { await fs.rm(directory, { recursive: true }); } diff --git a/frontend/test-support/real-disk.ts b/frontend/test-support/real-disk.ts index d3c42639..ffe05c40 100644 --- a/frontend/test-support/real-disk.ts +++ b/frontend/test-support/real-disk.ts @@ -4,7 +4,6 @@ import * as path from "node:path"; import { constants } from "node:fs"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { randomUUID } from "node:crypto"; import type { FileSystemOperations, FileSnapshot } from "sync-client"; const exec = promisify(execFile); @@ -52,17 +51,6 @@ export class RealDisk implements FileSystemOperations { throw error; } } - private async flush(absolute: string): Promise { - const handle = await fs.open( - absolute, - constants.O_RDONLY | constants.O_NOFOLLOW - ); - try { - await handle.sync(); - } finally { - await handle.close(); - } - } public async stat(relative: string) { const info = await this.info(await this.resolve(relative)); return info @@ -140,8 +128,6 @@ export class RealDisk implements FileSystemOperations { throw error; } assert((await fs.lstat(next)).isDirectory()); - await this.flush(next); - await this.flush(current); current = next; } } @@ -154,44 +140,16 @@ export class RealDisk implements FileSystemOperations { "This disk-only fixture has no editor selections" ); const destination = await this.resolve(relative); - const staging = ".vault-link-sync/adapter-staging"; - await this.createDirectory(staging); - const source = path.join(this.root, staging, randomUUID()); - const handle = await fs.open(source, "wx", 0o600); - try { - await handle.writeFile(snapshot.content); - await handle.sync(); - } finally { - await handle.close(); - } - try { - await exec(this.helper, ["rename", source, destination]); - await this.flush(path.dirname(source)); - await this.flush(path.dirname(destination)); - } finally { - await fs.unlink(source).catch((error: NodeJS.ErrnoException) => { - if (error.code !== "ENOENT") throw error; - }); - } + await fs.writeFile(destination, snapshot.content, { + flag: "wx", + mode: 0o600 + }); } public async rename(from: string, to: string): Promise { const source = await this.resolve(from), destination = await this.resolve(to); assert((await this.info(source))?.isFile()); - await this.flush(source); await exec(this.helper, ["rename", source, destination]); - await this.flush(path.dirname(source)); - await this.flush(path.dirname(destination)); - } - public async flushPaths(paths: readonly string[]): Promise { - for (const relative of paths) { - let absolute = await this.resolve(relative); - for (;;) { - if (await this.info(absolute)) await this.flush(absolute); - if (absolute === this.root) break; - absolute = path.dirname(absolute); - } - } } public async delete(relative: string): Promise { assert(relative, "Cannot prune root"); @@ -202,7 +160,6 @@ export class RealDisk implements FileSystemOperations { for (const child of await fs.readdir(absolute)) await this.delete(`${relative}/${child}`); await fs.rmdir(absolute); - await this.flush(path.dirname(absolute)); } public async deleteFile(relative: string): Promise { const absolute = await this.resolve(relative); @@ -210,6 +167,5 @@ export class RealDisk implements FileSystemOperations { if (!entry) return; assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); await fs.unlink(absolute); - await this.flush(path.dirname(absolute)); } } diff --git a/frontend/test-support/recovery.test.ts b/frontend/test-support/recovery.test.ts index 8da22c4a..25163213 100644 --- a/frontend/test-support/recovery.test.ts +++ b/frontend/test-support/recovery.test.ts @@ -1,6 +1,5 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { isDeepStrictEqual } from "node:util"; import { MemoryDisk, MemoryPersistence } from "./storage"; import { Database, @@ -9,7 +8,6 @@ import { } from "../sync-client/src/persistence/database"; import { FileOperations } from "../sync-client/src/file-operations/file-operations"; import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; -import { Logger } from "../sync-client/src/tracing/logger"; import type { ServerConfig } from "../sync-client/src/services/server-config"; import { mergeFileManifests } from "../sync-client/src/sync-operations/file-manifest"; import { assertManifest, pathDecision } from "./oracles"; @@ -45,7 +43,6 @@ async function fixture( const persistence = new MemoryPersistence({ database: initial }); const restart = (changes: LocalChange[] = []) => { const database = new Database( - new Logger(), persistence.snapshot().database, async (next) => persistence.save({ database: next }), "recovery-test", @@ -85,7 +82,25 @@ async function swapFixture() { ["b.md", bytes("A")], ["nested/c.md", bytes("C")] ]); - return { ...current, next, writes, expected, retained: [] as string[] }; + return { ...current, next, writes, expected }; +} + +async function directoryCycleFixture() { + const current = await fixture({ + a: { path: "a", text: "A" }, + b: { path: "b/deep/child", text: "B" } + }); + const next = structuredClone(current.initial); + next.local = { a: "b", b: "a/deep/child" }; + return { + ...current, + next, + writes: {}, + expected: new Map([ + ["b", bytes("A")], + ["a/deep/child", bytes("B")] + ]) + }; } async function crossDirectoryFixture() { @@ -101,7 +116,7 @@ async function crossDirectoryFixture() { ["dst/a.md", bytes("A")], ["src/keep.md", bytes("untouched sibling")] ]); - return { ...current, next, writes: {}, expected, retained: [] as string[] }; + return { ...current, next, writes: {}, expected }; } async function mergeFixture() { @@ -123,8 +138,7 @@ async function mergeFixture() { "note.md", bytes("local first paragraph\n\nremote last paragraph\n") ] - ]), - retained: [local] + ]) }; } @@ -145,8 +159,7 @@ async function deletionFixture() { }) } }, - expected: new Map([["a.bin", bytes("remote binary\0")]]), - retained: ["private binary edit\0", "unsent deleted bytes"] + expected: new Map([["a.bin", bytes("remote binary\0")]]) }; } @@ -175,19 +188,19 @@ async function permanentDestinationFixture() { expected: new Map([ ["Recovered a.md", bytes("REMOTE")], ["good.md", bytes("B")] - ]), - retained: ["A"] + ]) }; } for (const [name, create] of [ ["swap and create", swapFixture], - ["cross-directory move with surviving sibling", crossDirectoryFixture], - ["merge during recovery", mergeFixture], - ["binary replacement, deletion and retained bytes", deletionFixture], - ["permanent destination replan", permanentDestinationFixture] + ["file/directory cycle", directoryCycleFixture], + ["cross-directory move", crossDirectoryFixture], + ["content merge", mergeFixture], + ["binary replacement and deletion", deletionFixture], + ["permanent destination fallback", permanentDestinationFixture] ] as const) { - test(`${name}: interrupt every apply and replay boundary, then power-cycle again`, async (t) => { + test(`${name}: interrupted operations leave metadata usable by ordinary scans`, async (t) => { const baseline = await create(); const labels: string[] = []; const trace = (label: string) => { @@ -200,172 +213,90 @@ for (const [name, create] of [ baseline.persistence.boundary = trace; await baseline.restart().files.apply(baseline.next, baseline.writes); assert.deepEqual(baseline.disk.userFiles(), baseline.expected); - assert(labels.some((label) => label.startsWith("visible:rename:"))); - assert(labels.includes("durable:save")); - let replayCrashes = 0; - for (const powerLoss of [false, true]) { - for (let index = 0; index < labels.length; index++) { + assert.deepEqual( + baseline.persistence.snapshot().database!.local, + baseline.expectedLocal ?? baseline.next.local + ); + assert(labels.length > 0); + for (const powerLoss of [false, true]) + for (let cut = 0; cut < labels.length; cut++) { await t.test( - `${powerLoss ? "power" : "process"} crash #${index}: ${labels[index]}`, + `${powerLoss ? "power" : "process"} interruption at ${labels[cut]}`, async () => { const current = await create(); - const clearFaults = () => { - current.disk.boundary = current.permanentFault; - current.persistence.boundary = () => {}; - }; - const interrupt = async ( - action: () => Promise, - at: number, - power: boolean - ) => { - let position = 0, - fired = false; - const fault = (label: string) => { - if (label.startsWith("read:")) return; - if (position++ === at) { - fired = true; - current.disk.crash(power); - throw new Error("Injected interruption"); - } - }; - current.disk.boundary = (label) => { - current.permanentFault(label); - fault(label); - }; - current.persistence.boundary = fault; - try { - await assert.rejects( - action(), - /Injected interruption/ - ); - assert(fired, "Fault index did not occur"); - } finally { - clearFaults(); + let seen = 0; + let fired = false; + const interrupt = (label: string) => { + if (label.startsWith("read:")) return; + if (seen++ === cut) { + fired = true; + current.disk.crash(powerLoss); + throw new Error("interrupted"); } }; - const verify = ( - runtime: ReturnType - ) => { - assert.equal( - runtime.database.state.application, - undefined - ); - assert.deepEqual( - runtime.database.state.local, - current.expectedLocal ?? current.next.local - ); - assert.deepEqual( - current.disk.userFiles(), - current.expected, - "Exact files must survive interrupted replay" - ); - const retained = current.disk - .image() - .visible.filter( - ([path, entry]) => - entry && - path.startsWith( - ".vault-link-sync/recovery/" - ) && - path.endsWith(".content") - ) - .map(([, entry]) => - Buffer.from(entry!.content).toString() - ) - .sort(); - assert.deepEqual( - retained, - [...current.retained].sort(), - "Recovery archives lost or duplicated displaced bytes" - ); - assert( - !current.disk - .image() - .visible.some( - ([path, entry]) => - entry && - path.startsWith( - ".vault-link-sync/transactions/" - ) - ), - "Completed recovery left transaction artifacts" - ); - }; - const finish = async (resubmit: boolean) => { - let runtime = current.restart(); - await runtime.files.recover(); - if (resubmit) - await runtime.files.apply( - current.next, - current.writes - ); - await runtime.files.recover(); - verify(runtime); - current.disk.crash(true); - runtime = current.restart(); - await runtime.files.recover(); - verify(runtime); - }; - await interrupt( - () => - current - .restart() - .files.apply(current.next, current.writes), - index, - powerLoss - ); - const saved = current.persistence.snapshot(); - const diskImage = current.disk.image(); - const intentWasNotSaved = isDeepStrictEqual( - saved.database, - current.initial - ); - // Trace replay from the actual interrupted state, including - // recovery-only branches such as already-visible installs. - const replayLabels: string[] = []; - const traceReplay = (label: string) => { - if (!label.startsWith("read:")) - replayLabels.push(label); - }; current.disk.boundary = (label) => { current.permanentFault(label); - traceReplay(label); + interrupt(label); }; - current.persistence.boundary = traceReplay; - await current.restart().files.recover(); - clearFaults(); - await finish(intentWasNotSaved); - for (const secondPowerLoss of [false, true]) { - for ( - let replay = 0; - replay < replayLabels.length; - replay++ - ) { - current.disk.crash(); - current.disk.restore(diskImage); - await current.persistence.save(saved); - try { - await interrupt( - () => current.restart().files.recover(), - replay, - secondPowerLoss - ); - await finish(false); - replayCrashes++; - } catch (error) { - throw new Error( - `Second ${secondPowerLoss ? "power" : "process"} crash #${replay}: ${replayLabels[replay]}`, - { cause: error } - ); - } - } + current.persistence.boundary = interrupt; + await assert.rejects( + current + .restart() + .files.apply(current.next, current.writes), + /interrupted/ + ); + assert(fired); + current.disk.boundary = () => {}; + current.persistence.boundary = () => {}; + // The user may edit after the interruption, including a partial + // write or a file whose metadata was never saved. + await current.disk.userWrite( + "after-restart.md", + bytes("user edit") + ); + const runtime = current.restart(); + const observed = current.disk.userFiles(); + await scanChanges(runtime, []); + assert.deepEqual( + current.disk.userFiles(), + observed, + "A scan must not replay old filesystem intentions" + ); + assertManifest(runtime.database.state.local); + assert.deepEqual( + new Set( + Object.values(runtime.database.state.local) + ), + new Set(observed.keys()) + ); + for (const [id, path] of Object.entries( + runtime.database.state.local + )) { + assert( + runtime.database.state.documents[id] + .materialized + ); + assert.equal( + runtime.database.state.documents[id] + .observedHash, + (await runtime.files.snapshot(path))!.hash + ); } + assert( + (await runtime.files.listFilesRecursively()).every( + (path) => !path.startsWith(".vault-link-sync/") + ) + ); + const state = structuredClone(runtime.database.state); + await scanChanges(current.restart(), []); + assert.deepEqual( + current.persistence.snapshot().database, + state, + "Second startup must be stable" + ); } ); } - } - assert(replayCrashes > 0, "No interrupted recovery was exercised"); - t.diagnostic(`${replayCrashes} second crashes during journal replay`); }); } @@ -647,49 +578,3 @@ test("normalization-sensitive disk does not lose a distinct decomposed filename" "Distinct Unicode file vanished from the scan" ); }); - -test("notified editor move during a remote swap keeps the moved document's UUID", async () => { - const current = await fixture(); - const changes: LocalChange[] = []; - const runtime = current.restart(changes); - let fired = false; - current.persistence.boundary = async (label) => { - if ( - !fired && - label === "durable:save" && - current.persistence.snapshot().database?.application - ) { - fired = true; - await current.disk.userRename("b.md", "editor.md"); - changes.push({ - type: "move", - oldPath: "b.md", - relativePath: "editor.md" - }); - } - }; - const next = structuredClone(runtime.database.state); - next.local = { a: "b.md", b: "a.md" }; - await runtime.files.apply(next); - assert(fired); - await scanLocalFiles({ - next: structuredClone(runtime.database.state), - changes, - files: runtime.files, - guard: () => {}, - ignored: () => false, - oversized: () => false, - commit: (next) => runtime.database.commit(next) - }); - assert.equal( - runtime.database.state.local.b, - "editor.md", - "Editor move was attributed to the wrong document" - ); - assert.equal( - Buffer.from( - current.disk.userFiles().get(runtime.database.state.local.b)! - ).toString(), - "B" - ); -}); diff --git a/frontend/test-support/restart-reconciliation.test.ts b/frontend/test-support/restart-reconciliation.test.ts new file mode 100644 index 00000000..897b3627 --- /dev/null +++ b/frontend/test-support/restart-reconciliation.test.ts @@ -0,0 +1,231 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { fixture, bytes, randomUUID } from "./sync-server-fixture"; +import { assertCanonical } from "./canonical"; +import { SyncClient } from "../sync-client/src/sync-client"; +import { MemoryDisk, MemoryPersistence } from "./storage"; + +for (const powerLoss of [false, true]) + for (const operation of ["create", "replace", "move", "metadata"] as const) + test( + `${operation}: a ${powerLoss ? "power" : "process"} interruption and later user edits converge without replay`, + { timeout: 20_000 }, + async () => { + const f = await fixture(); + let peer: SyncClient | undefined; + try { + const id = randomUUID(); + const head = await f.put( + `/documents/${id}`, + f.content("BASE\n") + ); + const manifest = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "note.md" } + }); + if (operation !== "create") { + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + if (operation === "move") + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: manifest.fileManifestId, + entries: { [id]: "moved.md" } + }); + else + await f.put( + `/documents/${id}`, + f.content("REMOTE\nBASE\n", head.vaultUpdateId) + ); + } + let fired = false; + let wrote = false; + const interrupt = () => { + fired = true; + f.disk.crash(powerLoss); + throw new Error("injected interruption"); + }; + f.disk.boundary = (label) => { + if (fired) return; + if (label === "durable:write:note.md") wrote = true; + if ( + operation === "move" && + label === "visible:rename:note.md->moved.md" + ) + interrupt(); + if ( + (operation === "create" || + operation === "replace") && + label === "visible:write:note.md" + ) + interrupt(); + }; + f.store.boundary = (label) => { + if ( + !fired && + operation === "metadata" && + wrote && + label === "before:save" + ) + interrupt(); + }; + await assert.rejects( + operation === "create" + ? f.client.start() + : f.client.setSetting("isSyncEnabled", true), + /injected interruption/ + ); + assert( + fired, + "The intended filesystem/save boundary must be reached" + ); + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + // Model torn content and further independent edits. Neither is + // recoverable from an old filesystem intention, nor should it be. + await f.disk.userWrite( + operation === "move" ? "moved.md" : "note.md", + bytes("partial\nUSER EDIT\n") + ); + await f.disk.userWrite( + "after.md", + bytes("AFTER INTERRUPTION\n") + ); + await f.reopenClient(); + await f.client.start(); + const inspectable = { + files: () => f.disk.userFiles(), + database: () => f.store.snapshot().database + }; + const peerDisk = new MemoryDisk(); + const peerStore = new MemoryPersistence({ + settings: f.store.snapshot().settings + }); + peer = await SyncClient.create({ + fs: peerDisk, + persistence: peerStore + }); + await peer.start(); + const clients = [ + inspectable, + { + files: () => peerDisk.userFiles(), + database: () => peerStore.snapshot().database + } + ]; + await assertCanonical( + clients, + f.url, + "test-token-change-me" + ); + assert.deepEqual( + f.disk.userFiles().get("after.md"), + bytes("AFTER INTERRUPTION\n") + ); + assert( + (await f.disk.listFilesRecursively()).every( + (path) => !path.startsWith(".vault-link-sync/") + ) + ); + const files = f.disk.userFiles(); + await f.reopenClient(); + await f.client.start(); + await assertCanonical( + clients, + f.url, + "test-token-change-me" + ); + assert.deepEqual( + f.disk.userFiles(), + files, + "A second restart must not replay old writes" + ); + } finally { + f.disk.boundary = () => {}; + f.store.boundary = () => {}; + await peer?.destroy(); + await f.dispose(); + } + } + ); + +for (const operation of ["create", "replace", "move"] as const) + test( + `${operation}: metadata can survive while completed user-file writes are lost`, + { timeout: 20_000 }, + async () => { + const f = await fixture(); + let peer: SyncClient | undefined; + try { + const id = randomUUID(); + const head = await f.put( + `/documents/${id}`, + f.content("BASE\n") + ); + const manifest = await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: 0, + entries: { [id]: "note.md" } + }); + if (operation !== "create") { + await f.client.start(); + await f.client.setSetting("isSyncEnabled", false); + if (operation === "replace") + await f.put( + `/documents/${id}`, + f.content("REMOTE\nBASE\n", head.vaultUpdateId) + ); + else + await f.put("/file-manifest", { + requestId: randomUUID(), + parentFileManifestId: manifest.fileManifestId, + entries: { [id]: "moved.md" } + }); + } + const oldDisk = f.disk.image(); + if (operation === "create") await f.client.start(); + else await f.client.setSetting("isSyncEnabled", true); + await f.client.destroy(); + // Metadata is newer than disk: completed filesystem calls have no + // flush contract. This is not covered by failing a call mid-flight. + f.disk.restore(oldDisk); + await f.disk.userWrite( + "after.md", + bytes("USER AFTER POWER LOSS\n") + ); + await f.reopenClient(); + await f.client.start(); + const peerDisk = new MemoryDisk(); + const peerStore = new MemoryPersistence({ + settings: f.store.snapshot().settings + }); + peer = await SyncClient.create({ + fs: peerDisk, + persistence: peerStore + }); + await peer.start(); + await assertCanonical( + [ + { + files: () => f.disk.userFiles(), + database: () => f.store.snapshot().database + }, + { + files: () => peerDisk.userFiles(), + database: () => peerStore.snapshot().database + } + ], + f.url, + "test-token-change-me" + ); + assert.deepEqual( + f.disk.userFiles().get("after.md"), + bytes("USER AFTER POWER LOSS\n") + ); + } finally { + await peer?.destroy(); + await f.dispose(); + } + } + ); diff --git a/frontend/test-support/storage.ts b/frontend/test-support/storage.ts index 816dc122..09e339a3 100644 --- a/frontend/test-support/storage.ts +++ b/frontend/test-support/storage.ts @@ -216,13 +216,6 @@ export class MemoryDisk implements FileSystemOperations { }); } - public async flushPaths(paths: readonly string[]): Promise { - const affected = new Set(paths); - for (const path of paths) - for (let p = parent(path); p; p = parent(p)) affected.add(p); - await this.change(`flush:${paths.join(",")}`, [...affected], () => {}); - } - public async delete(path: string): Promise { this.check(path); assert(path !== "", "Cannot remove vault root"); diff --git a/frontend/test-support/sync-audit.test.ts b/frontend/test-support/sync-audit.test.ts index ae842a83..d3c49e0c 100644 --- a/frontend/test-support/sync-audit.test.ts +++ b/frontend/test-support/sync-audit.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { SUPPORTED_API_VERSION } from "../sync-client/src/consts"; import { PermanentSyncError } from "../sync-client/src/errors/errors"; import { test } from "node:test"; import type { Database } from "../sync-client/src/persistence/database"; @@ -221,52 +222,6 @@ test("local undo of accepted-but-rejected manifest must survive replay", async ( assert.equal(f.database.state.local.a, "a.md"); }); -test("disabled startup should recover an active journal before returning", async () => { - const disk = new MemoryDisk(); - const key = JSON.stringify(["http://offline.test", "test"]); - const state = emptyState(key); - state.initialized = true; - state.local = { a: "a.md" }; - state.documents = { a: { materialized: true } }; - const next = structuredClone(state); - next.local = { a: "b.md" }; - const prefix = ".vault-link-sync/transactions/active/a"; - await disk.userWrite(prefix + ".source", bytes("ONLY COPY")); - state.application = { - id: "active", - extensions: ["md"], - next, - steps: [ - { - documentId: "a", - from: "a.md", - to: "b.md", - staged: prefix + ".source", - output: prefix + ".output", - phase: "staged" - } - ] - }; - const store = new MemoryPersistence({ - settings: { - remoteUri: "http://offline.test", - vaultName: "test", - isSyncEnabled: false - }, - database: state - }); - const client = await SyncClient.create({ fs: disk, persistence: store }); - try { - await client.start(); - assert.deepEqual( - (await disk.readSnapshot("b.md"))?.content, - bytes("ONLY COPY") - ); - } finally { - await client.destroy(); - } -}); - test("a settings update resumes the enabled engine", async () => { const f = await settingsLifecycleFixture(); try { @@ -381,7 +336,7 @@ test("silent open WebSocket must eventually trigger catchup or reconnect", async const path = String(input); if (path.endsWith("/ping")) return Response.json({ - supportedApiVersion: 5, + supportedApiVersion: SUPPORTED_API_VERSION, isAuthenticated: true, mergeableFileExtensions: ["md"], serverVersion: "test" @@ -436,7 +391,6 @@ test("permanent destination failure replans the affected file and finishes the b next.fileManifest = { fileManifestId: 2, entries: { ...next.local } }; await f.files.apply(next); await f.settings.setSettings({ ignorePatterns: ["unsupported/**"] }); - await f.files.recover(); assert.deepEqual( [...f.disk.userFiles().values()] .map((b) => Buffer.from(b).toString()) @@ -469,63 +423,85 @@ test("conflict allocation terminates when a truncated extension ends in a space" assert.equal(result.status, 0, String(result.error ?? result.stderr)); }); -test("paged catchup folds remote edits and reversions before touching local bytes", async () => { - const f = await fixture( - { "a.md": "base LOCAL" }, - { getDocumentVersionContent: async () => bytes("base") } - ); - f.database.state.documents.a.base = { - ...head("a", 1, "base"), - hash: (await toStoredSnapshot({ content: bytes("base") })).hash - }; - f.database.state.remoteHeads.a = head("a", 1, "base"); - f.database.state.lastSeenUpdateId = 1; - const first = { - headEventId: 3, - endEventId: 2, - events: [ - { - eventId: 2, - requestId: "remote-1", - type: "content" as const, - document: head("a", 2, "base REMOTE") +for (const interrupted of [false, true]) { + test(`paged catchup folds edits and reversions${interrupted ? " again after an interrupted fetch" : ""}`, async () => { + const first = { + headEventId: 3, + endEventId: 2, + events: [ + { + eventId: 2, + requestId: "remote-1", + type: "content" as const, + document: head("a", 2, "base REMOTE") + } + ] + }; + const last = { + headEventId: 3, + endEventId: 3, + events: [ + { + eventId: 3, + requestId: "remote-2", + type: "content" as const, + document: head("a", 3, "base") + } + ] + }; + let fail = interrupted; + let f: Awaited>; + const fetchedAfter: number[] = []; + const service: Partial = { + getDocumentVersionContent: async () => bytes("base"), + events: async (after) => { + fetchedAfter.push(after); + if (after === 1) return first; + assert.equal(after, 2); + assert.equal(f.database.state.lastSeenUpdateId, 1); + assert.deepEqual( + f.persistence.snapshot(), + saved, + "partial pages must not save metadata" + ); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); + if (fail) { + fail = false; + throw new Error("interrupted page fetch"); + } + return last; } - ] - }; - await f.internals.incorporateEventBatch(first); - assert.equal( - f.database.state.lastSeenUpdateId, - 1, - "partial history must not become an incorporated base" - ); - assert.deepEqual( - (await f.disk.readSnapshot("a.md"))?.content, - bytes("base LOCAL") - ); - assert.equal( - await f.internals.preparePush(), - false, - "do not submit against a partially replayed history" - ); - const last = { - headEventId: 3, - endEventId: 3, - events: [ - { - eventId: 3, - requestId: "remote-2", - type: "content" as const, - document: head("a", 3, "base") - } - ] - }; - await f.internals.incorporateEventBatch(last); - assert.equal(f.database.state.lastSeenUpdateId, 3); - assert.deepEqual( - (await f.disk.readSnapshot("a.md"))?.content, - bytes("base LOCAL") - ); -}); + }; + f = await fixture({ "a.md": "base LOCAL" }, service); + f.database.state.documents.a.base = { + ...head("a", 1, "base"), + hash: (await toStoredSnapshot({ content: bytes("base") })).hash + }; + f.database.state.remoteHeads.a = head("a", 1, "base"); + f.database.state.lastSeenUpdateId = 1; + await f.database.save(); + const saved = f.persistence.snapshot(); + const catchUp = async () => + f.internals.incorporateEventBatch( + await service.events!(f.database.state.lastSeenUpdateId) + ); + if (interrupted) { + await assert.rejects(catchUp(), /interrupted page fetch/); + f.disk.crash(true); + f = await fixture({}, service, f); + } + await catchUp(); + assert.deepEqual(fetchedAfter, interrupted ? [1, 2, 1, 2] : [1, 2]); + assert.equal(f.database.state.lastSeenUpdateId, 3); + assert.deepEqual( + (await f.disk.readSnapshot("a.md"))?.content, + bytes("base LOCAL") + ); + }); +} test("an ignored occupant arriving after the scan keeps its path and privacy", async () => { let f: Awaited>; @@ -580,6 +556,10 @@ test("content notifications during every snapshot do not starve a complete names for (const boundary of ["before:save", "durable:save"]) { test(`receipt incorporation after paged replay survives ${boundary} and process replacement`, async () => { const service: Partial = { + events: async (after) => { + assert.equal(after, 2); + return page; + }, putFileContent: async () => { throw new PermanentSyncError("HTTP 413"); }, @@ -604,18 +584,18 @@ for (const boundary of ["before:save", "durable:save"]) { const { requestId } = f.database.state.pending!.request; await f.internals.finishPending(); await f.disk.userWrite("a.md", bytes("base")); - await f.internals.incorporateEventBatch({ + const first = { headEventId: 3, endEventId: 2, events: [ { eventId: 2, requestId, - type: "content", + type: "content" as const, document: head("a", 2, "base A") } ] - }); + }; const page = { headEventId: 3, endEventId: 3, @@ -632,16 +612,14 @@ for (const boundary of ["before:save", "durable:save"]) { if (label === boundary) throw new Error("injected crash"); }; await assert.rejects( - f.internals.incorporateEventBatch(page), + f.internals.incorporateEventBatch(first), /injected crash/ ); f.persistence.boundary = () => {}; f.disk.crash(true); f = await fixture({}, service, f); - await f.files.recover(); - assert.equal(f.database.state.eventReplay?.after, 2); - await f.internals.incorporateEventBatch(page); - assert.equal(f.database.state.eventReplay, undefined); + assert.equal(f.database.state.lastSeenUpdateId, 1); + await f.internals.incorporateEventBatch(first); assert.equal(f.database.state.lastSeenUpdateId, 3); assert.deepEqual( (await f.disk.readSnapshot("a.md"))?.content, @@ -676,7 +654,7 @@ test("initial vault settings bind the engine and survive restart", async () => { fetch: async (input: RequestInfo | URL) => { if (String(input).endsWith("/ping")) return Response.json({ - supportedApiVersion: 5, + supportedApiVersion: SUPPORTED_API_VERSION, isAuthenticated: true, mergeableFileExtensions: ["md"], serverVersion: "test" @@ -804,7 +782,7 @@ async function settingsLifecycleFixture() { const path = String(input); if (path.endsWith("/ping")) return Response.json({ - supportedApiVersion: 5, + supportedApiVersion: SUPPORTED_API_VERSION, isAuthenticated: true, mergeableFileExtensions: ["md"], serverVersion: "test" diff --git a/frontend/test-support/sync-fixture.ts b/frontend/test-support/sync-fixture.ts index da54142a..30bf6a85 100644 --- a/frontend/test-support/sync-fixture.ts +++ b/frontend/test-support/sync-fixture.ts @@ -59,7 +59,6 @@ export async function fixture( recovered?.persistence ?? new MemoryPersistence({ database: initial }); const logger = new Logger(); const database = new Database( - logger, initial, async (next) => persistence.save({ database: next }), "test", -- 2.54.0 From 82929d7479e5b3a2375b2158723396aa174a5420 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 22 Sep 2026 08:00:25 +0100 Subject: [PATCH 47/48] simplify 2 --- docs/architecture/sync-algorithm.md | 5 +- frontend/local-client-cli/README.md | 8 +- frontend/local-client-cli/src/args.ts | 26 +- frontend/local-client-cli/src/cli.ts | 27 +- .../obsidian-plugin/src/vault-link-plugin.ts | 3 +- frontend/sync-client/README.md | 4 +- frontend/sync-client/src/consts.ts | 2 +- .../src/persistence/settings.test.ts | 30 ++ .../sync-client/src/persistence/settings.ts | 24 ++ .../src/services/fetch-controller.test.ts | 177 ---------- .../src/services/fetch-controller.ts | 149 -------- .../src/services/sync-service.test.ts | 146 ++++++++ .../sync-client/src/services/sync-service.ts | 211 ++++++------ .../src/services/types/WebSocketHandshake.ts | 2 +- .../services/types/WebSocketServerMessage.ts | 3 +- .../src/services/websocket-manager.test.ts | 26 +- .../src/services/websocket-manager.ts | 9 +- frontend/sync-client/src/sync-client.ts | 65 ++-- .../src/sync-operations/syncer.test.ts | 34 +- .../sync-client/src/sync-operations/syncer.ts | 53 ++- .../src/utils/data-structures/locks.test.ts | 322 +++--------------- .../src/utils/data-structures/locks.ts | 169 +-------- .../debugging/slow-web-socket-factory.ts | 29 +- frontend/test-support/protocol-audit.test.ts | 6 +- frontend/test-support/protocol.test.ts | 42 ++- frontend/test-support/sync-fixture.ts | 7 +- sync-server/README.md | 12 +- sync-server/src/app_state/websocket/models.rs | 7 +- sync-server/src/consts.rs | 2 +- sync-server/src/server/endpoints/websocket.rs | 21 +- 30 files changed, 499 insertions(+), 1122 deletions(-) delete mode 100644 frontend/sync-client/src/services/fetch-controller.test.ts delete mode 100644 frontend/sync-client/src/services/fetch-controller.ts create mode 100644 frontend/sync-client/src/services/sync-service.test.ts diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index 229e9879..4f3dc8bb 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -76,8 +76,9 @@ old base are preserved as separate local documents, with conflict names if neede Interrupted recovery saves are replayable. Backups must be restored with the server stopped. -WebSocket messages wake HTTP replay; missed notifications are recovered by replay -and transport liveness checks. Authentication and permanent errors remain visible; +WebSocket `vaultChanged` hints wake HTTP replay. The server periodically compares +its history checkpoint to detect missed broadcasts; transport liveness checks +recover disconnected clients. Authentication and permanent errors remain visible; transient errors retry according to client settings. Observer callback failures are isolated from the background sync loop. diff --git a/frontend/local-client-cli/README.md b/frontend/local-client-cli/README.md index e91322f9..f0cc6388 100644 --- a/frontend/local-client-cli/README.md +++ b/frontend/local-client-cli/README.md @@ -62,7 +62,6 @@ vaultlink \ | `--ignore-pattern ` | - | Glob pattern to ignore (repeatable) | | `--websocket-retry-interval-ms ` | `3500` | WebSocket reconnection interval | | `--log-level ` | `INFO` | Log level: DEBUG, INFO, WARNING, ERROR | -| `--line-endings ` | `auto` | Line ending style: auto, lf, crlf | | `-q, --quiet` | - | Suppress startup banner for non-interactive use | | `-h, --help` | - | Show help | | `-V, --version` | - | Show version | @@ -96,12 +95,7 @@ vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ --log-level DEBUG --quiet ``` -Force LF line endings (useful for cross-platform vaults): - -```bash -vaultlink -l ./vault -r wss://sync.example.com -t token123 -v default \ - --line-endings lf -``` +File bytes, including line endings, are preserved on every platform. ## Docker Deployment diff --git a/frontend/local-client-cli/src/args.ts b/frontend/local-client-cli/src/args.ts index 34d839b1..cb6ce421 100644 --- a/frontend/local-client-cli/src/args.ts +++ b/frontend/local-client-cli/src/args.ts @@ -2,9 +2,6 @@ import { Command, Option } from "commander"; import packageJson from "../package.json"; import { LogLevel } from "sync-client"; -export const LINE_ENDING_MODES = ["auto", "lf", "crlf"] as const; -export type LineEndingMode = (typeof LINE_ENDING_MODES)[number]; - interface CliArgs { remoteUri: string; token: string; @@ -17,7 +14,6 @@ interface CliArgs { health?: string; enableTelemetry?: boolean; quiet: boolean; - lineEndings: LineEndingMode; } const VALID_PROTOCOLS = ["http://", "https://", "ws://", "wss://"]; @@ -131,15 +127,6 @@ export function parseArgs(argv: string[]): CliArgs { "[OPTIONAL] Suppress startup banner for non-interactive use" ).env("VAULTLINK_QUIET") ) - .addOption( - new Option( - "--line-endings ", - "[OPTIONAL] Line ending style: auto (platform default), lf, crlf" - ) - .default("auto") - .choices([...LINE_ENDING_MODES]) - .env("VAULTLINK_LINE_ENDINGS") - ) .addHelpText( "after", ` @@ -173,7 +160,6 @@ Environment variables: const health = opts.health as string | undefined; const enableTelemetry = opts.enableTelemetry as boolean | undefined; const quiet = (opts.quiet as boolean | undefined) ?? false; - const lineEndingsStr = (opts.lineEndings as string | undefined) ?? "auto"; /* eslint-enable @typescript-eslint/no-unsafe-type-assertion */ const requiredLocalPath = requireOption(localPath, "localPath"); @@ -203,15 +189,6 @@ Environment variables: } const logLevel = logLevelUpper; - const isLineEndingMode = (value: string): value is LineEndingMode => - (LINE_ENDING_MODES as readonly string[]).includes(value); - if (!isLineEndingMode(lineEndingsStr)) { - throw new Error( - `Invalid line endings mode '${lineEndingsStr}'. Valid values are: ${LINE_ENDING_MODES.join(", ")}` - ); - } - const lineEndings = lineEndingsStr; - return { localPath: requiredLocalPath, remoteUri: requiredRemoteUri, @@ -223,7 +200,6 @@ Environment variables: logLevel, health, enableTelemetry, - quiet, - lineEndings + quiet }; } diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index e6aabc68..0a40d9fb 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -1,8 +1,7 @@ import * as path from "path"; import * as fs from "fs/promises"; import * as fsSync from "fs"; -import type { NetworkConnectionStatus , - Logger} from "sync-client"; +import type { NetworkConnectionStatus, Logger } from "sync-client"; import { SyncClient, DEFAULT_SETTINGS, @@ -11,7 +10,7 @@ import { type SyncSettings, type StoredDatabase } from "sync-client"; -import { parseArgs, type LineEndingMode } from "./args"; +import { parseArgs } from "./args"; import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem"; import { FileWatcher } from "./file-watcher"; import { formatLogLine } from "./logger-formatter"; @@ -41,7 +40,6 @@ const LOG_LEVEL_ORDER = { function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { return (logLine: LogLine): void => { if (LOG_LEVEL_ORDER[logLine.level] >= LOG_LEVEL_ORDER[minLevel]) { - console.log(formatLogLine(logLine)); } }; @@ -50,17 +48,6 @@ function createLogHandler(minLevel: LogLevel): (logLine: LogLine) => void { const HEALTH_CHECK_INTERVAL_MS = 30 * 1000; const PROGRESS_LOG_INTERVAL_MS = 2000; -function resolveLineEndings(mode: LineEndingMode): string { - switch (mode) { - case "lf": - return "\n"; - case "crlf": - return "\r\n"; - case "auto": - return process.platform === "win32" ? "\r\n" : "\n"; - } -} - async function main(): Promise { const args = parseArgs(process.argv); const absolutePath = path.resolve(args.localPath); @@ -96,12 +83,6 @@ async function main(): Promise { emitBoot(LogLevel.INFO, `Local path: ${absolutePath}`); emitBoot(LogLevel.INFO, `Remote URI: ${args.remoteUri}`); emitBoot(LogLevel.INFO, `Vault name: ${args.vaultName}`); - if (args.lineEndings !== "auto") { - emitBoot( - LogLevel.INFO, - `Line endings: ${args.lineEndings.toUpperCase()}` - ); - } } const dataDir = path.join(absolutePath, VAULTLINK_DIR); @@ -158,8 +139,7 @@ async function main(): Promise { JSON.stringify(persistedDatabase, null, 2) ); } - }, - nativeLineEndings: resolveLineEndings(args.lineEndings) + } }); if (args.health !== undefined) { @@ -279,7 +259,6 @@ async function main(): Promise { } main().catch((error: unknown) => { - console.error( `Unexpected error: ${error instanceof Error ? error.message : String(error)}` ); diff --git a/frontend/obsidian-plugin/src/vault-link-plugin.ts b/frontend/obsidian-plugin/src/vault-link-plugin.ts index e222796b..2c456b02 100644 --- a/frontend/obsidian-plugin/src/vault-link-plugin.ts +++ b/frontend/obsidian-plugin/src/vault-link-plugin.ts @@ -5,7 +5,7 @@ import type { TAbstractFile, WorkspaceLeaf } from "obsidian"; -import { Notice, Platform, Plugin, TFile } from "obsidian"; +import { Notice, Plugin, TFile } from "obsidian"; import "../manifest.json"; import { HistoryView } from "./views/history/history-view"; import { StatusBar } from "./views/status-bar/status-bar"; @@ -132,7 +132,6 @@ export default class VaultLinkPlugin extends Plugin { load: this.loadData.bind(this), save: this.saveData.bind(this) }, - nativeLineEndings: Platform.isWin ? "\r\n" : "\n", ...(IS_DEBUG_BUILD ? { fetch: debugging.slowFetchFactory(1), diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md index 0a5bc956..c92671db 100644 --- a/frontend/sync-client/README.md +++ b/frontend/sync-client/README.md @@ -1,7 +1,7 @@ # Sync client (API v5) The engine uses one serialized loop per vault and requires Web Crypto (SHA-256). Local notifications and WebSocket -messages wake it; complete disk scans and the durable server event log are the +`vaultChanged` hints wake it; complete disk scans and the durable server event log are the sources of truth. Periodic reconciliation is opt-in: set `syncIntervalMs` to a positive interval to repair missed notifications. Unset or `0` disables polling. Create, delete and move notifications persist logical identity changes before @@ -13,6 +13,8 @@ as content updates. This distinguishes a deliberate delete/recreate from an edit Use `waitUntilFinished()` to await the active attempt; errors leave pending requests saved and are reported to the caller. Transient failures retry; authentication and protocol errors wait for an explicit wake or settings change. +Pausing or destroying the client aborts its HTTP session, including body reads; +resuming starts a fresh session. Connection checks can run while sync is disabled. ## State and policy diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index edb133fb..a649d067 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -2,5 +2,5 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; -export const SUPPORTED_API_VERSION = 4; +export const SUPPORTED_API_VERSION = 5; export const WEBSOCKET_DISCONNECT_TIMEOUT_IN_S = 10; diff --git a/frontend/sync-client/src/persistence/settings.test.ts b/frontend/sync-client/src/persistence/settings.test.ts index b17df261..cff24895 100644 --- a/frontend/sync-client/src/persistence/settings.test.ts +++ b/frontend/sync-client/src/persistence/settings.test.ts @@ -57,3 +57,33 @@ it("owns settings arrays instead of sharing them with callers and listeners", as await settings.setSettings({ maxFileSizeMB: 12 }); assert.deepEqual(settings.getSettings().ignorePatterns, ["updated"]); }); + +it("file eligibility follows successfully saved settings", async () => { + let fail = false; + const settings = new Settings( + new Logger(), + { + ignorePatterns: ["private/**"], + maxFileSizeMB: 1 + }, + async () => { + if (fail) throw new Error("save failed"); + } + ); + assert(settings.isIgnored("private/note.md")); + assert(settings.isIgnored(".vault-link-sync/state.json")); + assert(!settings.isIgnored("public/note.md")); + assert(!settings.isOversized(1024 * 1024)); + assert(settings.isOversized(1024 * 1024 + 1)); + fail = true; + await assert.rejects( + settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 }), + /save failed/ + ); + assert(settings.isIgnored("private/note.md")); + assert(settings.isOversized(1024 * 1024 + 1)); + fail = false; + await settings.setSettings({ ignorePatterns: [], maxFileSizeMB: 2 }); + assert(!settings.isIgnored("private/note.md")); + assert(!settings.isOversized(1024 * 1024 + 1)); +}); diff --git a/frontend/sync-client/src/persistence/settings.ts b/frontend/sync-client/src/persistence/settings.ts index 9cc3bbe3..9a93e0dd 100644 --- a/frontend/sync-client/src/persistence/settings.ts +++ b/frontend/sync-client/src/persistence/settings.ts @@ -1,4 +1,6 @@ import type { Logger } from "../tracing/logger"; +import { globsToRegexes } from "../utils/globs-to-regexes"; +import { isInternalPath } from "../utils/portable-path"; import { Lock } from "../utils/data-structures/locks"; import { EventListeners } from "../utils/data-structures/event-listeners"; @@ -43,6 +45,7 @@ export class Settings { >(); private settings: SyncSettings; + private ignorePatterns: RegExp[]; private readonly lock: Lock = new Lock(); public constructor( @@ -55,11 +58,27 @@ export class Settings { ...structuredClone(initialState ?? {}) }; + this.ignorePatterns = globsToRegexes( + this.settings.ignorePatterns, + logger + ); + this.logger.debug( `Loaded settings: ${JSON.stringify(redactSettings(this.settings), null, 2)}` ); } + public isIgnored(path: string): boolean { + return ( + isInternalPath(path) || + this.ignorePatterns.some((pattern) => pattern.test(path)) + ); + } + + public isOversized(size: number): boolean { + return size > this.settings.maxFileSizeMB * 1024 * 1024; + } + public getSettings(): SyncSettings { return { ...this.settings, @@ -91,6 +110,11 @@ export class Settings { const next = { ...this.settings, ...update }; await this.saveData(next); this.settings = next; + if (update.ignorePatterns !== undefined) + this.ignorePatterns = globsToRegexes( + next.ignorePatterns, + this.logger + ); return [this.getSettings(), oldSettings] as const; }); diff --git a/frontend/sync-client/src/services/fetch-controller.test.ts b/frontend/sync-client/src/services/fetch-controller.test.ts deleted file mode 100644 index a0a70d80..00000000 --- a/frontend/sync-client/src/services/fetch-controller.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import type { Mock } from "node:test"; -import { describe, it, mock, beforeEach, afterEach } from "node:test"; -import assert from "node:assert"; -import { FetchController } from "./fetch-controller"; -import { Logger } from "../tracing/logger"; -import { SyncResetError } from "../errors/errors"; -import { sleep } from "../utils/sleep"; - -describe("FetchController", () => { - const createMockFetch = ( - shouldSleep: boolean - ): Mock<() => Promise> => - mock.fn(async () => { - if (shouldSleep) { - await sleep(30); - } - return Promise.resolve(new Response("OK", { status: 200 })); - }); - - beforeEach(() => { - mock.timers.enable({ apis: ["setTimeout"] }); - }); - - afterEach(() => { - mock.timers.reset(); - }); - - it("should allow fetch when canFetch is true", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(false); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - const response = await controlledFetch("http://example.com"); - - assert.strictEqual(await response.text(), "OK"); - assert.strictEqual(mockFetch.mock.calls.length, 1); - }); - - it("should block fetch until canFetch becomes true", async () => { - const logger = new Logger(); - const controller = new FetchController(false, logger); - const mockFetch = createMockFetch(true); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - const fetchPromise = controlledFetch("http://example.com"); - assert.strictEqual(mockFetch.mock.calls.length, 0); - - controller.canFetch = true; - await Promise.resolve(); - mock.timers.tick(30); - - const response = await fetchPromise; - assert.strictEqual(await response.text(), "OK"); - assert.strictEqual(mockFetch.mock.calls.length, 1); - }); - - it("should reject during reset", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(true); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - const firstRequest = controlledFetch("http://example.com"); - assert.strictEqual(mockFetch.mock.calls.length, 1); - - controller.startReset(); - - const secondRequest = controlledFetch("http://example.com"); - - await assert.rejects( - firstRequest, - (error: unknown) => error instanceof SyncResetError - ); - await assert.rejects( - secondRequest, - (error: unknown) => error instanceof SyncResetError - ); - assert.strictEqual(mockFetch.mock.calls.length, 1); - }); - - it("should allow fetch after reset finishes", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(false); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - controller.startReset(); - controller.finishReset(); - - const response = await controlledFetch("http://example.com"); - assert.strictEqual(await response.text(), "OK"); - }); - - it("should defer canFetch changes during reset", async () => { - const logger = new Logger(); - const controller = new FetchController(false, logger); - const mockFetch = createMockFetch(true); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - controller.startReset(); - controller.canFetch = true; - - await assert.rejects( - async () => controlledFetch("http://example.com"), - (error: unknown) => error instanceof SyncResetError - ); - - controller.finishReset(); - - const fetchPromise = controlledFetch("http://example.com"); - mock.timers.tick(30); - - const response = await fetchPromise; - assert.strictEqual(await response.text(), "OK"); - }); - - it("should handle different input types", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = createMockFetch(false); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - await controlledFetch("http://example.com"); - await controlledFetch(new URL("http://example.com")); - await controlledFetch( - new Request("http://example.com", { method: "POST" }) - ); - - assert.strictEqual(mockFetch.mock.calls.length, 3); - }); - - it("should handle fetch errors", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - const mockFetch = mock.fn(async () => { - throw new Error("Network error"); - }); - const controlledFetch = controller.getControlledFetchImplementation( - logger, - mockFetch - ); - - await assert.rejects( - async () => controlledFetch("http://example.com"), - (error: unknown) => - error instanceof Error && error.message === "Network error" - ); - }); - - it("should not create unhandled rejection on reset with no waiting fetches", async () => { - const logger = new Logger(); - const controller = new FetchController(true, logger); - - controller.startReset(); - mock.timers.tick(10); - controller.finishReset(); - }); -}); diff --git a/frontend/sync-client/src/services/fetch-controller.ts b/frontend/sync-client/src/services/fetch-controller.ts deleted file mode 100644 index c1bd29c9..00000000 --- a/frontend/sync-client/src/services/fetch-controller.ts +++ /dev/null @@ -1,149 +0,0 @@ -import type { Logger } from "../tracing/logger"; -import { createPromise } from "../utils/create-promise"; -import { SyncResetError } from "../errors/errors"; - -/** - * Offers a resettable fetch implementation that waits until syncing is enabled - * and aborts outstanding requests when a reset is started. - */ -export class FetchController { - private static readonly UNTIL_RESOLUTION = Symbol(); - - private isResetting = false; - - // Promise resolves on the next state change: sync enabled/disabled or reset started/ended - private until: Promise; - private resolveUntil: (result: symbol) => unknown; - private rejectUntil: (reason: unknown) => unknown; - - public constructor( - private _canFetch: boolean, - private readonly logger: Logger - ) { - [this.until, this.resolveUntil, this.rejectUntil] = - createPromise(); - } - - /** - * Whether the fetch implementation can immediately send requests once outside of a reset. - */ - public get canFetch(): boolean { - return this._canFetch; - } - - /** - * Allow or disallow fetching. The changes only take effect if not resetting. - * When called during a reset, its effect is deferred until the reset is finished. - * - * @param canFetch Whether fetching is enabled - */ - public set canFetch(canFetch: boolean) { - this._canFetch = canFetch; - - if (!this.isResetting) { - const previousResolve = this.resolveUntil; - [this.until, this.resolveUntil, this.rejectUntil] = - createPromise(); - previousResolve(FetchController.UNTIL_RESOLUTION); - } - } - - private static getUrlFromInput(input: RequestInfo | URL): string { - if (input instanceof URL) { - return input.href; - } - if (typeof input === "string") { - return input; - } - return input.url; - } - - /** - * Starts a reset, causing all ongoing and future fetches to be rejected - * with a SyncResetError until finishReset is called. - */ - public startReset(): void { - this.isResetting = true; - this.rejectUntil(new SyncResetError()); - // Catch unhandled rejection if no fetches are waiting - this.until.catch(() => { - // Intentionally ignore - this rejection is handled by waiting fetches - }); - } - - /** - * Finishes a reset, allowing fetches to proceed or wait again depending on - * the current sync settings. - */ - public finishReset(): void { - if (!this.isResetting) { - return; - } - - this.isResetting = false; - [this.until, this.resolveUntil, this.rejectUntil] = createPromise(); - } - - /** - * - * |------------------|---------------|-----------------------------------------------------| - * | | Sync enabled | Sync disabled | - * |------------------|-------------- |-----------------------------------------------------| - * | During reset | Rejects with SyncResetError without sending request | - * |------------------|-------------- |-----------------------------------------------------| - * | Outside of reset | Same as fetch | Blocks until sync is enabled and then same as fetch | - * |------------------|---------------|-----------------------------------------------------| - * - * @param logger for errors - * @param fetch to wrap - * @returns a wrapped fetch implementation affected by the FetchController state - */ - public getControlledFetchImplementation( - logger: Logger, - fetch: typeof globalThis.fetch = globalThis.fetch - ): typeof globalThis.fetch { - return async ( - input: RequestInfo | URL, - init?: RequestInit - ): Promise => { - while (!this.canFetch || this.isResetting) { - await this.until; - } - - try { - // https://github.com/jonbern/fetch-retry/blob/8684ef4e688375f623bd76f13add76dbc1d67cfb/index.js#L67C1-L70C21 - const _input = - typeof Request !== "undefined" && input instanceof Request - ? input.clone() - : input; - - const fetchPromise = fetch(_input, init); - - // We only want to catch rejections from `this.until` - let result: symbol | Response | undefined = undefined; - do { - result = await Promise.race([this.until, fetchPromise]); - } while (result === FetchController.UNTIL_RESOLUTION); - - const fetchResult: Response = result as Response; // eslint-disable-line @typescript-eslint/no-unsafe-type-assertion - - if (!fetchResult.ok) { - this.logger.warn( - `Fetch for ${FetchController.getUrlFromInput( - input - )}, got status ${fetchResult.status}` - ); - } - - return fetchResult; - } catch (error) { - logger.warn( - `Fetch for ${FetchController.getUrlFromInput( - input - )}, got error: ${error}` - ); - throw error; - } - }; - } -} diff --git a/frontend/sync-client/src/services/sync-service.test.ts b/frontend/sync-client/src/services/sync-service.test.ts new file mode 100644 index 00000000..d4f72b47 --- /dev/null +++ b/frontend/sync-client/src/services/sync-service.test.ts @@ -0,0 +1,146 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { SyncResetError } from "../errors/errors"; +import { SyncService } from "./sync-service"; + +const settings = () => + new Settings( + new Logger(), + { + remoteUri: "http://test", + vaultName: "test", + requestTimeoutMs: 1000 + }, + async () => {} + ); + +test("paused sync sends no requests, while connection checks still work", async () => { + let requests = 0; + const service = new SyncService("device", settings(), async () => { + requests++; + return Response.json({ headEventId: 0, events: [] }); + }); + await assert.rejects(service.events(0), SyncResetError); + assert.equal(requests, 0); + await service.ping(); + assert.equal(requests, 1); + service.resume(); + assert.equal((await service.events(0)).headEventId, 0); + assert.equal(requests, 2); +}); + +test("pause aborts requests and fences late responses from an injected fetch", async () => { + const response = Promise.withResolvers(); + const signals: AbortSignal[] = []; + const saved: (string | undefined)[] = []; + const service = new SyncService( + "device", + settings(), + async (_, init) => { + signals.push(init!.signal!); + return signals.length === 1 + ? response.promise + : Response.json({ headEventId: 0, events: [] }); + }, + { + get: () => undefined, + save: async (value) => { + saved.push(value); + } + } + ); + service.resume(); + const first = service.events(0); + service.pause(); + await assert.rejects(first, SyncResetError); + assert.equal(signals[0].aborted, true); + service.resume(); + await service.events(0); + assert.equal(signals[1].aborted, false); + response.resolve( + Response.json( + {}, + { headers: { "X-Vault-Link-History": "99:abandoned" } } + ) + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(saved, []); +}); + +for (const binary of [false, true]) { + test(`pause interrupts a stalled ${binary ? "binary" : "JSON"} response body`, async () => { + const headersSaved = Promise.withResolvers(); + let body!: ReadableStreamDefaultController; + const service = new SyncService( + "device", + settings(), + async () => + new Response( + new ReadableStream({ + start(controller) { + body = controller; + } + }), + { headers: { "X-Vault-Link-History": "1:head" } } + ), + { + get: () => undefined, + save: async () => { + headersSaved.resolve(); + } + } + ); + service.resume(); + const request = binary + ? service.getDocumentVersionContent({ + documentId: "a", + vaultUpdateId: 1 + }) + : service.events(0); + await headersSaved.promise; + await new Promise((resolve) => setImmediate(resolve)); + service.pause(); + await assert.rejects(request, SyncResetError); + body.enqueue(new TextEncoder().encode("{}")); + body.close(); + }); +} + +test("pause waits for a checkpoint save already in progress", async () => { + const saving = Promise.withResolvers(); + const release = Promise.withResolvers(); + let settled = false; + let checkpoint: string | undefined; + const service = new SyncService( + "device", + settings(), + async () => + Response.json( + {}, + { + headers: { "X-Vault-Link-History": "1:head" } + } + ), + { + get: () => checkpoint, + save: async (value) => { + saving.resolve(); + await release.promise; + checkpoint = value; + } + } + ); + service.resume(); + const request = service.events(0).finally(() => { + settled = true; + }); + await saving.promise; + service.pause(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(settled, false); + release.resolve(); + await assert.rejects(request, SyncResetError); + assert.equal(checkpoint, "1:head"); +}); diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index 48629d49..cf918151 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -1,11 +1,7 @@ import type { Settings } from "../persistence/settings"; -import type { FetchController } from "./fetch-controller"; -import type { Logger } from "../tracing/logger"; import type { DocumentUpdateResponse } from "./types/DocumentUpdateResponse"; -import type { DocumentVersion } from "./types/DocumentVersion"; import type { DocumentVersionWithoutContent } from "./types/DocumentVersionWithoutContent"; import type { EventBatch } from "./types/EventBatch"; -import type { FileManifest } from "./types/FileManifest"; import type { FileManifestUpdateResponse } from "./types/FileManifestUpdateResponse"; import type { PutFileContent } from "./types/PutFileContent"; import type { PushFileManifest } from "./types/PushFileManifest"; @@ -14,29 +10,47 @@ import type { PingResponse } from "./types/PingResponse"; import { AuthenticationError, PermanentSyncError, - ServerHistoryChangedError + ServerHistoryChangedError, + SyncResetError } from "../errors/errors"; +// Injected fetch implementations may ignore cancellation. Stop waiting anyway, +// and never let their late responses reach checkpoint or engine persistence. +async function abortable( + signal: AbortSignal, + operation: () => Promise +): Promise { + signal.throwIfAborted(); + let abort = (): void => undefined; + try { + return await Promise.race([ + new Promise((_, reject) => { + abort = (): void => { + // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- Only reset errors and timeout errors reach this helper. + reject(signal.reason); + }; + signal.addEventListener("abort", abort, { once: true }); + }), + operation() + ]); + } finally { + signal.removeEventListener("abort", abort); + } +} + export class SyncService { - private readonly client: typeof fetch; - private readonly rawFetch: typeof fetch; + private session = new AbortController(); public constructor( private readonly deviceId: string, - fetchController: FetchController, private readonly settings: Settings, - logger: Logger, - fetchImplementation: typeof fetch = globalThis.fetch, + private readonly fetchImplementation: typeof fetch = globalThis.fetch, private readonly history?: { get: () => string | undefined; save: (checkpoint: string | undefined) => Promise; } ) { - this.rawFetch = async (...args) => fetchImplementation(...args); - this.client = fetchController.getControlledFetchImplementation( - logger, - this.rawFetch - ); + this.pause(); } public get verifiesHistory(): boolean { @@ -46,21 +60,80 @@ export class SyncService { return this.history?.get() !== undefined; } + public resume(): void { + this.session = new AbortController(); + } + + public pause(): void { + this.session.abort(new SyncResetError()); + } + public async resetHistory(): Promise { await this.history?.save(undefined); } + public async ping(): Promise { + return this.request("/ping", undefined, true); + } + + public async vaultSnapshot(): Promise { + return this.request("/vault-snapshot"); + } + + public async events(after: number): Promise { + return this.request(`/events-since?after=${after}`); + } + + public async pushFileManifest( + request: PushFileManifest + ): Promise { + return this.request("/file-manifest", request); + } + + public async putFileContent( + id: string, + request: PutFileContent + ): Promise { + return this.request(`/documents/${id}`, request); + } + + public async metadata(id: string): Promise { + return this.request(`/documents/${id}/metadata`); + } + + public async getDocumentVersionContent({ + documentId, + vaultUpdateId + }: { + documentId: string; + vaultUpdateId: number; + }): Promise { + return this.request( + `/documents/${documentId}/versions/${vaultUpdateId}/content`, + undefined, + false, + async (response) => new Uint8Array(await response.arrayBuffer()) + ); + } + private historyHeaders(): Record { const checkpoint = this.history?.get(); - return checkpoint ? { "X-Vault-Link-History": checkpoint } : {}; + return checkpoint !== undefined && checkpoint !== "" + ? { "X-Vault-Link-History": checkpoint } + : {}; } private async recordHistory(response: Response): Promise { const checkpoint = response.headers.get("x-vault-link-history"); - if (checkpoint && this.history) { + if ( + checkpoint !== null && + checkpoint !== "" && + this.history !== undefined + ) { const previous = this.history.get(); if ( - !previous || + previous === undefined || + previous === "" || Number(checkpoint.split(":")[0]) >= Number(previous.split(":")[0]) ) @@ -75,7 +148,7 @@ export class SyncService { ); } - public getUrl(path: string): string { + private getUrl(path: string): string { const { remoteUri, vaultName } = this.settings.getSettings(); return `${remoteUri.replace(/\/$/u, "")}/vaults/${encodeURIComponent(vaultName)}${path}`; } @@ -83,35 +156,42 @@ export class SyncService { private async request( path: string, body?: unknown, - raw = false + raw = false, + decode: (response: Response) => Promise = async (response) => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- Endpoint methods supply the generated protocol response type. + response.json() as Promise ): Promise { - const response = await (raw ? this.rawFetch : this.client)( - this.getUrl(path), - { + const timeout = AbortSignal.timeout( + this.settings.getSettings().requestTimeoutMs + ); + const signal = raw + ? timeout + : AbortSignal.any([timeout, this.session.signal]); + const response = await abortable(signal, async () => + this.fetchImplementation(this.getUrl(path), { method: body === undefined ? "GET" : "PUT", body: body === undefined ? undefined : JSON.stringify(body), - signal: AbortSignal.timeout( - this.settings.getSettings().requestTimeoutMs - ), + signal, headers: { ...(raw ? {} : this.historyHeaders()), Authorization: `Bearer ${this.settings.getSettings().token}`, "Device-Id": this.deviceId, "Content-Type": "application/json" } - } + }) ); - + signal.throwIfAborted(); this.checkHistory(response); if (!response.ok) { this.throwForErrorResponse( response, - `HTTP ${response.status}: ${await response.text()}` + `HTTP ${response.status}: ${await abortable(signal, async () => response.text())}` ); } + // Finish metadata saves even if the transport is paused meanwhile. if (!raw) await this.recordHistory(response); - return (await response.json()) as T; + return abortable(signal, async () => decode(response)); } private throwForErrorResponse(response: Response, message: string): never { @@ -128,75 +208,4 @@ export class SyncService { } throw new Error(message); } - - public async ping(): Promise { - return this.request("/ping", undefined, true); - } - - public async vaultSnapshot(): Promise { - return this.request("/vault-snapshot"); - } - - public async events(after: number): Promise { - return this.request(`/events-since?after=${after}`); - } - - public async fileManifest(): Promise { - return this.request("/file-manifest"); - } - - public async pushFileManifest( - request: PushFileManifest - ): Promise { - return this.request("/file-manifest", request); - } - - public async putFileContent( - id: string, - request: PutFileContent - ): Promise { - return this.request(`/documents/${id}`, request); - } - - public async get(id: string): Promise { - return this.request(`/documents/${id}`); - } - - public async metadata(id: string): Promise { - return this.request(`/documents/${id}/metadata`); - } - - public async getDocumentVersionContent({ - documentId, - vaultUpdateId - }: { - documentId: string; - vaultUpdateId: number; - }): Promise { - const response = await this.client( - this.getUrl( - `/documents/${documentId}/versions/${vaultUpdateId}/content` - ), - { - signal: AbortSignal.timeout( - this.settings.getSettings().requestTimeoutMs - ), - headers: { - ...this.historyHeaders(), - Authorization: `Bearer ${this.settings.getSettings().token}` - } - } - ); - - this.checkHistory(response); - if (!response.ok) { - this.throwForErrorResponse( - response, - `Cannot fetch version ${vaultUpdateId}: HTTP ${response.status}` - ); - } - - await this.recordHistory(response); - return new Uint8Array(await response.arrayBuffer()); - } } diff --git a/frontend/sync-client/src/services/types/WebSocketHandshake.ts b/frontend/sync-client/src/services/types/WebSocketHandshake.ts index 85c2cf0d..310c0bb8 100644 --- a/frontend/sync-client/src/services/types/WebSocketHandshake.ts +++ b/frontend/sync-client/src/services/types/WebSocketHandshake.ts @@ -1,3 +1,3 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type WebSocketHandshake = { token: string, deviceId: string, lastSeenVaultUpdateId: number | null, }; +export type WebSocketHandshake = { token: string, deviceId: string, }; diff --git a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts index 2a7a7813..c9759e04 100644 --- a/frontend/sync-client/src/services/types/WebSocketServerMessage.ts +++ b/frontend/sync-client/src/services/types/WebSocketServerMessage.ts @@ -1,5 +1,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { CursorPositionFromServer } from "./CursorPositionFromServer"; -import type { EventBatch } from "./EventBatch"; -export type WebSocketServerMessage = { "type": "vaultEvents" } & EventBatch | { "type": "cursorPositions" } & CursorPositionFromServer; +export type WebSocketServerMessage = { "type": "vaultChanged" } | { "type": "cursorPositions" } & CursorPositionFromServer; diff --git a/frontend/sync-client/src/services/websocket-manager.test.ts b/frontend/sync-client/src/services/websocket-manager.test.ts index d5223b6a..645fa627 100644 --- a/frontend/sync-client/src/services/websocket-manager.test.ts +++ b/frontend/sync-client/src/services/websocket-manager.test.ts @@ -136,19 +136,13 @@ describe("WebSocketManager", () => { .webSocket; mockWs.simulateMessage({ - type: "vaultEvents", - headEventId: 0, - events: [] + type: "vaultChanged" }); mockWs.simulateMessage({ - type: "vaultEvents", - headEventId: 0, - events: [] + type: "vaultChanged" }); mockWs.simulateMessage({ - type: "vaultEvents", - headEventId: 0, - events: [] + type: "vaultChanged" }); await new Promise((resolve) => setTimeout(resolve, 100)); @@ -208,8 +202,7 @@ describe("WebSocketManager", () => { manager.sendHandshakeMessage({ type: "handshake", token: "test", - deviceId: "test", - lastSeenVaultUpdateId: null + deviceId: "test" }); }); @@ -317,9 +310,7 @@ describe("WebSocketManager", () => { const mockWs = (manager as unknown as { webSocket: MockWebSocket }) .webSocket; mockWs.simulateMessage({ - type: "vaultEvents", - headEventId: 0, - events: [] + type: "vaultChanged" }); await new Promise((resolve) => setTimeout(resolve, 10)); @@ -465,9 +456,7 @@ for (const closeBehavior of ["throws", "stays open"]) { assert.equal(manager.isWebSocketConnected, false); assert.equal(connected, false); socket.simulateMessage({ - type: "vaultEvents", - headEventId: 0, - events: [] + type: "vaultChanged" }); await manager.waitUntilFinished(); assert.equal(delivered, 0); @@ -499,8 +488,7 @@ it("a handshake send failure retires the connection without escaping the open ca manager.sendHandshakeMessage({ type: "handshake", token: "test", - deviceId: "test", - lastSeenVaultUpdateId: 0 + deviceId: "test" }); }); const statuses: boolean[] = []; diff --git a/frontend/sync-client/src/services/websocket-manager.ts b/frontend/sync-client/src/services/websocket-manager.ts index 3fb80947..8d90fbbd 100644 --- a/frontend/sync-client/src/services/websocket-manager.ts +++ b/frontend/sync-client/src/services/websocket-manager.ts @@ -5,7 +5,6 @@ import type { WebSocketClientMessage } from "./types/WebSocketClientMessage"; import type { CursorPositionFromClient } from "./types/CursorPositionFromClient"; import type { ClientCursors } from "./types/ClientCursors"; import { createPromise } from "../utils/create-promise"; -import type { EventBatch } from "./types/EventBatch"; import { WEBSOCKET_DISCONNECT_TIMEOUT_IN_S } from "../consts"; import { removeFromArray } from "../utils/remove-from-array"; import { EventListeners } from "../utils/data-structures/event-listeners"; @@ -17,7 +16,7 @@ export class WebSocketManager { >(); public readonly onRemoteVaultUpdateReceived = new EventListeners< - (update: EventBatch) => Promise + () => Promise >(); public readonly onRemoteCursorsUpdateReceived = new EventListeners< @@ -303,7 +302,7 @@ export class WebSocketManager { event.data ) as WebSocketServerMessage; - if (["vaultEvents", "cursorPositions"].includes(message.type)) + if (["vaultChanged", "cursorPositions"].includes(message.type)) this.expectResponse(socket); // Track the message handling promise const messageHandlingPromise = this.handleWebSocketMessage( @@ -340,8 +339,8 @@ export class WebSocketManager { private async handleWebSocketMessage( message: WebSocketServerMessage ): Promise { - if (message.type === "vaultEvents") { - await this.onRemoteVaultUpdateReceived.triggerAsync(message); + if (message.type === "vaultChanged") { + await this.onRemoteVaultUpdateReceived.triggerAsync(); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition } else if (message.type === "cursorPositions") { diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 51b92dfa..8660c0b1 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -11,7 +11,6 @@ import type { FileSystemOperations } from "./file-operations/filesystem-operatio import { FileOperations } from "./file-operations/file-operations"; import { Logger, LogLevel } from "./tracing/logger"; import { SyncHistory } from "./tracing/sync-history"; -import { FetchController } from "./services/fetch-controller"; import { SyncService } from "./services/sync-service"; import { ServerConfig } from "./services/server-config"; import { WebSocketManager } from "./services/websocket-manager"; @@ -23,7 +22,6 @@ import { DocumentSyncStatus } from "./types/document-sync-status"; import type { CursorSpan } from "./services/types/CursorSpan"; import { isInternalPath } from "./utils/portable-path"; import { FixedSizeDocumentCache } from "./utils/data-structures/fix-sized-cache"; -import { globsToRegexes } from "./utils/globs-to-regexes"; import { Lock } from "./utils/data-structures/locks"; import { unappliedChanges, @@ -61,7 +59,7 @@ export class SyncClient { private readonly database: Database, private readonly syncer: Syncer, private readonly webSocketManager: WebSocketManager, - private readonly fetchController: FetchController, + private readonly service: SyncService, private readonly serverConfig: ServerConfig, private readonly cursorTracker: CursorTracker, private readonly fileChangeNotifier: FileChangeNotifier, @@ -95,8 +93,6 @@ export class SyncClient { persistence: PersistenceProvider; fetch?: typeof globalThis.fetch; webSocket?: typeof globalThis.WebSocket; - /** V4 preserves bytes and line endings as stored, on every platform. */ - nativeLineEndings?: string; }): Promise { const logger = new Logger(); let stored = (await persistence.load()) ?? {}; @@ -152,32 +148,21 @@ export class SyncClient { "Offline notifications belong to another vault; use a separate state store" ); } - const fetchController = new FetchController( - settings.getSettings().isSyncEnabled, - logger - ); const deviceId = createClientId(); - const service = new SyncService( - deviceId, - fetchController, - settings, - logger, - fetch, - { - get: () => - stored.historyCheckpoint?.vaultKey === - vaultKey(settings.getSettings()) - ? stored.historyCheckpoint.checkpoint - : undefined, - save: async (checkpoint) => - save({ - historyCheckpoint: { - vaultKey: vaultKey(settings.getSettings()), - checkpoint - } - }) - } - ); + const service = new SyncService(deviceId, settings, fetch, { + get: () => + stored.historyCheckpoint?.vaultKey === + vaultKey(settings.getSettings()) + ? stored.historyCheckpoint.checkpoint + : undefined, + save: async (checkpoint) => + save({ + historyCheckpoint: { + vaultKey: vaultKey(settings.getSettings()), + checkpoint + } + }) + }); const serverConfig = new ServerConfig(service); const notifier = new FileChangeNotifier(); let syncer: Syncer; @@ -192,15 +177,8 @@ export class SyncClient { entries: () => localChanges, flush: async () => syncer?.flushLocalChanges() }, - (path, size) => { - const current = settings.getSettings(); - return ( - size > current.maxFileSizeMB * 1024 * 1024 || - globsToRegexes(current.ignorePatterns, logger).some( - (pattern) => pattern.test(path) - ) - ); - } + (path, size) => + settings.isIgnored(path) || settings.isOversized(size) ); const websocket = new WebSocketManager( deviceId, @@ -240,7 +218,7 @@ export class SyncClient { database, syncer, websocket, - fetchController, + service, serverConfig, cursors, notifier, @@ -308,15 +286,14 @@ export class SyncClient { if (this.destroyed) return; if (!this.settings.getSettings().isSyncEnabled) return; await this.database.bindVault(vaultKey(this.settings.getSettings())); - this.fetchController.finishReset(); - this.fetchController.canFetch = true; + this.service.resume(); this.syncer.start(); await this.syncer.waitUntilFinished(); } private async pause(): Promise { const stopped = this.syncer.stop(); - this.fetchController.startReset(); + this.service.pause(); await this.webSocketManager.stop(); await stopped; } @@ -470,7 +447,7 @@ export class SyncClient { this.destroyed = true; // Interrupt a start/reset awaiting network progress before waiting for // its lifecycle operation to finish. - this.fetchController.startReset(); + this.service.pause(); const stopped = this.syncer.stop(); this.destroying = this.lifecycle.withLock(async () => { try { diff --git a/frontend/sync-client/src/sync-operations/syncer.test.ts b/frontend/sync-client/src/sync-operations/syncer.test.ts index 862c56e1..524fbabb 100644 --- a/frontend/sync-client/src/sync-operations/syncer.test.ts +++ b/frontend/sync-client/src/sync-operations/syncer.test.ts @@ -4,16 +4,15 @@ import { afterEach, beforeEach, describe, it, mock } from "node:test"; import type { Database } from "../persistence/database"; import { DEFAULT_SETTINGS, - type Settings, + Settings, type SyncSettings } from "../persistence/settings"; import type { FileOperations } from "../file-operations/file-operations"; import type { SyncService } from "../services/sync-service"; import type { ServerConfig } from "../services/server-config"; import type { WebSocketManager } from "../services/websocket-manager"; -import type { Logger } from "../tracing/logger"; +import { Logger } from "../tracing/logger"; import type { SyncHistory } from "../tracing/sync-history"; -import type { EventBatch } from "../services/types/EventBatch"; import { EventListeners } from "../utils/data-structures/event-listeners"; import { Syncer } from "./syncer"; import { FixedSizeDocumentCache } from "../utils/data-structures/fix-sized-cache"; @@ -24,12 +23,11 @@ function createSyncer(overrides: Partial = {}): { syncer: Syncer; runCount: () => number; } { - const settings = { - getSettings: () => ({ - ...DEFAULT_SETTINGS, - ...overrides - }) - } as unknown as Settings; + const settings = new Settings( + new Logger(), + overrides, + async () => undefined + ); const database = { state: {} } as unknown as Database; @@ -162,7 +160,7 @@ describe("Syncer per-file state and event delivery", () => { (connected: boolean) => unknown >(), onRemoteVaultUpdateReceived: new EventListeners< - (batch: EventBatch) => Promise + () => Promise >() } as unknown as WebSocketManager; const service = { @@ -174,9 +172,11 @@ describe("Syncer per-file state and event delivery", () => { "device", {} as Logger, database, - { - getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) - } as Settings, + new Settings( + new Logger(), + { syncIntervalMs: 0 }, + async () => undefined + ), service, websocket, {} as FileOperations, @@ -238,9 +238,11 @@ describe("Syncer diff uploads", () => { "device", {} as Logger, { state: {} } as Database, - { - getSettings: () => ({ ...DEFAULT_SETTINGS, syncIntervalMs: 0 }) - } as Settings, + new Settings( + new Logger(), + { syncIntervalMs: 0 }, + async () => undefined + ), { getDocumentVersionContent: async () => parent } as unknown as SyncService, diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 9753c7c5..8a123c04 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -26,7 +26,6 @@ import type { VaultSnapshot } from "../services/types/VaultSnapshot"; import type { PushContent } from "../services/types/PushContent"; import type { Logger } from "../tracing/logger"; import { EventListeners } from "../utils/data-structures/event-listeners"; -import { globsToRegexes } from "../utils/globs-to-regexes"; import { mergeContent, toStoredSnapshot } from "./content"; import { scanLocalFiles, type LocalChange } from "./scan"; import { @@ -91,8 +90,7 @@ export class Syncer { websocket.sendHandshakeMessage({ type: "handshake", token: settings.getSettings().token, - deviceId, - lastSeenVaultUpdateId: database.state.lastSeenUpdateId + deviceId }); } // Failed upgrades and lost transports still need HTTP catchup, @@ -128,7 +126,7 @@ export class Syncer { !this.hasScanned || this.pendingPaths.has(path) || this.unsyncablePaths.has(path) || - this.ignored(path) + this.settings.isIgnored(path) ) { return false; } @@ -414,7 +412,10 @@ export class Syncer { const info = await this.files.fs.stat(path); if (info?.kind !== "file") continue; const doc = previous.documents[id]; - if (this.ignored(path) || this.oversized(info.size)) { + if ( + this.settings.isIgnored(path) || + this.settings.isOversized(info.size) + ) { next.local[id] = path; next.documents[id] = { materialized: true, @@ -457,20 +458,6 @@ export class Syncer { }; } - private ignored(path: string): boolean { - return ( - isInternalPath(path) || - globsToRegexes( - this.settings.getSettings().ignorePatterns, - this.logger - ).some((pattern) => pattern.test(path)) - ); - } - - private oversized(size: number): boolean { - return size > this.settings.getSettings().maxFileSizeMB * 1024 * 1024; - } - private async cacheContent( updateId: number, path: string | undefined, @@ -550,8 +537,8 @@ export class Syncer { guard: this.localGuard(), changes: this.changes, files: this.files, - ignored: (path) => this.ignored(path), - oversized: (size) => this.oversized(size), + ignored: (path) => this.settings.isIgnored(path), + oversized: (size) => this.settings.isOversized(size), commit: async (next) => this.database.commit(next) }, initial @@ -735,8 +722,8 @@ export class Syncer { }); if ( next.excluded?.[head.documentId] !== undefined || - this.ignored(path) || - this.oversized(head.contentSize) + this.settings.isIgnored(path) || + this.settings.isOversized(head.contentSize) ) { return undefined; } @@ -748,7 +735,7 @@ export class Syncer { : undefined; if ( local !== undefined && - this.oversized(base64ToBytes(local.contentBase64).length) + this.settings.isOversized(base64ToBytes(local.contentBase64).length) ) { return undefined; } @@ -869,9 +856,9 @@ export class Syncer { for (const [id, oldPath] of Object.entries(before)) { const path = next.local[id]; if ( - this.ignored(oldPath) || + this.settings.isIgnored(oldPath) || this.unsyncablePaths.has(oldPath) || - (path !== undefined && this.ignored(path)) + (path !== undefined && this.settings.isIgnored(path)) ) { next.excluded ??= {}; if (next.excluded[id] === undefined) @@ -882,7 +869,7 @@ export class Syncer { for (const [id, path] of Object.entries(next.excluded ?? {})) { if ( before[id] === undefined && - (path === null || !this.ignored(path)) + (path === null || !this.settings.isIgnored(path)) ) delete next.excluded![id]; } @@ -906,7 +893,7 @@ export class Syncer { next.documents[id].materialized = false; if ( next.excluded?.[id] === undefined && - !this.ignored(path) && + !this.settings.isIgnored(path) && remote.entries[id] !== undefined && (!this.database.state.local[id] || !next.documents[id].base || @@ -971,11 +958,11 @@ export class Syncer { } for (const [id, path] of Object.entries(this.database.state.local)) { if ( - this.ignored(path) || + this.settings.isIgnored(path) || this.database.state.excluded?.[id] !== undefined || !this.database.state.documents[id]?.materialized || this.unsyncablePaths.has(path) || - this.oversized( + this.settings.isOversized( // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Document heads are a sparse ID map at runtime. this.database.state.remoteHeads[id]?.contentSize ?? 0 ) @@ -985,7 +972,9 @@ export class Syncer { const snapshot = await this.files.snapshot(path); if ( !snapshot || - this.oversized(base64ToBytes(snapshot.contentBase64).length) + this.settings.isOversized( + base64ToBytes(snapshot.contentBase64).length + ) ) { continue; } @@ -1088,7 +1077,7 @@ export class Syncer { doc?.rejected && doc.observedHash === doc.rejected.hash && doc.base?.hash !== doc.rejected.hash && - !this.ignored(path) && + !this.settings.isIgnored(path) && state.excluded?.[id] === undefined ) return doc.rejected.message; diff --git a/frontend/sync-client/src/utils/data-structures/locks.test.ts b/frontend/sync-client/src/utils/data-structures/locks.test.ts index f7f0acf2..e325496b 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.test.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.test.ts @@ -1,292 +1,46 @@ -import { describe, it, beforeEach } from "node:test"; -import assert from "node:assert"; -import { Logger } from "../../tracing/logger"; -import type { RelativePath } from "../../persistence/database"; -import { Locks } from "./locks"; -import { awaitAll } from "../await-all"; -import { sleep } from "../sleep"; -import { SyncResetError } from "../../errors/errors"; +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Lock } from "./locks"; -describe("withLock", () => { - const testPath: RelativePath = "test/document/path"; - const testPath2: RelativePath = "test/document/path2"; - const logger = new Logger(); - - // eslint-disable-next-line @typescript-eslint/init-declarations - let locks: Locks; - - beforeEach(() => { - locks = new Locks(logger); +test("queued operations run in order and retain their results", async () => { + const lock = new Lock(); + const held = Promise.withResolvers(); + const started = Promise.withResolvers(); + const calls: number[] = []; + const first = lock.withLock(async () => { + calls.push(1); + started.resolve(); + await held.promise; + calls.push(2); + return "first"; }); - - it("should execute function with single key lock", async () => { - let executionCount = 0; - const result = await locks.withLock(testPath, () => { - executionCount++; - return "success"; - }); - - assert.strictEqual(result, "success"); - assert.strictEqual(executionCount, 1); + const second = lock.withLock(() => { + calls.push(3); + return "second"; }); - - it("should execute async function with single key lock", async () => { - let executionCount = 0; - const result = await locks.withLock(testPath, async () => { - executionCount++; - await sleep(10); - return "async-success"; - }); - - assert.strictEqual(result, "async-success"); - assert.strictEqual(executionCount, 1); - }); - - it("should execute function with multiple key locks", async () => { - let executionCount = 0; - const result = await locks.withLock([testPath, testPath2], () => { - executionCount++; - return "multi-success"; - }); - - assert.strictEqual(result, "multi-success"); - assert.strictEqual(executionCount, 1); - }); - - it("should sort multiple keys to prevent deadlocks", async () => { - const executionOrder: string[] = []; - - // Start two concurrent operations with keys in different orders - const promise1 = locks.withLock([testPath2, testPath], async () => { - executionOrder.push("operation1-start"); - await sleep(50); - executionOrder.push("operation1-end"); - return "result1"; - }); - - const promise2 = locks.withLock([testPath, testPath2], async () => { - executionOrder.push("operation2-start"); - await sleep(50); - executionOrder.push("operation2-end"); - return "result2"; - }); - - const [result1, result2] = await awaitAll([promise1, promise2]); - - assert.strictEqual(result1, "result1"); - assert.strictEqual(result2, "result2"); - // One operation should complete entirely before the other starts - assert.deepStrictEqual(executionOrder, [ - "operation1-start", - "operation1-end", - "operation2-start", - "operation2-end" - ]); - }); - - it("should serialize access to same key", async () => { - const executionOrder: string[] = []; - - const promise1 = locks.withLock(testPath, async () => { - executionOrder.push("operation1-start"); - await sleep(50); - executionOrder.push("operation1-end"); - return "result1"; - }); - - const promise2 = locks.withLock(testPath, async () => { - executionOrder.push("operation2-start"); - await sleep(30); - executionOrder.push("operation2-end"); - return "result2"; - }); - - const [result1, result2] = await awaitAll([promise1, promise2]); - - assert.strictEqual(result1, "result1"); - assert.strictEqual(result2, "result2"); - assert.deepStrictEqual(executionOrder, [ - "operation1-start", - "operation1-end", - "operation2-start", - "operation2-end" - ]); - }); - - it("should allow concurrent access to different keys", async () => { - const executionOrder: string[] = []; - - const promise1 = locks.withLock(testPath, async () => { - executionOrder.push("operation1-start"); - await sleep(50); - - executionOrder.push("operation1-end"); - return "result1"; - }); - - const promise2 = locks.withLock(testPath2, async () => { - executionOrder.push("operation2-start"); - await sleep(30); - executionOrder.push("operation2-end"); - return "result2"; - }); - - const [result1, result2] = await awaitAll([promise1, promise2]); - - assert.strictEqual(result1, "result1"); - assert.strictEqual(result2, "result2"); - // Both operations should run concurrently - assert.strictEqual(executionOrder[0], "operation1-start"); - assert.strictEqual(executionOrder[1], "operation2-start"); - }); - - it("should release locks even if function throws", async () => { - const error = new Error("test error"); - - await assert.rejects( - locks.withLock(testPath, () => { - throw error; - }), - { message: "test error" } - ); - - // Lock should be released, allowing another operation - const result = await locks.withLock( - testPath, - () => "success-after-error" - ); - assert.strictEqual(result, "success-after-error"); - }); - - it("should release locks even if async function throws", async () => { - const error = new Error("async test error"); - - await assert.rejects( - locks.withLock(testPath, async () => { - await sleep(10); - - throw error; - }), - { message: "async test error" } - ); - - // Lock should be released, allowing another operation - const result = await locks.withLock( - testPath, - () => "success-after-async-error" - ); - assert.strictEqual(result, "success-after-async-error"); - }); - - it("should handle empty array of keys", async () => { - const result = await locks.withLock([], () => "empty-keys"); - assert.strictEqual(result, "empty-keys"); - }); - - it("should maintain FIFO order for multiple waiters", async () => { - const executionOrder: string[] = []; - - // Start first operation that holds the lock - const firstPromise = locks.withLock(testPath, async () => { - executionOrder.push("first-start"); - await sleep(100); - executionOrder.push("first-end"); - return "first"; - }); - - // Small delay to ensure first operation starts - await sleep(10); - - // Queue second and third operations - const secondPromise = locks.withLock(testPath, async () => { - executionOrder.push("second-start"); - await sleep(50); - executionOrder.push("second-end"); - return "second"; - }); - - const thirdPromise = locks.withLock(testPath, async () => { - executionOrder.push("third-start"); - await sleep(20); - executionOrder.push("third-end"); - return "third"; - }); - - const [first, second, third] = await awaitAll([ - firstPromise, - secondPromise, - thirdPromise - ]); - - assert.strictEqual(first, "first"); - assert.strictEqual(second, "second"); - assert.strictEqual(third, "third"); - assert.deepStrictEqual(executionOrder, [ - "first-start", - "first-end", - "second-start", - "second-end", - "third-start", - "third-end" - ]); + const third = lock.withLock(() => { + calls.push(4); + return "third"; }); + await started.promise; + assert.deepEqual(calls, [1]); + held.resolve(); + assert.equal(await first, "first"); + assert.equal(await second, "second"); + assert.equal(await third, "third"); + assert.deepEqual(calls, [1, 2, 3, 4]); }); -describe("reset", () => { - const testPath: RelativePath = "test/document/path"; - const logger = new Logger(); - - // eslint-disable-next-line @typescript-eslint/init-declarations - let locks: Locks; - - beforeEach(() => { - locks = new Locks(logger); - }); - - it("should reject pending waiters with SyncResetError while running operation completes", async () => { - const firstPromise = locks.withLock(testPath, async () => { - await sleep(2); - return "first"; - }); - - await sleep(1); - - const secondPromise = locks.withLock(testPath, async () => "second"); - void secondPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function - - locks.reset(); - - assert.strictEqual(await firstPromise, "first"); - - await assert.rejects(secondPromise, (err: Error) => { - assert.ok(err instanceof SyncResetError); - return true; +for (const asynchronous of [false, true]) { + test(`a ${asynchronous ? "rejected promise" : "thrown error"} releases the lock`, async () => { + const lock = new Lock(); + const error = new Error("failed operation"); + const failed = lock.withLock(() => { + if (asynchronous) return Promise.reject(error); + throw error; }); + const next = lock.withLock(() => "continued"); + await assert.rejects(failed, (caught) => caught === error); + assert.equal(await next, "continued"); }); - - it("should allow locks to work normally after reset", async () => { - const firstPromise = locks.withLock(testPath, async () => { - await sleep(1); - return "first"; - }); - - await sleep(1); - - const secondPromise = locks.withLock(testPath, async () => "second"); - void secondPromise.catch(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function - - locks.reset(); - - await firstPromise; - - const result = await locks.withLock(testPath, () => "after-reset"); - assert.strictEqual(result, "after-reset"); - }); - - it("should handle reset with no pending operations", async () => { - locks.reset(); - - const result = await locks.withLock(testPath, () => "success"); - assert.strictEqual(result, "success"); - }); -}); +} diff --git a/frontend/sync-client/src/utils/data-structures/locks.ts b/frontend/sync-client/src/utils/data-structures/locks.ts index 096cacdf..34bf2e60 100644 --- a/frontend/sync-client/src/utils/data-structures/locks.ts +++ b/frontend/sync-client/src/utils/data-structures/locks.ts @@ -1,166 +1,11 @@ -import { SyncResetError } from "../../errors/errors"; -import type { Logger } from "../../tracing/logger"; -import { awaitAll } from "../await-all"; - -/** - * Manages exclusive locks on items to prevent concurrent modifications. - * Locks are granted in FIFO order. - * - * @template T The type of the key used for locking - */ -export class Locks { - /** Currently locked keys */ - private readonly locked = new Set(); - - /** Queue of resolve functions waiting for each key */ - private readonly waiters = new Map< - T, - [() => unknown, (err: unknown) => unknown][] - >(); - - public constructor(private readonly logger?: Logger) {} - - /** - * Executes a function while holding exclusive locks on one or more keys. - * - * This method ensures that the provided function runs with exclusive access to the - * specified key(s). Multiple keys are sorted to prevent deadlocks when different - * operations request the same keys in different orders. - * - * @template R The return type of the function to execute - * @param keyOrKeys A single key or array of keys to lock during function execution - * @param fn The function to execute while holding the lock(s). Can be sync or async. - * @returns A Promise that resolves to the return value of the executed function - * - * @example - * ```typescript - * // Lock a single key - * const result = await locks.withLock('file1', () => { - * // Critical section - only one operation can access 'file1' at a time - * return processFile('file1'); - * }); - * - * // Lock multiple keys (prevents deadlocks through consistent ordering) - * await locks.withLock(['file1', 'file2'], async () => { - * // Critical section - exclusive access to both files - * await moveFile('file1', 'file2'); - * }); - * ``` - * - * @throws Any error thrown by the provided function will be propagated after locks are released - */ - public async withLock( - keyOrKeys: T | T[], - fn: () => R | Promise - ): Promise { - const keys = Array.isArray(keyOrKeys) ? keyOrKeys : [keyOrKeys]; - - // Deduplicate keys to prevent deadlock from acquiring same lock twice - const uniqueKeys = Array.from(new Set(keys)); - uniqueKeys.sort((a, b) => String(a).localeCompare(String(b))); // Ensure consistent order to prevent deadlocks - - await awaitAll(uniqueKeys.map(async (key) => this.waitForLock(key))); - - try { - return await fn(); - } finally { - uniqueKeys.forEach((key) => { - this.unlock(key); - }); - } - } - - public reset(): void { - // Resolve all waiting promises before clearing to prevent deadlock - // Any operation waiting for a lock will be granted access immediately - for (const waiting of this.waiters.values()) { - for (const [_, reject] of waiting) { - reject(new SyncResetError()); - } - } - this.locked.clear(); - this.waiters.clear(); - } - - /** - * Attempts to acquire a lock immediately without waiting. - * Must call `unlock()` if successful. - * - * @param key The key to lock - * @returns `true` if lock acquired, `false` if already locked - */ - public tryLock(key: T): boolean { - if (this.locked.has(key)) { - return false; - } - - this.locked.add(key); - - return true; - } - - /** - * Waits to acquire a lock, blocking until available. - * Operations are queued in FIFO order. Must call `unlock()` when done. - * - * @param key The key to wait for and lock - * @returns Promise that resolves when lock is acquired - */ - public async waitForLock(key: T): Promise { - if (this.tryLock(key)) { - return Promise.resolve(); - } - - this.logger?.debug(`Waiting for lock on ${key}`); - - return new Promise((resolve, reject) => { - // DefaultDict behavior - let waiting = this.waiters.get(key); - if (!waiting) { - waiting = []; - this.waiters.set(key, waiting); - } - - waiting.push([resolve, reject]); - }); - } - - /** - * Releases a lock and grants access to the next waiting operation in FIFO order. - * Removes the key from locked set if no waiters. - * - * @param key The key to unlock - * @throws {Error} If key is not currently locked - */ - public unlock(key: T): void { - if (!this.locked.has(key)) { - return; - } - - // Remove first waiter to ensure FIFO order - const [resolveNextWaiting, _] = this.waiters.get(key)?.shift() ?? []; - - if (resolveNextWaiting) { - this.logger?.debug(`Granted lock on ${key}`); - resolveNextWaiting(); - } else { - this.locked.delete(key); - } - } -} - +/** Serializes operations in arrival order; failures do not block the queue. */ export class Lock { - private readonly locks: Locks; + private tail: Promise = Promise.resolve(); - public constructor(logger?: Logger) { - this.locks = new Locks(logger); - } - - public async withLock(fn: () => R | Promise): Promise { - return this.locks.withLock(true, fn); - } - - public reset(): void { - this.locks.reset(); + // eslint-disable-next-line @typescript-eslint/promise-function-async -- Preserve the queued operation promise without an extra async wrapper. + public withLock(fn: () => R | Promise): Promise { + const operation = this.tail.then(fn); + this.tail = operation.catch(() => undefined); + return operation; } } diff --git a/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts index c64bff18..258db398 100644 --- a/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts +++ b/frontend/sync-client/src/utils/debugging/slow-web-socket-factory.ts @@ -1,5 +1,5 @@ import { sleep } from "../sleep"; -import { Locks } from "../data-structures/locks"; +import { Lock } from "../data-structures/locks"; import type { Logger } from "../../tracing/logger"; export function slowWebSocketFactory( @@ -8,10 +8,8 @@ export function slowWebSocketFactory( ): typeof WebSocket { // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion return class FlakyWebSocket extends WebSocket { - private static readonly RECEIVE_KEY = "websocket-receive"; - private static readonly SEND_KEY = "websocket-send"; - - private readonly locks = new Locks(logger); + private readonly receiving = new Lock(); + private readonly sending = new Lock(); public set onopen(callback: ((event: Event) => void) | null) { super.onopen = async (event: Event): Promise => { @@ -25,18 +23,15 @@ export function slowWebSocketFactory( public set onmessage(callback: ((event: MessageEvent) => void) | null) { super.onmessage = async (event: MessageEvent): Promise => { - await this.locks.withLock( - FlakyWebSocket.RECEIVE_KEY, - async () => { - if (jitterScaleInSeconds > 0) { - await sleep( - Math.random() * jitterScaleInSeconds * 1000 - ); - } - - callback?.(event); + await this.receiving.withLock(async () => { + if (jitterScaleInSeconds > 0) { + await sleep( + Math.random() * jitterScaleInSeconds * 1000 + ); } - ); + + callback?.(event); + }); }; } @@ -70,7 +65,7 @@ export function slowWebSocketFactory( data: string | ArrayBufferLike | Blob | ArrayBufferView ): Promise { // maintain message order - await this.locks.withLock(FlakyWebSocket.SEND_KEY, async () => { + await this.sending.withLock(async () => { if (jitterScaleInSeconds > 0) { await sleep(Math.random() * jitterScaleInSeconds * 1000); } diff --git a/frontend/test-support/protocol-audit.test.ts b/frontend/test-support/protocol-audit.test.ts index 2cb7d782..bf888cad 100644 --- a/frontend/test-support/protocol-audit.test.ts +++ b/frontend/test-support/protocol-audit.test.ts @@ -63,8 +63,7 @@ test( JSON.stringify({ type: "handshake", token, - deviceId: randomUUID(), - lastSeenVaultUpdateId: 0 + deviceId: randomUUID() }) ); ws.onmessage = () => resolve(true); @@ -302,8 +301,7 @@ test( JSON.stringify({ type: "handshake", token, - deviceId, - lastSeenVaultUpdateId: 0 + deviceId }) ); ws.onmessage = () => resolve(); diff --git a/frontend/test-support/protocol.test.ts b/frontend/test-support/protocol.test.ts index 50b0a38d..6e96bda6 100644 --- a/frontend/test-support/protocol.test.ts +++ b/frontend/test-support/protocol.test.ts @@ -282,7 +282,7 @@ for (const [kind, format, originalPath] of [ } test( - "concurrent CAS commits have identical HTTP/event-log/WebSocket total order and reconnect catchup", + "WebSocket hints trigger HTTP catchup in commit order, including after reconnect", { timeout: 20_000 }, async () => { const server = makeServer(); @@ -291,17 +291,29 @@ test( await server.start(); const url = `${server.remoteUri}/vaults/order-${randomUUID()}`; const received: number[] = []; - const connect = async (after: number) => { + let catchup = Promise.resolve(); + let catchupError: unknown; + const connect = async () => { ws = new WebSocket(`${url.replace(/^http/, "ws")}/ws`); ws.addEventListener("message", (message) => { - const batch = JSON.parse(String(message.data)) as { - type: string; - events: { eventId: number }[]; - }; - if (batch.type === "vaultEvents") - received.push( - ...batch.events.map((event) => event.eventId) - ); + const hint = JSON.parse(String(message.data)); + if (hint.type !== "vaultChanged") return; + catchup = catchup + .then(async () => { + assert.deepEqual(hint, { type: "vaultChanged" }); + const batch = await getJson<{ + events: { eventId: number }[]; + }>( + `${url}/events-since?after=${received.at(-1) ?? 0}`, + token + ); + received.push( + ...batch.events.map((event) => event.eventId) + ); + }) + .catch((error) => { + catchupError = error; + }); }); await new Promise((resolve, reject) => { ws!.onopen = () => resolve(); @@ -311,12 +323,11 @@ test( JSON.stringify({ type: "handshake", token, - deviceId: "order-observer", - lastSeenVaultUpdateId: after + deviceId: "order-observer" }) ); }; - await connect(0); + await connect(); const ids = Array.from({ length: 12 }, () => randomUUID()); const requests = ids.map((id, i) => ({ id, @@ -360,10 +371,11 @@ test( const deadline = Date.now() + 5_000; while (received.length < head && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 10)); + if (catchupError) throw catchupError; assert.deepEqual( received, Array.from({ length: head }, (_, i) => i + 1), - "WebSocket reordered, lost or duplicated an event" + "HTTP catchup after WebSocket hints lost or duplicated an event" ); }; await waitForEvents(13); @@ -375,7 +387,7 @@ test( `${url}/documents/${ids[0]}`, content("next", replies[0].vaultUpdateId) ); - await connect(13); + await connect(); await waitForEvents(14); const log = await getJson<{ events: { eventId: number }[] }>( `${url}/events-since?after=0`, diff --git a/frontend/test-support/sync-fixture.ts b/frontend/test-support/sync-fixture.ts index 30bf6a85..79047f5d 100644 --- a/frontend/test-support/sync-fixture.ts +++ b/frontend/test-support/sync-fixture.ts @@ -11,7 +11,6 @@ import { Logger } from "../sync-client/src/tracing/logger"; import { SyncHistory } from "../sync-client/src/tracing/sync-history"; import { FixedSizeDocumentCache } from "../sync-client/src/utils/data-structures/fix-sized-cache"; import { EventListeners } from "../sync-client/src/utils/data-structures/event-listeners"; -import { globsToRegexes } from "../sync-client/src/utils/globs-to-regexes"; import type { ServerConfig } from "../sync-client/src/services/server-config"; import type { SyncService } from "../sync-client/src/services/sync-service"; import type { WebSocketManager } from "../sync-client/src/services/websocket-manager"; @@ -81,11 +80,7 @@ export async function fixture( config, undefined, { entries: () => changes, flush: async () => {} }, - (path, size) => - size > settings.getSettings().maxFileSizeMB * 1024 * 1024 || - globsToRegexes(settings.getSettings().ignorePatterns, logger).some( - (pattern) => pattern.test(path) - ) + (path, size) => settings.isIgnored(path) || settings.isOversized(size) ); const websocket = { onWebSocketStatusChanged: new EventListeners(), diff --git a/sync-server/README.md b/sync-server/README.md index 27e68c20..a5c40318 100644 --- a/sync-server/README.md +++ b/sync-server/README.md @@ -86,13 +86,11 @@ content/file-manifest version IDs are the event IDs that created them. Rejected requests and retries of accepted requests append no events. Timestamps never determine event ordering. -The WebSocket handshake retains `lastSeenVaultUpdateId`. The server sends -`{type: "vaultEvents", headEventId, events}` in increasing order, including events -originated by that client. Each connection drains the database through one sender. -In-memory notifications only wake it; polling covers missed wakes and requests -interrupted after commit. Reconnect with the last durably processed event ID. The -HTTP replay endpoint uses the same history, so losing the final notification is -recoverable. +WebSocket messages are `{type: "vaultChanged"}` hints. Clients fetch ordered +history through HTTP from their last incorporated event ID. The server compares +its current history checkpoint after broadcasts and periodically, so missed wakes +and requests interrupted after commit still trigger catchup. Reconnecting sends a +fresh hint without replaying events over the socket. Cursor messages are unchanged. A fresh client reads `/vault-snapshot`, applies it durably, then replays after its watermark. Successful HTTP responses include `X-Vault-Link-History`, containing an event ID and a random event-incarnation token. Clients persist and echo this diff --git a/sync-server/src/app_state/websocket/models.rs b/sync-server/src/app_state/websocket/models.rs index 32d97966..4eb1fa10 100644 --- a/sync-server/src/app_state/websocket/models.rs +++ b/sync-server/src/app_state/websocket/models.rs @@ -1,16 +1,13 @@ use serde::{Deserialize, Serialize}; use ts_rs::TS; -use crate::app_state::database::models::{DeviceId, DocumentId, EventBatch, VaultUpdateId}; +use crate::app_state::database::models::{DeviceId, DocumentId, VaultUpdateId}; #[derive(TS, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] pub struct WebSocketHandshake { pub token: String, pub device_id: DeviceId, - - #[ts(as = "Option")] - pub last_seen_vault_update_id: Option, } #[derive(TS, Deserialize, Clone, Debug)] @@ -67,6 +64,6 @@ pub enum WebSocketClientMessage { #[serde(rename_all = "camelCase", tag = "type")] #[ts(export)] pub enum WebSocketServerMessage { - VaultEvents(EventBatch), + VaultChanged, CursorPositions(CursorPositionFromServer), } diff --git a/sync-server/src/consts.rs b/sync-server/src/consts.rs index 83d99a82..e1fee209 100644 --- a/sync-server/src/consts.rs +++ b/sync-server/src/consts.rs @@ -20,4 +20,4 @@ pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; -pub const SUPPORTED_API_VERSION: u32 = 4; +pub const SUPPORTED_API_VERSION: u32 = 5; diff --git a/sync-server/src/server/endpoints/websocket.rs b/sync-server/src/server/endpoints/websocket.rs index 3a97911b..a4b779de 100644 --- a/sync-server/src/server/endpoints/websocket.rs +++ b/sync-server/src/server/endpoints/websocket.rs @@ -69,26 +69,21 @@ async fn websocket( let device = authenticated.handshake.device_id; let session = state.cursors.register_connection(&vault, &device).await; - let mut after = authenticated - .handshake - .last_seen_vault_update_id - .unwrap_or(0); + let mut last_checkpoint = None; let mut notifications = state.broadcasts.get_receiver(vault.clone()).await; let mut timer = tokio::time::interval(Duration::from_secs(2)); timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); info!("WebSocket connected to vault {vault}"); - // Exactly one sender drains durable events in commit order. In-memory - // notifications are only wakeups, so lag and commit/notify crashes are safe. + // HTTP owns event replay. Poll the cheap checkpoint to recover missed + // broadcasts, including a commit followed by an interrupted notification. let result = async { loop { - let batch = state.database.events_after(&vault, after).await.map_err(server_error)?; - - if !batch.events.is_empty() { - let head = batch.end_event_id.unwrap_or(batch.head_event_id); - send_update_over_websocket(&WebSocketServerMessage::VaultEvents(batch), &mut sender).await?; - after = head; + let checkpoint = state.database.history_checkpoint(&vault).await.map_err(server_error)?; + if last_checkpoint.as_ref() != Some(&checkpoint) { + send_update_over_websocket(&WebSocketServerMessage::VaultChanged, &mut sender).await?; + last_checkpoint = Some(checkpoint); } tokio::select! { @@ -99,7 +94,7 @@ async fn websocket( send_update_over_websocket(&WebSocketServerMessage::CursorPositions(positions), &mut sender).await?; }, - Err(RecvError::Lagged(_)) => {}, // Drain the durable log on the next iteration. + Err(RecvError::Lagged(_)) => {}, // Check the durable head on the next iteration. Err(RecvError::Closed) => break, }, -- 2.54.0 From 96d720eed96e965a426558496f040024ec9003b2 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 22 Sep 2026 08:00:29 +0100 Subject: [PATCH 48/48] simplify 3 --- docs/architecture/sync-algorithm.md | 16 +- frontend/local-client-cli/README.md | 13 +- frontend/local-client-cli/src/cli.ts | 32 +- .../src/client-persistence.test.ts | 80 ++++ .../src/client-persistence.ts | 67 +++ frontend/local-client-cli/src/file-watcher.ts | 93 ++--- .../src/node-filesystem.test.ts | 77 ++-- .../local-client-cli/src/node-filesystem.ts | 381 ++++++++---------- frontend/local-client-cli/webpack.config.js | 1 + frontend/obsidian-plugin/README.md | 82 ++-- frontend/obsidian-plugin/package.json | 2 +- .../obsidian-plugin/src/mobile-file-system.ts | 130 ++++++ .../src/obsidian-file-system.test.ts | 167 ++++++++ .../src/obsidian-file-system.ts | 265 ++++-------- .../obsidian-plugin/src/vault-link-plugin.ts | 113 ++++-- .../src/views/cursors/file-explorer.ts | 2 +- .../views/cursors/remote-cursors-plugin.ts | 2 +- .../status-description/status-description.ts | 2 +- frontend/obsidian-plugin/tsconfig.json | 1 + frontend/obsidian-plugin/webpack.config.js | 5 + frontend/package.json | 2 +- frontend/sync-client/README.md | 14 +- frontend/sync-client/src/consts.ts | 1 - frontend/sync-client/src/index.ts | 2 +- .../src/services/server-config.test.ts | 79 ++++ .../sync-client/src/services/server-config.ts | 49 +-- .../src/services/sync-service.test.ts | 35 ++ .../sync-client/src/services/sync-service.ts | 1 + frontend/sync-client/src/sync-client.ts | 4 +- .../sync-client/src/sync-operations/syncer.ts | 8 +- .../utils/data-structures/fix-sized-cache.ts | 108 +---- frontend/test-support/README.md | 15 +- .../test-support/engine-regressions.test.ts | 4 +- frontend/test-support/native-fs.c | 27 -- .../test-support/production-adapters.test.ts | 207 ++++++++++ frontend/test-support/real-disk.test.ts | 41 +- frontend/test-support/real-disk.ts | 171 -------- frontend/test-support/storage.ts | 9 +- scripts/e2e-runner.mjs | 7 +- 39 files changed, 1319 insertions(+), 996 deletions(-) create mode 100644 frontend/local-client-cli/src/client-persistence.test.ts create mode 100644 frontend/local-client-cli/src/client-persistence.ts create mode 100644 frontend/obsidian-plugin/src/mobile-file-system.ts create mode 100644 frontend/obsidian-plugin/src/obsidian-file-system.test.ts create mode 100644 frontend/sync-client/src/services/server-config.test.ts delete mode 100644 frontend/test-support/native-fs.c create mode 100644 frontend/test-support/production-adapters.test.ts delete mode 100644 frontend/test-support/real-disk.ts diff --git a/docs/architecture/sync-algorithm.md b/docs/architecture/sync-algorithm.md index 4f3dc8bb..6272040e 100644 --- a/docs/architecture/sync-algorithm.md +++ b/docs/architecture/sync-algorithm.md @@ -54,7 +54,12 @@ updates only wake the loop to reread current bytes. Namespace changes detected before mutations invalidate the plan. Notifications during application are incorporated between files using their document identities; the next scan rereads content. Content is reread before replacement. Ignored and oversized occupants stay in place. Adapters -report atomic editor saves as updates and suppress engine-generated notifications. +use raw filesystem events only to wake a scan. These include engine writes and +atomic editor replacements, so they must not be reported as logical create/delete +notifications. Integrations with reliable user-action events can persist those +explicit notifications to preserve rename identities, as Obsidian does for +`Vault.rename`. Raw watchers infer a move +only when the missing and new files have a unique matching, nonempty content hash. Metadata persistence must replace a complete value atomically, leaving a readable old/new value after interruption. Metadata and user files need not describe the @@ -83,5 +88,10 @@ transient errors retry according to client settings. Observer callback failures are isolated from the background sync loop. API v5 clients and servers must be deployed together. V4 databases migrate without -removing history. The old Obsidian and CLI adapters still require the filesystem -and persistence contract required by the core engine. +removing history. CLI and desktop Obsidian share the Node filesystem adapter and +complete-record metadata persistence. Node file moves use exclusive creation +followed by unlink, so interruption may leave two ordinary files. Metadata alone +is flushed before an atomic rename; there is no user-file durability protocol. +Obsidian mobile uses the host storage API. Its exclusive-copy API requires a +scratch input file in the ignored internal directory; leftovers are never replayed +or used for recovery. Mobile metadata persistence uses Obsidian's data store. diff --git a/frontend/local-client-cli/README.md b/frontend/local-client-cli/README.md index f0cc6388..9c519a68 100644 --- a/frontend/local-client-cli/README.md +++ b/frontend/local-client-cli/README.md @@ -205,10 +205,21 @@ docker build -f local-client-cli/Dockerfile -t vault-link-cli:test . 1. Creates `.vaultlink` directory for sync metadata 2. Performs initial sync of local files to server -3. Watches filesystem for changes using Node's `fs.watch` +3. Uses filesystem events to wake a fresh scan, including atomic editor saves 4. Syncs changes bidirectionally in real-time 5. Handles graceful shutdown on SIGINT/SIGTERM +The complete client record (database, settings, queued logical notifications and +server history checkpoint) is saved atomically in `.vaultlink/sync-data.json`. +Existing database-only files migrate on the next save. Invalid or unreadable +metadata is reported instead of silently starting with empty state. + +File moves use an exclusive copy followed by deletion of the source. An +interruption may leave both files; the next scan reconciles the visible files. +Raw watcher events do not distinguish user edits from sync writes. A rename +preserves identity when the scan finds a unique matching nonempty content hash; +a simultaneous rename and edit may instead appear as deletion and creation. + ## License MIT diff --git a/frontend/local-client-cli/src/cli.ts b/frontend/local-client-cli/src/cli.ts index 0a40d9fb..5f2a8c9f 100644 --- a/frontend/local-client-cli/src/cli.ts +++ b/frontend/local-client-cli/src/cli.ts @@ -7,12 +7,12 @@ import { DEFAULT_SETTINGS, LogLevel, LogLine, - type SyncSettings, - type StoredDatabase + type SyncSettings } from "sync-client"; import { parseArgs } from "./args"; import { NodeFileSystemOperations, VAULTLINK_DIR } from "./node-filesystem"; import { FileWatcher } from "./file-watcher"; +import { ClientPersistence } from "./client-persistence"; import { formatLogLine } from "./logger-formatter"; import packageJson from "../package.json"; @@ -114,32 +114,7 @@ async function main(): Promise { const client = await SyncClient.create({ fs: fileSystem, - persistence: { - load: async () => { - let database: Partial | undefined = undefined; - try { - const content = await fs.readFile(dataFile, "utf-8"); - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - database = JSON.parse(content) as Partial; - } catch { - emitBoot( - LogLevel.WARNING, - `Cannot read data file at ${dataFile}` - ); - } - - return { - settings, - database - }; - }, - save: async ({ database: persistedDatabase }) => { - await fs.writeFile( - dataFile, - JSON.stringify(persistedDatabase, null, 2) - ); - } - } + persistence: new ClientPersistence(dataFile, settings) }); if (args.health !== undefined) { @@ -212,7 +187,6 @@ async function main(): Promise { client.logger.info(`${signal} received, shutting down gracefully`); fileWatcher.stop(); - await client.waitUntilFinished(); await client.destroy(); if (totalSyncOps > 0) { diff --git a/frontend/local-client-cli/src/client-persistence.test.ts b/frontend/local-client-cli/src/client-persistence.test.ts new file mode 100644 index 00000000..b400d50f --- /dev/null +++ b/frontend/local-client-cli/src/client-persistence.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import fs from "node:fs/promises"; +import { tmpdir } from "node:os"; +import * as path from "node:path"; +import { ClientPersistence } from "./client-persistence"; +import type { StoredClient } from "sync-client"; + +test("CLI persistence retains the whole record and migrates database-only saves", async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-metadata-") + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const file = path.join(directory, "state.json"); + const store = new ClientPersistence(file, { token: "current-token" }); + assert.deepEqual(await store.load(), { + settings: { token: "current-token" } + }); + const record: StoredClient = { + settings: { token: "old-token", vaultName: "vault" }, + database: { lastSeenUpdateId: 12 }, + localChanges: [ + { type: "delete", path: "removed.md", changeId: "queued" } + ], + localChangesVaultKey: "vault", + historyCheckpoint: { vaultKey: "vault", checkpoint: "12:incarnation" } + }; + await store.save(record); + assert.deepEqual(await store.load(), { + ...record, + settings: { ...record.settings, token: "current-token" } + }); + const legacy = { + fileManifest: { fileManifestId: 0, entries: {} }, + lastSeenUpdateId: 0 + }; + await fs.writeFile(file, JSON.stringify(legacy)); + assert.deepEqual((await store.load()).database, legacy); + await fs.writeFile(file, "{truncated"); + await assert.rejects(store.load(), SyntaxError); + await fs.rm(file); + await fs.mkdir(file); + await assert.rejects(store.load(), /EISDIR/); +}); + +for (const afterReplacement of [false, true]) { + test(`failed metadata save ${afterReplacement ? "after" : "before"} replacement leaves a complete record`, async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-save-") + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const store = new ClientPersistence( + path.join(directory, "state.json"), + {} + ); + const old: StoredClient = { + historyCheckpoint: { vaultKey: "v", checkpoint: "1:old" } + }; + const next: StoredClient = { + ...old, + localChanges: [{ type: "delete", path: "note.md", changeId: "new" }] + }; + await store.save(old); + const rename = fs.rename; + t.mock.method( + fs, + "rename", + async (...args: Parameters) => { + if (afterReplacement) await rename(...args); + throw new Error("interrupted replacement"); + } + ); + await assert.rejects(store.save(next), /interrupted replacement/); + assert.deepEqual(await store.load(), { + ...(afterReplacement ? next : old), + settings: {} + }); + assert.deepEqual(await fs.readdir(directory), ["state.json"]); + }); +} diff --git a/frontend/local-client-cli/src/client-persistence.ts b/frontend/local-client-cli/src/client-persistence.ts new file mode 100644 index 00000000..ce421654 --- /dev/null +++ b/frontend/local-client-cli/src/client-persistence.ts @@ -0,0 +1,67 @@ +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { randomUUID } from "node:crypto"; +import type { + PersistenceProvider, + StoredClient, + StoredDatabase, + SyncSettings +} from "sync-client"; + +/** Only client metadata needs an atomic replacement; user files do not. */ +export class ClientPersistence implements PersistenceProvider { + public constructor( + private readonly file: string, + private readonly settings: Partial + ) {} + + public async load(): Promise { + const content = await fs + .readFile(this.file, "utf8") + .catch((error: unknown) => { + if ( + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) + return undefined; + throw error; + }); + if (content === undefined) return { settings: this.settings }; + const parsed: unknown = JSON.parse(content); + if ( + parsed === null || + typeof parsed !== "object" || + Array.isArray(parsed) + ) + throw new Error("Invalid client metadata"); + // The sync engine validates and migrates the persisted fields. + const data = parsed as StoredClient & Partial; + // Earlier CLI versions saved the database alone. + const stored: StoredClient = + "fileManifest" in data ? { database: data } : data; + return { + ...stored, + settings: { ...stored.settings, ...this.settings } + }; + } + + public async save(data: StoredClient): Promise { + await fs.mkdir(path.dirname(this.file), { recursive: true }); + const temporary = `${this.file}.${randomUUID()}.tmp`; + try { + const handle = await fs.open(temporary, "wx", 0o600); + try { + await handle.writeFile(JSON.stringify(data)); + // Flush the complete value before exposing its name. A crash can + // keep the previous value, but must not expose truncated JSON. + await handle.sync(); + } finally { + await handle.close(); + } + await fs.rename(temporary, this.file); + } finally { + await fs.rm(temporary, { force: true }); + } + } +} diff --git a/frontend/local-client-cli/src/file-watcher.ts b/frontend/local-client-cli/src/file-watcher.ts index c273a412..541192f7 100644 --- a/frontend/local-client-cli/src/file-watcher.ts +++ b/frontend/local-client-cli/src/file-watcher.ts @@ -1,93 +1,54 @@ import Watcher from "watcher"; -import * as path from "path"; -import type { SyncClient, RelativePath } from "sync-client"; +import * as path from "node:path"; +import type { SyncClient } from "sync-client"; import { toUnixPath, matchesGlob } from "./path-utils"; +/** OS events include sync writes and editor temporary-file replacements. They + * wake reconciliation; only a fresh scan determines the current namespace. */ export class FileWatcher { private watcher: Watcher | undefined; - private isRunning = false; - private readonly ignorePatterns: string[]; public constructor( private readonly basePath: string, private readonly client: SyncClient, - ignorePatterns: string[] = [] - ) { - this.ignorePatterns = ignorePatterns; - } + private readonly ignorePatterns: string[] = [] + ) {} public start(): void { - if (this.isRunning) { - return; - } - - this.isRunning = true; - + if (this.watcher) return; this.watcher = new Watcher(this.basePath, { recursive: true, - renameDetection: true, - renameTimeout: 125, + renameDetection: false, ignoreInitial: true, - ignore: (filePath: string): boolean => this.shouldIgnore(filePath) + ignore: (filePath: string): boolean => + this.ignorePatterns.some((pattern) => + matchesGlob(this.toRelativePath(filePath), pattern) + ) }); - - this.watcher.on("add", (filePath: string) => { - this.handleCreate(this.toRelativePath(filePath)); + this.watcher.on("all", (_event: string, filePath: string) => { + void this.client + .syncLocallyUpdatedFile({ + relativePath: this.toRelativePath(filePath) + }) + .catch((error: unknown) => { + this.client.logger.error( + `File notification failed: ${error}` + ); + }); }); - - this.watcher.on("change", (filePath: string) => { - this.handleChange(this.toRelativePath(filePath)); + this.watcher.on("error", (error: Error) => { + this.client.logger.error(`File watcher failed: ${error.message}`); }); - - this.watcher.on("unlink", (filePath: string) => { - this.handleDelete(this.toRelativePath(filePath)); - }); - - this.watcher.on("rename", (oldPath: string, newPath: string) => { - this.handleRename( - this.toRelativePath(oldPath), - this.toRelativePath(newPath) - ); - }); - this.client.logger.info("File watcher started"); } public stop(): void { - if (this.watcher !== undefined) { - this.watcher.close(); - this.watcher = undefined; - } - this.isRunning = false; + this.watcher?.close(); + this.watcher = undefined; this.client.logger.info("File watcher stopped"); } - private shouldIgnore(filePath: string): boolean { - const rel = toUnixPath(path.relative(this.basePath, filePath)); - return this.ignorePatterns.some((pattern) => matchesGlob(rel, pattern)); - } - - private handleCreate(relativePath: RelativePath): void { - this.client.syncLocallyCreatedFile(relativePath); - } - - private handleChange(relativePath: RelativePath): void { - this.client.syncLocallyUpdatedFile({ relativePath }); - } - - private handleDelete(relativePath: RelativePath): void { - this.client.syncLocallyDeletedFile(relativePath); - } - - private handleRename(oldPath: RelativePath, newPath: RelativePath): void { - this.client.logger.info(`File renamed: ${oldPath} -> ${newPath}`); - this.client.syncLocallyUpdatedFile({ - oldPath, - relativePath: newPath - }); - } - - private toRelativePath(absolutePath: string): RelativePath { + private toRelativePath(absolutePath: string): string { return toUnixPath(path.relative(this.basePath, absolutePath)); } } diff --git a/frontend/local-client-cli/src/node-filesystem.test.ts b/frontend/local-client-cli/src/node-filesystem.test.ts index 55b286a4..3cab1147 100644 --- a/frontend/local-client-cli/src/node-filesystem.test.ts +++ b/frontend/local-client-cli/src/node-filesystem.test.ts @@ -11,10 +11,13 @@ test("NodeFileSystemOperations - read and write files", async () => { try { const content = new TextEncoder().encode("Hello, world!"); - await fsOps.write("test.txt", content); + await fsOps.write("test.txt", { content }); - const readContent = await fsOps.read("test.txt"); - assert.equal(new TextDecoder().decode(readContent), "Hello, world!"); + const readContent = await fsOps.readSnapshot("test.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "Hello, world!" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -27,10 +30,13 @@ test("NodeFileSystemOperations - create nested directories with forward slashes" try { const content = new TextEncoder().encode("Nested file"); // Always use forward slashes in API - await fsOps.write("dir1/dir2/test.txt", content); + await fsOps.write("dir1/dir2/test.txt", { content }); - const readContent = await fsOps.read("dir1/dir2/test.txt"); - assert.equal(new TextDecoder().decode(readContent), "Nested file"); + const readContent = await fsOps.readSnapshot("dir1/dir2/test.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "Nested file" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -43,7 +49,9 @@ test("NodeFileSystemOperations - exists with forward slashes", async () => { try { assert.equal(await fsOps.exists("test.txt"), false); - await fsOps.write("test.txt", new TextEncoder().encode("test")); + await fsOps.write("test.txt", { + content: new TextEncoder().encode("test") + }); assert.equal(await fsOps.exists("test.txt"), true); } finally { @@ -56,7 +64,9 @@ test("NodeFileSystemOperations - deleteFile with forward slashes", async () => { const fsOps = new NodeFileSystemOperations(tempDir); try { - await fsOps.write("test.txt", new TextEncoder().encode("test")); + await fsOps.write("test.txt", { + content: new TextEncoder().encode("test") + }); assert.equal(await fsOps.exists("test.txt"), true); await fsOps.deleteFile("test.txt"); @@ -74,15 +84,18 @@ test("NodeFileSystemOperations - rename with forward slashes", async () => { try { const content = new TextEncoder().encode("test content"); - await fsOps.write("old.txt", content); + await fsOps.write("old.txt", { content }); await fsOps.rename("old.txt", "new.txt"); assert.equal(await fsOps.exists("old.txt"), false); assert.equal(await fsOps.exists("new.txt"), true); - const readContent = await fsOps.read("new.txt"); - assert.equal(new TextDecoder().decode(readContent), "test content"); + const readContent = await fsOps.readSnapshot("new.txt"); + assert.equal( + new TextDecoder().decode(readContent!.content), + "test content" + ); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } @@ -94,7 +107,7 @@ test("NodeFileSystemOperations - rename to nested path with forward slashes", as try { const content = new TextEncoder().encode("test content"); - await fsOps.write("old.txt", content); + await fsOps.write("old.txt", { content }); await fsOps.rename("old.txt", "dir1/dir2/new.txt"); @@ -105,42 +118,21 @@ test("NodeFileSystemOperations - rename to nested path with forward slashes", as } }); -test("NodeFileSystemOperations - getFileSize", async () => { +test("NodeFileSystemOperations - stat", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); try { const content = new TextEncoder().encode("Hello!"); - await fsOps.write("test.txt", content); + await fsOps.write("test.txt", { content }); - const size = await fsOps.getFileSize("test.txt"); + const size = (await fsOps.stat("test.txt"))!.size; assert.equal(size, content.length); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); -test("NodeFileSystemOperations - atomicUpdateText", async () => { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); - const fsOps = new NodeFileSystemOperations(tempDir); - - try { - await fsOps.write("test.txt", new TextEncoder().encode("Hello")); - - const result = await fsOps.atomicUpdateText("test.txt", (current) => ({ - text: current.text + " World", - cursors: [] - })); - - assert.equal(result, "Hello World"); - - const content = await fsOps.read("test.txt"); - assert.equal(new TextDecoder().decode(content), "Hello World"); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } -}); - test("NodeFileSystemOperations - handles paths with forward slashes on all platforms", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "vaultlink-test-")); const fsOps = new NodeFileSystemOperations(tempDir); @@ -150,11 +142,11 @@ test("NodeFileSystemOperations - handles paths with forward slashes on all platf const testPath = "deep/nested/directory/file.txt"; const content = new TextEncoder().encode("test"); - await fsOps.write(testPath, content); + await fsOps.write(testPath, { content }); assert.equal(await fsOps.exists(testPath), true); - const readContent = await fsOps.read(testPath); - assert.equal(new TextDecoder().decode(readContent), "test"); + const readContent = await fsOps.readSnapshot(testPath); + assert.equal(new TextDecoder().decode(readContent!.content), "test"); await fsOps.deleteFile(testPath); assert.equal(await fsOps.exists(testPath), false); @@ -186,10 +178,9 @@ test("NodeFileSystemOperations - delete does not remove files", async () => { const fsOps = new NodeFileSystemOperations(tempDir); try { - await fsOps.write( - "parent/file.txt", - new TextEncoder().encode("preserve me") - ); + await fsOps.write("parent/file.txt", { + content: new TextEncoder().encode("preserve me") + }); await assert.rejects(async () => fsOps.delete("parent")); assert.equal(await fsOps.exists("parent/file.txt"), true); diff --git a/frontend/local-client-cli/src/node-filesystem.ts b/frontend/local-client-cli/src/node-filesystem.ts index 5902db8a..dadc8c5f 100644 --- a/frontend/local-client-cli/src/node-filesystem.ts +++ b/frontend/local-client-cli/src/node-filesystem.ts @@ -1,239 +1,188 @@ -import * as fs from "fs/promises"; -import type { Dirent } from "fs"; -import * as path from "path"; -import { randomUUID } from "crypto"; -import type { - FileSystemOperations, - RelativePath, - TextWithCursors -} from "sync-client"; -import { toUnixPath } from "./path-utils"; +import assert from "node:assert/strict"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { constants, type Stats } from "node:fs"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +function hasCode(error: unknown, ...codes: string[]): boolean { + return ( + error instanceof Error && + "code" in error && + typeof error.code === "string" && + codes.includes(error.code) + ); +} -// VaultLink's per-vault metadata directory. Holds the persisted sync database -// and the tmp files atomicWrite renames into place; the matching `${VAULTLINK_DIR}/**` -// ignore pattern keeps everything in here invisible to the file watcher. export const VAULTLINK_DIR = ".vaultlink"; +/** Preflights reject unsafe paths and links. This is not a sandbox against a + * hostile process concurrently replacing ancestor directories. */ export class NodeFileSystemOperations implements FileSystemOperations { - public constructor(private readonly basePath: string) {} - - public async listFilesRecursively( - directory: RelativePath | undefined - ): Promise { - const files: RelativePath[] = []; - await this.walkDirectory(directory ?? "", files); - return files; + public constructor(private readonly root: string) {} + public async stat( + relative: string + ): ReturnType { + const info = await this.info(await this.resolve(relative)); + return info + ? { + kind: info.isFile() + ? ("file" as const) + : ("directory" as const), + size: info.size + } + : undefined; } - - public async read(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - return await fs.readFile(fullPath); - } catch (error) { - throw new Error( - `Failed to read file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } + public async exists(relative: string): Promise { + return (await this.stat(relative)) !== undefined; } - - public async write( - relativePath: RelativePath, - content: Uint8Array - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - const dir = path.dirname(fullPath); - - try { - await fs.mkdir(dir, { recursive: true }); - await this.atomicWrite(fullPath, content); - } catch (error) { - throw new Error( - `Failed to write file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async atomicUpdateText( - relativePath: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - - try { - const currentContent = await fs.readFile(fullPath, "utf-8"); - const result = updater({ text: currentContent, cursors: [] }); - await this.atomicWrite(fullPath, result.text, "utf-8"); - return result.text; - } catch (error) { - throw new Error( - `Failed to atomically update file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async getFileSize(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - const stats = await fs.stat(fullPath); - return stats.size; - } catch (error) { - throw new Error( - `Failed to get file size for ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async exists(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.access(fullPath); - return true; - } catch { - return false; - } - } - - public async createDirectory(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - await fs.mkdir(fullPath, { recursive: false }); - } catch (error) { - throw new Error( - `Failed to create directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async delete(relativePath: RelativePath): Promise { - if (!relativePath) { - throw new Error("Cannot delete the vault root"); - } - - const fullPath = path.join(this.basePath, relativePath); - try { - await this.deleteDirectoryTree(fullPath); - } catch (error) { - throw new Error( - `Failed to delete directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async deleteFile(relativePath: RelativePath): Promise { - const fullPath = path.join(this.basePath, relativePath); - try { - const entry = await fs.lstat(fullPath).catch((error: unknown) => { - if (this.isMissing(error)) return undefined; + public async readSnapshot( + relative: string + ): Promise { + const absolute = await this.resolve(relative); + const handle = await fs + .open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW) + .catch((error: unknown) => { + if (hasCode(error, "ENOENT", "ENOTDIR")) return undefined; throw error; }); - if (!entry) return; - if (!entry.isFile() || entry.nlink !== 1) { - throw new Error("Cannot unlink a non-regular or linked file"); - } - - await fs.unlink(fullPath).catch((error: unknown) => { - if (!this.isMissing(error)) throw error; - }); - } catch (error) { - throw new Error( - `Failed to delete file ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); - } - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - const oldFullPath = path.join(this.basePath, oldPath); - const newFullPath = path.join(this.basePath, newPath); - const newDir = path.dirname(newFullPath); - + if (!handle) return undefined; try { - await fs.mkdir(newDir, { recursive: true }); - await fs.rename(oldFullPath, newFullPath); - } catch (error) { - throw new Error( - `Failed to rename file from ${oldFullPath} to ${newFullPath}: ${error instanceof Error ? error.message : String(error)}` + const before = await handle.stat({ bigint: true }); + assert( + before.isFile() && before.nlink === 1n, + "Not an independent regular file" ); + const content = new Uint8Array(await handle.readFile()); + const after = await handle.stat({ bigint: true }); + const atPath = await fs.lstat(absolute, { bigint: true }); + assert( + before.ino === after.ino && + after.ino === atPath.ino && + before.size === after.size && + before.mtimeNs === after.mtimeNs && + before.ctimeNs === after.ctimeNs, + "File changed during read" + ); + return { content }; + } finally { + await handle.close(); } } - - private async atomicWrite( - fullPath: string, - content: Uint8Array | string, - encoding?: BufferEncoding - ): Promise { - const tmpDir = path.join(this.basePath, VAULTLINK_DIR); - await fs.mkdir(tmpDir, { recursive: true }); - const tmpPath = path.join(tmpDir, `atomic-write-${randomUUID()}.tmp`); - try { - await fs.writeFile(tmpPath, content, encoding); - await fs.rename(tmpPath, fullPath); - } catch (error) { - await fs.unlink(tmpPath).catch(() => undefined); - throw error; + public async listFilesRecursively(root = ""): Promise { + const result: string[] = []; + for (const name of await fs.readdir(await this.resolve(root))) { + const relative = root ? `${root}/${name}` : name; + const entry = await this.stat(relative); + assert(entry, `File changed during scan: ${relative}`); + if (entry.kind === "directory") + result.push(...(await this.listFilesRecursively(relative))); + else result.push(relative); } + return result.sort(); } - - private async deleteDirectoryTree(fullPath: string): Promise { - const entry = await fs.lstat(fullPath).catch((error: unknown) => { - if (this.isMissing(error)) return undefined; - throw error; - }); - if (!entry) return; - if (!entry.isDirectory()) { - throw new Error("Cannot delete a regular file as a directory"); - } - - for (const child of await fs.readdir(fullPath, { - withFileTypes: true - })) { - if (!child.isDirectory()) { - throw new Error( - `Directory contains a non-directory entry: ${path.join(fullPath, child.name)}` - ); + public async createDirectory(relative: string): Promise { + await this.resolve(relative); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + const next = path.join(current, component); + try { + await fs.mkdir(next); + } catch (error) { + if (!hasCode(error, "EEXIST")) throw error; } - await this.deleteDirectoryTree(path.join(fullPath, child.name)); + assert((await fs.lstat(next)).isDirectory()); + current = next; } - - await fs.rmdir(fullPath); } - - private isMissing(error: unknown): boolean { - return ( - typeof error === "object" && - error !== null && - "code" in error && - error.code === "ENOENT" + public async write( + relative: string, + snapshot: FileSnapshot + ): Promise { + const destination = await this.resolve(relative); + await this.createDirectory( + path.posix.dirname(relative) === "." + ? "" + : path.posix.dirname(relative) ); + await fs.writeFile(destination, snapshot.content, { + flag: "wx", + mode: 0o600 + }); } - - private async walkDirectory( - relativePath: string, - files: RelativePath[] - ): Promise { - const fullPath = path.join(this.basePath, relativePath); - let entries: Dirent[] = []; - - try { - entries = await fs.readdir(fullPath, { withFileTypes: true }); - } catch (error) { - throw new Error( - `Failed to read directory ${fullPath}: ${error instanceof Error ? error.message : String(error)}` - ); + public async rename(from: string, to: string): Promise { + const snapshot = await this.readSnapshot(from); + assert(snapshot, `Missing source: ${from}`); + // Node's rename overwrites destinations on POSIX. Exclusive copy followed + // by unlink leaves ordinary discoverable files if interrupted. + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + assert( + current && + Buffer.from(current.content).equals( + Buffer.from(snapshot.content) + ), + `Source changed during move: ${from}` + ); + await this.deleteFile(from); + } + public async delete(relative: string): Promise { + assert(relative, "Cannot prune root"); + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isDirectory(), "Cannot delete regular file"); + for (const child of await fs.readdir(absolute)) + await this.delete(`${relative}/${child}`); + await fs.rmdir(absolute); + } + public async deleteFile(relative: string): Promise { + const absolute = await this.resolve(relative); + const entry = await this.info(absolute); + if (!entry) return; + assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); + await fs.unlink(absolute).catch((error: unknown) => { + if (!hasCode(error, "ENOENT")) throw error; + }); + } + private async resolve(relative: string): Promise { + assert( + relative === "" || + relative + .split("/") + .every( + (p) => + p !== "" && + p !== "." && + p !== ".." && + !p.includes("\\") && + !p.includes("\0") && + (process.platform !== "win32" || !p.includes(":")) + ), + "Unsafe path" + ); + assert( + (await fs.lstat(this.root)).isDirectory(), + "Vault root must be a directory, not a symlink" + ); + let current = this.root; + for (const component of relative.split("/").filter(Boolean)) { + current = path.join(current, component); + const entry = await this.info(current); + if (entry) + assert( + entry.isDirectory() || + (entry.isFile() && entry.nlink === 1), + "Symlink, hardlink or special file" + ); } - - for (const entry of entries) { - const entryName = entry.name; - const entryRelativePath = path.join(relativePath, entryName); - - if (entry.isDirectory()) { - await this.walkDirectory(entryRelativePath, files); - } else if (entry.isFile()) { - // Always return forward slashes - files.push(toUnixPath(entryRelativePath)); - } + return current; + } + private async info(absolute: string): Promise { + try { + return await fs.lstat(absolute); + } catch (error) { + if (hasCode(error, "ENOENT", "ENOTDIR")) return undefined; + throw error; } } } diff --git a/frontend/local-client-cli/webpack.config.js b/frontend/local-client-cli/webpack.config.js index 9226b9dc..d96048e0 100644 --- a/frontend/local-client-cli/webpack.config.js +++ b/frontend/local-client-cli/webpack.config.js @@ -7,6 +7,7 @@ module.exports = { healthcheck: "./src/healthcheck.ts" }, target: "node", + externals: { bufferutil: "bufferutil", "utf-8-validate": "utf-8-validate" }, mode: "production", optimization: { minimize: false diff --git a/frontend/obsidian-plugin/README.md b/frontend/obsidian-plugin/README.md index 68e10a83..de3f3027 100644 --- a/frontend/obsidian-plugin/README.md +++ b/frontend/obsidian-plugin/README.md @@ -1,63 +1,33 @@ -# Obsidian Sample Plugin +# VaultLink Obsidian plugin -This is a sample plugin for Obsidian (https://obsidian.md). +Syncs an Obsidian vault with the VaultLink server, including active editor text, +selections, remote cursors, history and connection status. Use an API v5 server. +The vault configuration directory, Git metadata and trash are ignored by default. -This project uses TypeScript to provide type checking and documentation. -The repo depends on the latest plugin API (obsidian.d.ts) in TypeScript Definition format, which contains TSDoc comments describing what it does. +From `frontend`: -**Note:** The Obsidian API is still in early alpha and is subject to change at any time! +```sh +npm run build -w sync-client -w vault-link-obsidian-plugin +npm run test -w vault-link-obsidian-plugin +``` -This sample plugin demonstrates some of the basic functionality the plugin API can do. +The production bundle is in `obsidian-plugin/dist`: `main.js`, `manifest.json` +and `styles.css`. Copy those files into the vault's plugin directory to install. -- Adds a ribbon icon, which shows a Notice when clicked. -- Adds a command "Open Sample Modal" which opens a Modal. -- Adds a plugin setting tab to the settings page. -- Registers a global click event and output 'click' to the console. -- Registers a global interval which logs 'setInterval' to the console. +Desktop Obsidian shares the CLI's Node filesystem and atomic metadata adapters. +The complete client record remains in the plugin's `data.json`; it includes +settings, sync state, queued logical notifications and the server history +checkpoint. Mobile uses Obsidian's storage and plugin data APIs. Because mobile +writes overwrite existing files, exclusive creation uses a disposable input for +`DataAdapter.copy` in `.vault-link-sync`. This is not a filesystem journal or a +recovery payload. -## First time developing plugins? +Raw create/modify/delete events wake a fresh scan, including events caused by +sync and editor replacements. Obsidian's logical rename event preserves file +and directory move identities. Editor snapshots include unsaved text and +selections; content application updates both disk and the active editor. -Quick starting guide for new plugin devs: - -- Check if [someone already developed a plugin for what you want](https://obsidian.md/plugins)! There might be an existing plugin similar enough that you can partner up with. -- Make a copy of this repo as a template with the "Use this template" button (login to GitHub if you don't see it). -- Clone your repo to a local development folder. For convenience, you can place this folder in your `.obsidian/plugins/your-plugin-name` folder. -- Install NodeJS, then run `npm i` in the command line under your repo folder. -- Run `npm run dev` to compile your plugin from `main.ts` to `main.js`. -- Make changes to `main.ts` (or create new `.ts` files). Those changes should be automatically compiled into `main.js`. -- Reload Obsidian to load the new version of your plugin. -- Enable plugin in settings window. -- For updates to the Obsidian API run `npm update` in the command line under your repo folder. - -## Releasing new releases - -- Update your `manifest.json` with your new version number, such as `1.0.1`, and the minimum Obsidian version required for your latest release. -- Update your `versions.json` file with `"new-plugin-version": "minimum-obsidian-version"` so older versions of Obsidian can download an older version of your plugin that's compatible. -- Create new GitHub release using your new version number as the "Tag version". Use the exact version number, don't include a prefix `v`. See here for an example: https://github.com/obsidianmd/obsidian-sample-plugin/releases -- Upload the files `manifest.json`, `main.js`, `styles.css` as binary attachments. Note: The manifest.json file must be in two places, first the root path of your repository and also in the release. -- Publish the release. - -> You can simplify the version bump process by running `npm version patch`, `npm version minor` or `npm version major` after updating `minAppVersion` manually in `manifest.json`. -> The command will bump version in `manifest.json` and `package.json`, and add the entry for the new version to `versions.json` - -## Adding your plugin to the community plugin list - -- Check the [plugin guidelines](https://docs.obsidian.md/Plugins/Releasing/Plugin+guidelines). -- Publish an initial version. -- Make sure you have a `README.md` file in the root of your repo. -- Make a pull request at https://github.com/obsidianmd/obsidian-releases to add your plugin. - -## How to use - -- Clone this repo. -- Make sure your NodeJS is at least v16 (`node --version`). -- `npm i` or `yarn` to install dependencies. -- `npm run dev` to start compilation in watch mode. - -## Manually installing the plugin - -- Copy over `main.js`, `styles.css`, `manifest.json` to your vault `VaultFolder/.obsidian/plugins/your-plugin-id/`. - -## API Documentation - -See https://github.com/obsidianmd/obsidian-api +Adapter tests cover unsaved text, cursor restoration, exclusive creation and +storage failures. The repository E2E runner builds the plugin and tests its +filesystem adapter against the server alongside real CLI processes. These tests +do not launch Obsidian itself. diff --git a/frontend/obsidian-plugin/package.json b/frontend/obsidian-plugin/package.json index d24e537b..f8d510eb 100644 --- a/frontend/obsidian-plugin/package.json +++ b/frontend/obsidian-plugin/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "webpack watch --mode development", "build": "webpack --mode production", - "test": "echo \"no tests defined\" && exit 0", + "test": "tsx --test 'src/**/*.test.ts'", "version": "node version-bump.mjs" }, "keywords": [], diff --git a/frontend/obsidian-plugin/src/mobile-file-system.ts b/frontend/obsidian-plugin/src/mobile-file-system.ts new file mode 100644 index 00000000..43869bcd --- /dev/null +++ b/frontend/obsidian-plugin/src/mobile-file-system.ts @@ -0,0 +1,130 @@ +import type { DataAdapter } from "obsidian"; +import type { FileSystemOperations, FileSnapshot } from "sync-client"; + +/** Mobile storage has no Node filesystem. DataAdapter.copy provides its + * documented exclusive-create primitive; writeBinary overwrites existing files. */ +export class MobileFileSystemOperations implements FileSystemOperations { + public constructor(private readonly adapter: DataAdapter) {} + + public async stat(path: string): ReturnType { + const entry = await this.adapter.stat(this.check(path)); + return entry + ? { + kind: + entry.type === "file" + ? ("file" as const) + : ("directory" as const), + size: entry.size + } + : undefined; + } + public async exists(path: string): Promise { + return (await this.stat(path)) !== undefined; + } + public async listFilesRecursively(root = ""): Promise { + const entries = await this.adapter.list(this.check(root)); + const files = [...entries.files]; + for (const folder of entries.folders) + files.push(...(await this.listFilesRecursively(folder))); + return files; + } + public async readSnapshot(path: string): Promise { + this.check(path); + const before = await this.adapter.stat(path); + if (!before) return undefined; + if (before.type !== "file") throw new Error(`Not a file: ${path}`); + const content = new Uint8Array(await this.adapter.readBinary(path)); + const after = await this.adapter.stat(path); + if ( + !after || + before.size !== after.size || + before.mtime !== after.mtime || + before.ctime !== after.ctime + ) + throw new Error(`File changed during read: ${path}`); + return { content }; + } + public async createDirectory(path: string): Promise { + this.check(path); + let current = ""; + for (const part of path.split("/").filter(Boolean)) { + current = current ? `${current}/${part}` : part; + if (!(await this.exists(current))) { + try { + await this.adapter.mkdir(current); + } catch (error) { + if ((await this.stat(current))?.kind !== "directory") + throw error; + } + } + if ((await this.stat(current))?.kind !== "directory") + throw new Error(`Not a directory: ${current}`); + } + } + public async write(path: string, snapshot: FileSnapshot): Promise { + this.check(path); + await this.createDirectory(path.split("/").slice(0, -1).join("/")); + await this.createDirectory(".vault-link-sync"); + const temporary = `.vault-link-sync/${crypto.randomUUID()}.tmp`; + try { + await this.adapter.writeBinary( + temporary, + new Uint8Array(snapshot.content).buffer + ); + await this.adapter.copy(temporary, path); + } finally { + if (await this.adapter.exists(temporary)) + await this.adapter.remove(temporary); + } + } + public async rename(from: string, to: string): Promise { + this.check(from); + this.check(to); + const snapshot = await this.readSnapshot(from); + if (!snapshot) throw new Error(`Missing source: ${from}`); + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + if ( + !current || + current.content.length !== snapshot.content.length || + current.content.some((byte, i) => byte !== snapshot.content[i]) + ) + throw new Error(`Source changed during move: ${from}`); + await this.deleteFile(from); + } + public async deleteFile(path: string): Promise { + const entry = await this.stat(path); + if (!entry) return; + if (entry.kind !== "file") throw new Error(`Not a file: ${path}`); + await this.adapter.remove(path); + } + public async delete(path: string): Promise { + if (!path) throw new Error("Cannot prune root"); + const entry = await this.stat(path); + if (!entry) return; + if (entry.kind !== "directory") + throw new Error(`Not a directory: ${path}`); + const children = await this.adapter.list(path); + if (children.files.length) + throw new Error(`Directory contains files: ${path}`); + for (const folder of children.folders) await this.delete(folder); + await this.adapter.rmdir(path, false); + } + private check(path: string): string { + if ( + path !== "" && + path + .split("/") + .some( + (part) => + !part || + part === "." || + part === ".." || + part.includes("\\") || + part.includes("\0") + ) + ) + throw new Error(`Unsafe path: ${path}`); + return path; + } +} diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.test.ts b/frontend/obsidian-plugin/src/obsidian-file-system.test.ts new file mode 100644 index 00000000..5faf5330 --- /dev/null +++ b/frontend/obsidian-plugin/src/obsidian-file-system.test.ts @@ -0,0 +1,167 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import * as fs from "node:fs/promises"; +import { constants } from "node:fs"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import type { DataAdapter, Editor, EditorSelection } from "obsidian"; +import { NodeFileSystemOperations } from "../../local-client-cli/src/node-filesystem"; +import { ObsidianFileSystemOperations } from "./obsidian-file-system"; +import { MobileFileSystemOperations } from "./mobile-file-system"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const text = (content: Uint8Array) => new TextDecoder().decode(content); + +test("Obsidian snapshots include unsaved editor text and restore merged selections", async (t) => { + const root = await fs.mkdtemp(path.join(tmpdir(), "vaultlink-editor-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const disk = new NodeFileSystemOperations(root); + await disk.write("note.md", { content: bytes("saved") }); + let value = "unsaved\ntext"; + let selections: EditorSelection[] = [ + { anchor: { line: 0, ch: 2 }, head: { line: 1, ch: 3 } } + ]; + const editor = { + getValue: () => value, + listSelections: () => selections, + setValue: (next: string) => { + value = next; + }, + setSelections: (next: EditorSelection[]) => { + selections = next; + } + } as Editor; + const adapter = new ObsidianFileSystemOperations(disk, () => ({ + path: "note.md", + editor + })); + const snapshot = await adapter.readSnapshot("note.md"); + assert.equal(text(snapshot!.content), value); + assert.deepEqual(snapshot!.cursors, [ + { id: 0, position: 2 }, + { id: 1, position: 11 } + ]); + assert.equal((await adapter.stat("note.md"))!.size, bytes(value).length); + await assert.rejects( + adapter.write("note.md", { content: bytes("collision") }) + ); + assert.equal(value, "unsaved\ntext"); + await adapter.deleteFile("note.md"); + await adapter.write("note.md", { + content: bytes("new\ntext"), + cursors: [ + { id: 1, position: 7 }, + { id: 0, position: 1 } + ] + }); + assert.equal(value, "new\ntext"); + assert.equal(text((await disk.readSnapshot("note.md"))!.content), value); + assert.deepEqual(selections, [ + { anchor: { line: 0, ch: 1 }, head: { line: 1, ch: 3 } } + ]); + value = "edited before rename"; + selections = [{ anchor: { line: 0, ch: 1 }, head: { line: 0, ch: 4 } }]; + await adapter.rename("note.md", "moved.md"); + assert.equal(text((await disk.readSnapshot("moved.md"))!.content), value); + assert.equal(await adapter.readSnapshot("note.md"), undefined); +}); + +// A storage host with Obsidian's documented semantics: writes replace, copies +// reject existing destinations. No runtime Obsidian module is needed in Node. +async function mobile(root: string) { + const disk = new NodeFileSystemOperations(root); + const host = { + stat: async (p: string) => { + const entry = await disk.stat(p); + return entry + ? { + type: entry.kind === "file" ? "file" : "folder", + size: entry.size, + mtime: 1, + ctime: 1 + } + : null; + }, + exists: (p: string) => disk.exists(p), + readBinary: async (p: string) => + new Uint8Array((await disk.readSnapshot(p))!.content).buffer, + writeBinary: (p: string, content: ArrayBuffer) => + fs.writeFile(path.join(root, p), new Uint8Array(content)), + copy: (from: string, to: string) => + fs.copyFile( + path.join(root, from), + path.join(root, to), + constants.COPYFILE_EXCL + ), + mkdir: (p: string) => disk.createDirectory(p), + remove: (p: string) => disk.deleteFile(p), + rmdir: (p: string) => fs.rmdir(path.join(root, p)), + list: async (p: string) => { + const entries = await fs.readdir(path.join(root, p), { + withFileTypes: true + }); + return { + files: entries + .filter((e) => e.isFile()) + .map((e) => (p ? `${p}/${e.name}` : e.name)), + folders: entries + .filter((e) => e.isDirectory()) + .map((e) => (p ? `${p}/${e.name}` : e.name)) + }; + } + } as unknown as DataAdapter; + return { adapter: new MobileFileSystemOperations(host), host }; +} + +test("mobile writes use exclusive copies and never overwrite a competing writer", async (t) => { + const root = await fs.mkdtemp(path.join(tmpdir(), "vaultlink-mobile-")); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { adapter } = await mobile(root); + await adapter.createDirectory("folder/nested"); + const writes = await Promise.allSettled([ + adapter.write("folder/note.md", { content: bytes("first") }), + adapter.write("folder/note.md", { content: bytes("second") }) + ]); + assert.equal(writes.filter((w) => w.status === "fulfilled").length, 1); + const original = (await adapter.readSnapshot("folder/note.md"))!; + await adapter.write("destination.md", { content: bytes("occupied") }); + await assert.rejects(adapter.rename("folder/note.md", "destination.md")); + assert.deepEqual(await adapter.readSnapshot("folder/note.md"), original); + assert.equal( + text((await adapter.readSnapshot("destination.md"))!.content), + "occupied" + ); + await assert.rejects(adapter.delete("folder")); + await adapter.rename("folder/note.md", "moved.md"); + assert.deepEqual(await adapter.readSnapshot("moved.md"), original); + await adapter.delete("folder"); + assert.equal(await adapter.exists("folder"), false); + assert.deepEqual(await fs.readdir(path.join(root, ".vault-link-sync")), []); + await assert.rejects( + adapter.write("../escape", { content: bytes("bad") }), + /Unsafe/ + ); +}); + +test("mobile scans propagate storage errors and snapshots reject read races", async (t) => { + const root = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-mobile-read-") + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { adapter, host } = await mobile(root); + await adapter.write("note.md", { content: bytes("before") }); + const read = host.readBinary.bind(host); + t.mock.method(host, "readBinary", async (p: string) => { + const content = await read(p); + await fs.writeFile(path.join(root, p), "changed length"); + return content; + }); + await assert.rejects( + adapter.readSnapshot("note.md"), + /changed during read/ + ); + t.mock.method(host, "list", async () => { + throw new Error("storage unavailable"); + }); + await assert.rejects(adapter.listFilesRecursively(), /storage unavailable/); +}); diff --git a/frontend/obsidian-plugin/src/obsidian-file-system.ts b/frontend/obsidian-plugin/src/obsidian-file-system.ts index fec25c59..2788ebff 100644 --- a/frontend/obsidian-plugin/src/obsidian-file-system.ts +++ b/frontend/obsidian-plugin/src/obsidian-file-system.ts @@ -1,204 +1,105 @@ -import type { Stat, Vault, Workspace } from "obsidian"; -import { MarkdownView, normalizePath } from "obsidian"; -import type { CursorPosition, TextWithCursors } from "sync-client"; +import type { Editor } from "obsidian"; import { utils, type FileSystemOperations, - type RelativePath + type FileSnapshot } from "sync-client"; import { getSelectionsFromEditor } from "./views/cursors/get-selections-from-editor"; +/** Editor contents and selections are one synchronous snapshot. Disk namespace + * operations remain the responsibility of the platform adapter. */ export class ObsidianFileSystemOperations implements FileSystemOperations { public constructor( - private readonly vault: Vault, - private readonly workspace: Workspace + private readonly disk: FileSystemOperations, + private readonly activeEditor: () => + | { path: string; editor: Editor } + | undefined ) {} - public async listFilesRecursively( - root: RelativePath | undefined - ): Promise { - // Let's implement this by hand because vault.adapter.listAllFiles doesn't always return all files. - const allFiles = []; - const remainingFolders = [root ?? this.vault.getRoot().path]; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - while (true) { - const folder = remainingFolders.pop(); - if (folder == undefined) { - break; - } - - // This would be a very bad idea to sync as it would mess with - // the integrity of the sync database. - if (folder.endsWith(".obsidian/plugins/vault-link/data.json")) { - continue; - } - - const files = await this.vault.adapter.list(normalizePath(folder)); - allFiles.push(...files.files); - remainingFolders.push(...files.folders); - } - - return allFiles; + public async listFilesRecursively(root?: string): Promise { + return this.disk.listFilesRecursively(root); + } + public async exists(path: string): Promise { + return this.disk.exists(path); + } + public async createDirectory(path: string): Promise { + return this.disk.createDirectory(path); + } + public async delete(path: string): Promise { + return this.disk.delete(path); + } + public async deleteFile(path: string): Promise { + return this.disk.deleteFile(path); + } + public async rename(from: string, to: string): Promise { + const snapshot = await this.readSnapshot(from); + if (!snapshot) throw new Error(`Missing source: ${from}`); + await this.write(to, snapshot); + const current = await this.readSnapshot(from); + if ( + !current || + current.content.length !== snapshot.content.length || + current.content.some((byte, i) => byte !== snapshot.content[i]) + ) + throw new Error(`Source changed during move: ${from}`); + await this.deleteFile(from); } - public async read(path: RelativePath): Promise { - path = normalizePath(path); - const view = this.workspace.getActiveViewOfType(MarkdownView); - if (view?.file?.path === path) { - return new TextEncoder().encode(view.editor.getValue()); - } - - return new Uint8Array(await this.vault.adapter.readBinary(path)); + public async stat(path: string): ReturnType { + const entry = await this.disk.stat(path); + const view = this.activeEditor(); + if (entry?.kind === "file" && view?.path === path) + return { + ...entry, + size: new TextEncoder().encode(view.editor.getValue()) + .byteLength + }; + return entry; } - public async write(path: RelativePath, content: Uint8Array): Promise { - path = normalizePath(path); - - const view = this.workspace.getActiveViewOfType(MarkdownView); - if (view?.file?.path === path) { - const position = view.editor.getCursor(); - view.editor.setValue(new TextDecoder().decode(content)); - view.editor.setCursor(position); - return; + public async readSnapshot(path: string): Promise { + // Validate the disk path even when an editor is open. + const entry = await this.disk.stat(path); + if (!entry) return undefined; + const view = this.activeEditor(); + if (entry.kind === "file" && view?.path === path) { + return { + content: new TextEncoder().encode(view.editor.getValue()), + cursors: getSelectionsFromEditor(view.editor).flatMap( + ({ id, start, end }) => [ + { id: 2 * id, position: start }, + { id: 2 * id + 1, position: end } + ] + ) + }; } + return this.disk.readSnapshot(path); + } - return this.vault.adapter.writeBinary( - path, - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion - content.buffer as ArrayBuffer + public async write(path: string, snapshot: FileSnapshot): Promise { + await this.disk.write(path, snapshot); + const view = this.activeEditor(); + if (view?.path !== path) return; + const text = new TextDecoder().decode(snapshot.content); + view.editor.setValue(text); + const cursors = [...(snapshot.cursors ?? [])].sort( + (a, b) => a.id - b.id ); - } - - public async atomicUpdateText( - path: RelativePath, - updater: (current: TextWithCursors) => TextWithCursors - ): Promise { - path = normalizePath(path); - - const view = this.workspace.getActiveViewOfType(MarkdownView); - - if (view?.file?.path === path) { - const text = view.editor.getValue(); - - const cursors: CursorPosition[] = getSelectionsFromEditor( - view.editor - ).flatMap(({ id, start: anchor, end: head }) => [ - { - id: 2 * id, - position: anchor - }, - { - id: 2 * id + 1, - position: head - } - ]); - - const result = updater({ + const selections = []; + for (let i = 0; i + 1 < cursors.length; i += 2) { + const anchor = utils.positionToLineAndColumn( text, - cursors + cursors[i].position + ); + const head = utils.positionToLineAndColumn( + text, + cursors[i + 1].position + ); + selections.push({ + anchor: { line: anchor.line, ch: anchor.column }, + head: { line: head.line, ch: head.column } }); - - if (result.text === text) { - return text; - } - - view.editor.setValue(result.text); - - const selections = []; - for (let i = 0; i < result.cursors.length / 2; i++) { - const from = result.cursors[2 * i]; - const to = result.cursors[2 * i + 1]; - const { line: fromLine, column: fromColumn } = - utils.positionToLineAndColumn(result.text, from.position); - - const { line: toLine, column: toColumn } = - utils.positionToLineAndColumn(result.text, to.position); - - selections.push({ - anchor: { line: fromLine, ch: fromColumn }, - head: { line: toLine, ch: toColumn } - }); - } - view.editor.setSelections(selections); - - return result.text; } - - return this.vault.adapter.process( - path, - (text) => - updater({ - text, - cursors: [] - }).text - ); - } - - public async getFileSize(path: RelativePath): Promise { - return (await this.statFile(path)).size; - } - - public async getModificationTime(path: RelativePath): Promise { - return new Date((await this.statFile(path)).mtime); - } - - public async exists(path: RelativePath): Promise { - return this.vault.adapter.exists(normalizePath(path)); - } - - public async createDirectory(path: RelativePath): Promise { - return this.vault.adapter.mkdir(normalizePath(path)); - } - - public async delete(path: RelativePath): Promise { - if (!path) { - throw new Error("Cannot delete the vault root"); - } - await this.deleteDirectoryTree(normalizePath(path)); - } - - public async deleteFile(path: RelativePath): Promise { - path = normalizePath(path); - const entry = await this.vault.adapter.stat(path); - if (!entry) return; - if (entry.type !== "file") { - throw new Error(`Cannot delete non-file: ${path}`); - } - await this.vault.adapter.remove(path); - } - - public async rename( - oldPath: RelativePath, - newPath: RelativePath - ): Promise { - return this.vault.adapter.rename(oldPath, newPath); - } - - private async statFile(path: string): Promise { - const file = await this.vault.adapter.stat(normalizePath(path)); - - if (!file) { - throw new Error(`File not found: ${path}`); - } - - return file; - } - - private async deleteDirectoryTree(path: string): Promise { - const entry = await this.vault.adapter.stat(path); - if (!entry) return; - if (entry.type !== "folder") { - throw new Error(`Cannot delete non-directory: ${path}`); - } - - const children = await this.vault.adapter.list(path); - if (children.files.length > 0) { - throw new Error(`Cannot delete non-empty directory: ${path}`); - } - for (const folder of children.folders) { - await this.deleteDirectoryTree(folder); - } - await this.vault.adapter.rmdir(path, false); + if (selections.length) view.editor.setSelections(selections); } } diff --git a/frontend/obsidian-plugin/src/vault-link-plugin.ts b/frontend/obsidian-plugin/src/vault-link-plugin.ts index 2c456b02..de4326f2 100644 --- a/frontend/obsidian-plugin/src/vault-link-plugin.ts +++ b/frontend/obsidian-plugin/src/vault-link-plugin.ts @@ -1,11 +1,16 @@ import type { - MarkdownView, Editor, MarkdownFileInfo, TAbstractFile, WorkspaceLeaf } from "obsidian"; -import { Notice, Plugin, TFile } from "obsidian"; +import { + Notice, + Plugin, + TFile, + MarkdownView, + FileSystemAdapter +} from "obsidian"; import "../manifest.json"; import { HistoryView } from "./views/history/history-view"; import { StatusBar } from "./views/status-bar/status-bar"; @@ -14,11 +19,14 @@ import { StatusDescription } from "./views/status-description/status-description import { SyncClient, rateLimit, - DEFAULT_SETTINGS, Logger, - debugging + debugging, + type FileSystemOperations, + type PersistenceProvider, + type StoredClient } from "sync-client"; import { ObsidianFileSystemOperations } from "./obsidian-file-system"; +import { MobileFileSystemOperations } from "./mobile-file-system"; import { SyncSettingsTab } from "./views/settings/settings-tab"; import { EditorStatusDisplayManager } from "./views/editor-status-display-manager/editor-status-display-manager"; import { remoteCursorsTheme } from "./views/cursors/remote-cursor-theme"; @@ -38,7 +46,7 @@ export default class VaultLinkPlugin extends Plugin { () => Promise >(); - private readonly syncClient: SyncClient | undefined; + private syncClient: SyncClient | undefined; private settingsTab: SyncSettingsTab | undefined; public async onload(): Promise { @@ -53,14 +61,19 @@ export default class VaultLinkPlugin extends Plugin { // eslint-disable-next-line (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = this; - const client = await this.createSyncClient(); + this.register(() => { + // eslint-disable-next-line + (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; + }); + const client = (this.syncClient = await this.createSyncClient()); this.registerObsidianExtensions(client); this.registerEditorEvents(client); this.register(async () => { - await client.waitUntilFinished(); + this.syncClient = undefined; + this.rateLimitedUpdatesPerFile.clear(); await client.destroy(); }); @@ -116,21 +129,55 @@ export default class VaultLinkPlugin extends Plugin { } private async createSyncClient(): Promise { - DEFAULT_SETTINGS.ignorePatterns.push( - ".obsidian/**", - ".git/**", - ".trash/**", - "**/.DS_Store" - ); - - const client = await SyncClient.create({ - fs: new ObsidianFileSystemOperations( - this.app.vault, - this.app.workspace - ), - persistence: { + const { vault, workspace } = this.app; + let disk: FileSystemOperations; + let persistence: PersistenceProvider; + if (vault.adapter instanceof FileSystemAdapter) { + // Keep Node modules unevaluated on mobile. + const { NodeFileSystemOperations } = + require("../../local-client-cli/src/node-filesystem") as typeof import("../../local-client-cli/src/node-filesystem"); + const { ClientPersistence } = + require("../../local-client-cli/src/client-persistence") as typeof import("../../local-client-cli/src/client-persistence"); + const root = vault.adapter.getBasePath(); + disk = new NodeFileSystemOperations(root); + persistence = new ClientPersistence( + `${root}/${this.manifest.dir ?? `${vault.configDir}/plugins/${this.manifest.id}`}/data.json`, + {} + ); + } else { + disk = new MobileFileSystemOperations(vault.adapter); + persistence = { load: this.loadData.bind(this), save: this.saveData.bind(this) + }; + } + const client = await SyncClient.create({ + fs: new ObsidianFileSystemOperations(disk, () => { + const view = workspace.getActiveViewOfType(MarkdownView); + return view?.file + ? { path: view.file.path, editor: view.editor } + : undefined; + }), + persistence: { + load: async () => { + const stored = await persistence.load(); + return { + ...stored, + settings: { + ...stored?.settings, + ignorePatterns: [ + ...new Set([ + ...(stored?.settings?.ignorePatterns ?? []), + `${vault.configDir}/**`, + ".git/**", + ".trash/**", + "**/.DS_Store" + ]) + ] + } + }; + }, + save: (stored) => persistence.save(stored) }, ...(IS_DEBUG_BUILD ? { @@ -141,7 +188,7 @@ export default class VaultLinkPlugin extends Plugin { }); if (IS_DEBUG_BUILD) { - debugging.logToConsole(client.logger); + debugging.logToConsole(client); } return client; @@ -195,11 +242,6 @@ export default class VaultLinkPlugin extends Plugin { this.register(() => { editorStatusDisplayManager.dispose(); }); - - this.register(() => { - // eslint-disable-next-line - (globalThis as any).VAULT_LINK_RUNNING_INSTANCE = null; - }); } private addRibbonIcons(): void { @@ -232,7 +274,7 @@ export default class VaultLinkPlugin extends Plugin { ), this.app.vault.on("create", (file: TAbstractFile) => { if (file instanceof TFile) { - client.syncLocallyCreatedFile(file.path); + void this.rateLimitedUpdate(file.path, client); } }), this.app.vault.on("modify", async (file: TAbstractFile) => { @@ -241,17 +283,17 @@ export default class VaultLinkPlugin extends Plugin { } }), this.app.vault.on("delete", (file: TAbstractFile) => { - client.syncLocallyDeletedFile(file.path); + void this.rateLimitedUpdate(file.path, client); }), + // Vault.rename is a logical move (including folders). Engine file + // moves use copy/unlink, so they cannot generate this event. this.app.vault.on( "rename", (file: TAbstractFile, oldPath: string) => { - if (file instanceof TFile) { - client.syncLocallyUpdatedFile({ - oldPath, - relativePath: file.path - }); - } + void client.syncLocallyUpdatedFile({ + oldPath, + relativePath: file.path + }); } ) ].forEach((event) => { @@ -267,7 +309,8 @@ export default class VaultLinkPlugin extends Plugin { this.rateLimitedUpdatesPerFile.set( path, rateLimit(async () => { - client.syncLocallyUpdatedFile({ + if (this.syncClient !== client) return; + await client.syncLocallyUpdatedFile({ relativePath: path }); }, MIN_WAIT_BETWEEN_UPDATES_IN_MS) diff --git a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts index 409402a4..bfebfbf4 100644 --- a/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts +++ b/frontend/obsidian-plugin/src/views/cursors/file-explorer.ts @@ -36,7 +36,7 @@ export function renderCursorsInFileExplorer( (parent) => { cursors.forEach((cursor) => { cursor.documentsWithCursors.forEach((document) => { - if (document.relativePath.startsWith(key)) { + if (document.relative_path.startsWith(key)) { parent.appendChild( createSpan({ text: cursor.userName, diff --git a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts index 4200a72a..d6650dcb 100644 --- a/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts +++ b/frontend/obsidian-plugin/src/views/cursors/remote-cursors-plugin.ts @@ -61,7 +61,7 @@ export class RemoteCursorsPluginValue implements PluginValue { return clientCursors.flatMap((cursor) => cursor.cursors.map((span) => ({ name: client.userName, - path: cursor.relativePath, + path: cursor.relative_path, deviceId: client.deviceId, isOutdated: client.isOutdated, span: { ...span } diff --git a/frontend/obsidian-plugin/src/views/status-description/status-description.ts b/frontend/obsidian-plugin/src/views/status-description/status-description.ts index 6d8d74fe..53fea486 100644 --- a/frontend/obsidian-plugin/src/views/status-description/status-description.ts +++ b/frontend/obsidian-plugin/src/views/status-description/status-description.ts @@ -88,7 +88,7 @@ export class StatusDescription { text: ` and has indexed approximately ` }); container.createSpan({ - text: `${this.syncClient.syncedDocumentCount}`, + text: `${this.syncClient.documentCount}`, cls: "number" }); container.createSpan({ diff --git a/frontend/obsidian-plugin/tsconfig.json b/frontend/obsidian-plugin/tsconfig.json index 7ec2a9cd..dcad6394 100644 --- a/frontend/obsidian-plugin/tsconfig.json +++ b/frontend/obsidian-plugin/tsconfig.json @@ -4,6 +4,7 @@ "module": "ESNext", "target": "ES2023", "strict": true, + "skipLibCheck": true, "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, "lib": ["DOM", "ES2024"] diff --git a/frontend/obsidian-plugin/webpack.config.js b/frontend/obsidian-plugin/webpack.config.js index 12844fd7..0d287eb2 100644 --- a/frontend/obsidian-plugin/webpack.config.js +++ b/frontend/obsidian-plugin/webpack.config.js @@ -12,6 +12,11 @@ module.exports = (env, argv) => ({ ignored: "**/node_modules" }, externals: { + "node:assert/strict": "commonjs node:assert/strict", + "node:fs/promises": "commonjs node:fs/promises", + "node:fs": "commonjs node:fs", + "node:path": "commonjs node:path", + "node:crypto": "commonjs node:crypto", obsidian: "commonjs obsidian", electron: "commonjs electron", "@codemirror/autocomplete": "commonjs @codemirror/autocomplete", diff --git a/frontend/package.json b/frontend/package.json index 9e33d1b0..17a03fb3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,7 +20,7 @@ "test": "npm run test --workspaces", "check:harness": "tsc --noEmit -p test-support/tsconfig.json && tsc --noEmit -p deterministic-tests/tsconfig.json && tsc --noEmit -p test-client/tsconfig.json", "test:harness": "npm run check:harness && tsx --test 'test-support/harness.test.ts' 'test-support/oracle-regressions.test.ts' 'test-support/recovery.test.ts' 'test-support/file-application.test.ts' 'test-support/observer.test.ts' 'test-support/event-delivery.test.ts' 'test-support/polling.test.ts' 'test-support/engine-regressions.test.ts' 'test-support/sync-audit.test.ts' 'test-client/src/**/*.test.ts' '../scripts/e2e-runner.test.mjs'", - "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/restart-reconciliation.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts'", + "test:protocol": "tsx --test 'test-support/protocol.test.ts' 'test-support/restart-reconciliation.test.ts' 'test-support/content-limits.test.ts' 'test-support/server-restore.test.ts' 'test-support/protocol-audit.test.ts' 'test-support/real-disk.test.ts' 'test-support/production-adapters.test.ts'", "lint": "eslint --fix sync-client obsidian-plugin test-client deterministic-tests local-client-cli && prettier --write \"**/*.ts\"", "update": "ncu -u -ws" }, diff --git a/frontend/sync-client/README.md b/frontend/sync-client/README.md index c92671db..2373c040 100644 --- a/frontend/sync-client/README.md +++ b/frontend/sync-client/README.md @@ -9,7 +9,9 @@ resolving, including while offline. Shutdown waits for those saves. Content-upda notifications only wake reconciliation; scans read the latest bytes, including after restart. Notifications describe logical external changes: adapters suppress self-generated create/delete/move notifications, and report atomic editor saves -as content updates. This distinguishes a deliberate delete/recreate from an edit. +as content updates. This distinguishes a deliberate delete/recreate from an edit +when the host supplies reliable logical events. Raw OS watchers instead send +content-update hints for every event and let scans infer the current namespace. Use `waitUntilFinished()` to await the active attempt; errors leave pending requests saved and are reported to the caller. Transient failures retry; authentication and protocol errors wait for an explicit wake or settings change. @@ -82,7 +84,9 @@ Their failures are reported, and remaining observers continue receiving updates. ## Required adapter contract `FileSystemOperations` and `PersistenceProvider` define the adapter operations. -The Obsidian and CLI adapters still need to adopt the current core interface. +The CLI and desktop Obsidian share a Node filesystem adapter and atomic full-record +metadata persistence. Mobile Obsidian uses its host storage APIs. The E2E runner +builds and tests both integrations, including actual CLI processes. The sync client does not coordinate separate processes accessing the same vault. The filesystem adapter must provide complete scans, coherent snapshots, recursive @@ -92,8 +96,10 @@ file removal. Reject links, special files and paths escaping the vault. An unrea file or directory must fail the scan rather than look like a deletion. Optional editor cursor metadata is applied with file content. -User-file writes have no power-loss durability requirement. There is no filesystem -journal, hidden staging area, replay, backup archive, or flush protocol. Swaps and +User-file writes have no power-loss durability requirement. The engine has no +filesystem journal, recovery staging area, replay, backup archive, or flush protocol. +Mobile storage needs a disposable scratch file to use its exclusive-copy primitive; +that file is not used for recovery. Swaps and cycles temporarily move occupied files to visible conflict names; a fresh scan can discover them if application stops partway through. Partial writes and edits made before, during or after an interruption are treated as current disk content. diff --git a/frontend/sync-client/src/consts.ts b/frontend/sync-client/src/consts.ts index a649d067..9cdf64d2 100644 --- a/frontend/sync-client/src/consts.ts +++ b/frontend/sync-client/src/consts.ts @@ -1,5 +1,4 @@ export const TIMEOUT_FOR_MERGING_HISTORY_ENTRIES_IN_SECONDS = 60; -export const DIFF_CACHE_SIZE_MB = 2; export const MAX_LOG_MESSAGE_COUNT = 100000; export const MAX_HISTORY_ENTRY_COUNT = 5000; export const SUPPORTED_API_VERSION = 5; diff --git a/frontend/sync-client/src/index.ts b/frontend/sync-client/src/index.ts index e7476470..8d1db5be 100644 --- a/frontend/sync-client/src/index.ts +++ b/frontend/sync-client/src/index.ts @@ -36,7 +36,7 @@ export type { } from "./errors/errors"; export type { MaybeOutdatedClientCursors } from "./types/maybe-outdated-client-cursors"; export { DocumentSyncStatus } from "./types/document-sync-status"; -export { SyncClient } from "./sync-client"; +export { SyncClient, type StoredClient } from "./sync-client"; export type { TextWithCursors, CursorPosition } from "reconcile-text"; export const debugging = { diff --git a/frontend/sync-client/src/services/server-config.test.ts b/frontend/sync-client/src/services/server-config.test.ts new file mode 100644 index 00000000..d82f8f06 --- /dev/null +++ b/frontend/sync-client/src/services/server-config.test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { ServerConfig } from "./server-config"; +import { SyncService } from "./sync-service"; +import { Settings } from "../persistence/settings"; +import { Logger } from "../tracing/logger"; +import { SUPPORTED_API_VERSION } from "../consts"; + +const config = { + supportedApiVersion: SUPPORTED_API_VERSION, + isAuthenticated: true, + mergeableFileExtensions: ["md"], + serverVersion: "test" +}; +const make = (fetch: typeof globalThis.fetch) => + new ServerConfig( + new SyncService( + "device", + new Settings( + new Logger(), + { remoteUri: "http://test" }, + async () => {} + ), + fetch + ) + ); + +test("configuration shares one validated request and retries failures", async () => { + let calls = 0; + const server = make(async () => { + if (++calls === 1) throw new Error("offline"); + return Response.json(config); + }); + await assert.rejects(server.getConfig(), /offline/); + const [a, b] = await Promise.all([server.getConfig(), server.getConfig()]); + assert.equal(a, b); + assert.equal(calls, 2); + await server.getConfig(); + assert.equal(calls, 2); +}); + +test("connection checks are independent and validate authentication and version", async () => { + let response = config; + const server = make(async () => Response.json(response)); + const cached = await server.getConfig(); + response = { ...config, isAuthenticated: false }; + assert.equal((await server.checkConnection()).isSuccessful, false); + response = { ...config, supportedApiVersion: SUPPORTED_API_VERSION + 1 }; + assert.equal((await server.checkConnection()).isSuccessful, false); + assert.equal(await server.getConfig(), cached); + server.reset(); + await assert.rejects(server.getConfig(), /Unsupported API/); + response = config; + assert.deepEqual(await server.getConfig(), config); +}); + +for (const fail of [false, true]) { + test(`reset fences a late ${fail ? "failed" : "successful"} configuration request`, async () => { + const first = Promise.withResolvers(); + let calls = 0; + const server = make(async () => + ++calls === 1 ? first.promise : Response.json(config) + ); + const old = server.getConfig(); + server.reset(); + const current = await server.getConfig(); + if (fail) { + first.reject(new Error("old request")); + await assert.rejects(old, /old request/); + } else { + first.resolve( + Response.json({ ...config, mergeableFileExtensions: ["txt"] }) + ); + await old; + } + assert.equal(await server.getConfig(), current); + assert.equal(calls, 2); + }); +} diff --git a/frontend/sync-client/src/services/server-config.ts b/frontend/sync-client/src/services/server-config.ts index fcd8adb9..102cf425 100644 --- a/frontend/sync-client/src/services/server-config.ts +++ b/frontend/sync-client/src/services/server-config.ts @@ -4,7 +4,6 @@ import { ServerVersionMismatchError } from "../errors/errors"; import type { SyncService } from "./sync-service"; -import type { PingResponse } from "./types/PingResponse"; export interface ServerConfigData { mergeableFileExtensions: string[]; @@ -13,8 +12,7 @@ export interface ServerConfigData { } export class ServerConfig { - private response: Promise | undefined; - private config: ServerConfigData | undefined; + private config: Promise | undefined; public constructor(private readonly syncService: SyncService) {} @@ -41,29 +39,16 @@ export class ServerConfig { await this.getConfig(); } - public async checkConnection(forceUpdate = false): Promise<{ + public async checkConnection(): Promise<{ isSuccessful: boolean; message: string; }> { try { - let { response } = this; - if (!response || forceUpdate) { - response = this.response = this.syncService.ping(); - } - - const result: PingResponse = await response; // it must be defined, otherwise we would have thrown above - this.config = result; - - if (result.isAuthenticated) { - return { - isSuccessful: true, - message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` - }; - } - + const result = await this.syncService.ping(); + ServerConfig.validateConfig(result); return { - isSuccessful: false, - message: `Successfully connected to server (version: ${result.serverVersion}) but failed to authenticate` + isSuccessful: true, + message: `Successfully connected to server (version: ${result.serverVersion}) and authenticated` }; } catch (e) { return { @@ -75,22 +60,22 @@ export class ServerConfig { public async getConfig(): Promise { if (!this.config) { - this.response ??= this.syncService.ping(); - try { - this.config = await this.response; - } catch (error) { - this.response = undefined; - throw error; - } + const pending = this.syncService + .ping() + .then((config) => { + ServerConfig.validateConfig(config); + return config; + }) + .catch((error: unknown) => { + if (this.config === pending) this.config = undefined; + throw error; + }); + this.config = pending; } - - ServerConfig.validateConfig(this.config); - return this.config; } public reset(): void { - this.response = undefined; this.config = undefined; } } diff --git a/frontend/sync-client/src/services/sync-service.test.ts b/frontend/sync-client/src/services/sync-service.test.ts index d4f72b47..bf56fec4 100644 --- a/frontend/sync-client/src/services/sync-service.test.ts +++ b/frontend/sync-client/src/services/sync-service.test.ts @@ -144,3 +144,38 @@ test("pause waits for a checkpoint save already in progress", async () => { await assert.rejects(request, SyncResetError); assert.equal(checkpoint, "1:head"); }); + +test("history saves only new checkpoints, including a changed token at the same event ID", async () => { + let checkpoint: string | undefined; + let responseCheckpoint = "1:first"; + const saved: (string | undefined)[] = []; + const service = new SyncService( + "device", + settings(), + async () => + Response.json( + {}, + { headers: { "X-Vault-Link-History": responseCheckpoint } } + ), + { + get: () => checkpoint, + save: async (value) => { + checkpoint = value; + saved.push(value); + } + } + ); + service.resume(); + for (const value of [ + "1:first", + "1:first", + "2:next", + "1:first", + "2:replacement", + "2:replacement" + ]) { + responseCheckpoint = value; + await service.events(0); + } + assert.deepEqual(saved, ["1:first", "2:next", "2:replacement"]); +}); diff --git a/frontend/sync-client/src/services/sync-service.ts b/frontend/sync-client/src/services/sync-service.ts index cf918151..24bac853 100644 --- a/frontend/sync-client/src/services/sync-service.ts +++ b/frontend/sync-client/src/services/sync-service.ts @@ -131,6 +131,7 @@ export class SyncService { this.history !== undefined ) { const previous = this.history.get(); + if (checkpoint === previous) return; if ( previous === undefined || previous === "" || diff --git a/frontend/sync-client/src/sync-client.ts b/frontend/sync-client/src/sync-client.ts index 8660c0b1..7ea60cf1 100644 --- a/frontend/sync-client/src/sync-client.ts +++ b/frontend/sync-client/src/sync-client.ts @@ -28,7 +28,7 @@ import { type LocalChange } from "./sync-operations/local-changes"; -type StoredClient = Partial<{ +export type StoredClient = Partial<{ settings: Partial; database: Partial; localChanges: ( @@ -372,7 +372,7 @@ export class SyncClient { } public async checkConnection() { - const result = await this.serverConfig.checkConnection(true); + const result = await this.serverConfig.checkConnection(); return { isSuccessful: result.isSuccessful, serverMessage: result.message, diff --git a/frontend/sync-client/src/sync-operations/syncer.ts b/frontend/sync-client/src/sync-operations/syncer.ts index 8a123c04..47ba91d0 100644 --- a/frontend/sync-client/src/sync-operations/syncer.ts +++ b/frontend/sync-client/src/sync-operations/syncer.ts @@ -340,18 +340,17 @@ export class Syncer { if (this.database.state.pending) { await this.finishPending(); } - await this.scan(); const batch = await this.service.events( this.database.state.lastSeenUpdateId ); await this.incorporateEventBatch(batch); - await this.scan(); // Previously excluded content becomes eligible after a setting // change. Its saved obligation survives advancing the cursor. if (Object.keys(this.database.state.excluded ?? {}).length) await this.incorporateFileManifest( this.database.state.fileManifest ); + else await this.scan(); for (const head of Object.values( this.database.state.remoteHeads )) { @@ -521,7 +520,6 @@ export class Syncer { bootstrap: vaultSnapshot }); } - await this.scan(vaultSnapshot); await this.incorporateFileManifest( vaultSnapshot.fileManifest, vaultSnapshot.headEventId, @@ -531,6 +529,7 @@ export class Syncer { private async scan(initial?: VaultSnapshot): Promise { await this.flushLocalChanges(); + const hadChanges = this.changes.length > 0; this.unsyncablePaths = await scanLocalFiles( { next: this.next(), @@ -544,7 +543,8 @@ export class Syncer { initial ); this.hasScanned = true; - await this.localChangeStorage.save(structuredClone(this.changes)); + if (hadChanges) + await this.localChangeStorage.save(structuredClone(this.changes)); } private async incorporateEventBatch(batch: EventBatch): Promise { diff --git a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts index 51ad41c1..7429cdf8 100644 --- a/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts +++ b/frontend/sync-client/src/utils/data-structures/fix-sized-cache.ts @@ -1,66 +1,32 @@ -// Implements an in-memory fixed-size cache for document contents, - import type { VaultUpdateId } from "../../persistence/database"; -// Doubly-linked list node for O(1) LRU operations -class LRUNode { - public constructor( - public key: VaultUpdateId, - public value: Uint8Array, - public prev: LRUNode | null = null, - public next: LRUNode | null = null - ) {} -} - -// evicting the least recently used documents when the size limit is exceeded. +/** Map insertion order tracks the least recently used document first. */ export class FixedSizeDocumentCache { - private currentSizeInBytes: number; - private readonly cache: Map; - private head: LRUNode | null; // Least recently used - private tail: LRUNode | null; // Most recently used + private currentSizeInBytes = 0; + private readonly cache = new Map(); - public constructor(private maxSizeInBytes: number) { - this.currentSizeInBytes = 0; - this.cache = new Map(); - this.head = null; - this.tail = null; - } + public constructor(private maxSizeInBytes: number) {} public get(updateId: VaultUpdateId): Uint8Array | undefined { - const node = this.cache.get(updateId); - if (node) { - this.moveToTail(node); - return node.value; + const content = this.cache.get(updateId); + if (content !== undefined) { + this.cache.delete(updateId); + this.cache.set(updateId, content); } - - return undefined; + return content; } public put(updateId: VaultUpdateId, content: Uint8Array): void { - if (content.byteLength > this.maxSizeInBytes) { - // Document is too large to fit in the cache - return; - } - - // If the document is already in the cache, update it - const existingNode = this.cache.get(updateId); - if (existingNode != null) { - this.currentSizeInBytes -= existingNode.value.byteLength; - this.removeNode(existingNode); - this.cache.delete(updateId); - } - - const newNode = new LRUNode(updateId, content); - this.cache.set(updateId, newNode); - this.addToTail(newNode); + if (content.byteLength > this.maxSizeInBytes) return; + this.currentSizeInBytes -= this.cache.get(updateId)?.byteLength ?? 0; + this.cache.delete(updateId); + this.cache.set(updateId, content); this.currentSizeInBytes += content.byteLength; this.fitBelowMaxSize(); } public reset(): void { this.cache.clear(); - this.head = null; - this.tail = null; this.currentSizeInBytes = 0; } @@ -70,50 +36,10 @@ export class FixedSizeDocumentCache { } private fitBelowMaxSize(): void { - // Evict least recently used documents if over size limit - while (this.currentSizeInBytes > this.maxSizeInBytes && this.head) { - const lruNode = this.head; - this.removeNode(lruNode); - this.cache.delete(lruNode.key); - this.currentSizeInBytes -= lruNode.value.byteLength; + for (const [id, content] of this.cache) { + if (this.currentSizeInBytes <= this.maxSizeInBytes) break; + this.cache.delete(id); + this.currentSizeInBytes -= content.byteLength; } } - - private removeNode(node: LRUNode): void { - if (node.prev) { - node.prev.next = node.next; - } else { - this.head = node.next; - } - - if (node.next) { - node.next.prev = node.prev; - } else { - this.tail = node.prev; - } - - node.prev = null; - node.next = null; - } - - private addToTail(node: LRUNode): void { - node.prev = this.tail; - node.next = null; - - if (this.tail) { - this.tail.next = node; - } - - this.tail = node; - - this.head ??= node; - } - - private moveToTail(node: LRUNode): void { - if (node === this.tail) { - return; - } - this.removeNode(node); - this.addToTail(node); - } } diff --git a/frontend/test-support/README.md b/frontend/test-support/README.md index 824714ab..6b5d3cb8 100644 --- a/frontend/test-support/README.md +++ b/frontend/test-support/README.md @@ -13,7 +13,8 @@ Test-only code shared by the deterministic and seeded clients. | `restart-reconciliation.test.ts` | Interrupted file application followed by user edits, two-client/server convergence and a second restart | | `server-restore.test.ts` | History checkpoints, server backup restores and interrupted metadata resets | | `content-limits.test.ts`, `protocol-audit.test.ts` | Content validation and API invariants | -| `real-disk.test.ts` | Host filesystem exclusive create/rename, unsafe links, and rename cycles | +| `real-disk.test.ts` | Production Node adapter exclusive create/move, unsafe links, and rename cycles | +| `production-adapters.test.ts` | Actual CLI processes, raw watchers, Obsidian adapter, complete metadata and restart after SIGKILL | The filesystem interruption tests assert the supported contract: complete metadata remains usable, scans observe the current files, and a second scan is stable. @@ -30,9 +31,12 @@ after replacement. Abandoned filesystem sessions are fenced against later writes `crash-worker.ts` runs a complete client in a worker, which the parent terminates at accepted-response checkpoints without graceful shutdown. A new runtime receives -the saved disk and metadata. The separate real-disk fixture uses a test-only native -exclusive-rename helper and ordinary writes without flushes. It is not a production -adapter or a sandbox against hostile concurrent changes to ancestor directories. +the saved disk and metadata. Real-disk tests exercise the production Node adapter. +The production integration suite launches built CLI processes and an Obsidian +filesystem adapter against a real server. Obsidian unit tests exercise unsaved editor +text, selections and mobile storage contracts; they do not launch the Obsidian app. +Filesystem path checks are not a sandbox against hostile concurrent changes to +ancestor directories. Scripted and seeded suites run multiple clients against real servers. They check visible bytes, UUID mappings, canonical server state, content heads and event cursors. @@ -46,7 +50,8 @@ Run from the repository root: E2E_WORKERS=32 E2E_SEED=12 E2E_TIMEOUT_SECONDS=9000 E2E_ITERATIONS=100 scripts/e2e.sh ``` -The command rebuilds peers, runs unit/harness/protocol/scripted suites, and then +The command rebuilds peers and both production integrations, runs core and adapter +unit tests plus harness/protocol/scripted suites, and then runs seeds 12 through 43. `E2E_ARTIFACTS` optionally selects the artifact directory. The runner tests also verify subprocess exit codes, log errors, spawn failures, timeouts, workspace dependency consistency and CI artifact wiring. diff --git a/frontend/test-support/engine-regressions.test.ts b/frontend/test-support/engine-regressions.test.ts index 81995ee6..f75aee62 100644 --- a/frontend/test-support/engine-regressions.test.ts +++ b/frontend/test-support/engine-regressions.test.ts @@ -3,7 +3,7 @@ import { PermanentSyncError } from "../sync-client/src/errors/errors"; import { test } from "node:test"; import { Database, emptyState } from "../sync-client/src/persistence/database"; import { Syncer } from "../sync-client/src/sync-operations/syncer"; -import { SyncClient } from "../sync-client/src/sync-client"; +import { SyncClient, type StoredClient } from "../sync-client/src/sync-client"; import { toStoredSnapshot } from "../sync-client/src/sync-operations/content"; import { allocatePortablePath } from "../sync-client/src/utils/portable-path"; import { MemoryDisk, MemoryPersistence } from "./storage"; @@ -83,7 +83,7 @@ test("legacy content notifications preserve the acknowledged namespace prefix on .hash }; initial.lastAppliedLocalChangeId = "consumed-update"; - const saved = { + const saved: StoredClient = { database: initial, settings: { remoteUri: "http://offline.test", vaultName: "test" }, localChangesVaultKey: initial.vaultKey, diff --git a/frontend/test-support/native-fs.c b/frontend/test-support/native-fs.c deleted file mode 100644 index 560c79c6..00000000 --- a/frontend/test-support/native-fs.c +++ /dev/null @@ -1,27 +0,0 @@ -/* Test-only POSIX primitives missing from Node's filesystem API. A preflight - * exists() followed by fs.rename() is not an exclusive rename. */ -#define _GNU_SOURCE -#include -#include -#include -#include -#if defined(__linux__) -#include -#endif - -int main(int argc, char **argv) { - if (argc == 4 && strcmp(argv[1], "rename") == 0) { - int result; -#if defined(__APPLE__) - result = renamex_np(argv[2], argv[3], RENAME_EXCL); -#elif defined(__linux__) - result = syscall(SYS_renameat2, AT_FDCWD, argv[2], AT_FDCWD, argv[3], 1 /* RENAME_NOREPLACE */); -#else -#error "The real filesystem tests require macOS or Linux" -#endif - if (result != 0) { perror("exclusive rename"); return 1; } - return 0; - } - fprintf(stderr, "usage: native-fs rename FROM TO\n"); - return 2; -} diff --git a/frontend/test-support/production-adapters.test.ts b/frontend/test-support/production-adapters.test.ts new file mode 100644 index 00000000..64832b8f --- /dev/null +++ b/frontend/test-support/production-adapters.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import { tmpdir } from "node:os"; +import { Logger, SyncClient } from "sync-client"; +import { ServerControl } from "../deterministic-tests/src/server-control"; +import { ClientPersistence } from "../local-client-cli/src/client-persistence"; +import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesystem"; +import { ObsidianFileSystemOperations } from "../obsidian-plugin/src/obsidian-file-system"; +import { getJson, type CanonicalSnapshot } from "./canonical"; + +const root = path.resolve(__dirname, "../.."); +const token = "test-token-change-me"; +const pause = () => new Promise((resolve) => setTimeout(resolve, 50)); +async function until(check: () => Promise, message: string) { + const deadline = Date.now() + 15_000; + while (!(await check())) { + assert(Date.now() < deadline, message); + await pause(); + } +} +async function contents(file: string) { + try { + return await fs.readFile(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return undefined; + throw error; + } +} +async function stop(child: ChildProcess, signal: NodeJS.Signals) { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); + child.kill(signal); + await exited; +} + +test( + "production CLI watchers, Obsidian adapter, and full metadata survive process replacement", + { timeout: 90_000 }, + async (t) => { + const directory = await fs.mkdtemp( + path.join(tmpdir(), "vaultlink-production-") + ); + const server = new ServerControl( + path.join(root, "sync-server/target/release/sync_server"), + path.join(root, "sync-server/config-e2e.yml"), + new Logger() + ); + const children: ChildProcess[] = []; + let obsidian: SyncClient | undefined; + t.after(async () => { + await obsidian?.destroy(); + await Promise.all(children.map((child) => stop(child, "SIGKILL"))); + await server.stop(); + await fs.rm(directory, { recursive: true, force: true }); + }); + await server.start(); + const a = path.join(directory, "a"), + b = path.join(directory, "b"), + c = path.join(directory, "obsidian"); + await Promise.all([a, b, c].map((p) => fs.mkdir(p))); + await fs.writeFile(path.join(a, "note.md"), "initial\n"); + async function launch(vault: string) { + const child = spawn( + process.execPath, + [ + path.join(root, "frontend/local-client-cli/dist/cli.js"), + "-l", + vault, + "-r", + server.remoteUri, + "-t", + token, + "-v", + "production", + "--ignore-pattern", + "**/*.tmp" + ], + { stdio: ["ignore", "pipe", "pipe"] } + ); + children.push(child); + let log = ""; + child.stdout!.on("data", (data: Buffer) => { + log += data.toString(); + }); + child.stderr!.on("data", (data: Buffer) => { + log += data.toString(); + }); + await until(async () => { + assert( + child.exitCode === null && child.signalCode === null, + `CLI exited: ${log}` + ); + return log.includes("File watcher started"); + }, `CLI did not start: ${log}`); + return child; + } + let first = await launch(a); + await launch(b); + await until( + async () => + (await contents(path.join(b, "note.md"))) === "initial\n", + "CLI initial download failed" + ); + const persistence = (vault: string) => + new ClientPersistence( + path.join(vault, ".vaultlink/sync-data.json"), + {} + ); + const original = (await persistence(a).load()).database!; + const [documentId] = Object.keys(original.local!); + assert(documentId); + + const settings = { + remoteUri: server.remoteUri, + token, + vaultName: "production", + isSyncEnabled: true, + enableTelemetry: false, + ignorePatterns: [".vaultlink/**"], + syncIntervalMs: 0 + }; + obsidian = await SyncClient.create({ + fs: new ObsidianFileSystemOperations( + new NodeFileSystemOperations(c), + () => undefined + ), + persistence: new ClientPersistence( + path.join(c, ".vaultlink/sync-data.json"), + settings + ) + }); + await obsidian.start(); + await obsidian.waitUntilFinished(); + assert.equal(await contents(path.join(c, "note.md")), "initial\n"); + await fs.writeFile( + path.join(a, "replacement.tmp"), + "atomic editor save\n" + ); + await fs.rename( + path.join(a, "replacement.tmp"), + path.join(a, "note.md") + ); + await until( + async () => + (await contents(path.join(b, "note.md"))) === + "atomic editor save\n" && + (await contents(path.join(c, "note.md"))) === + "atomic editor save\n", + "Editor replacement did not converge" + ); + assert.equal( + (await persistence(a).load()).database!.local![documentId], + "note.md" + ); + await fs.rename(path.join(b, "note.md"), path.join(b, "renamed.md")); + await until( + async () => + (await contents(path.join(a, "renamed.md"))) === + "atomic editor save\n" && + (await contents(path.join(a, "note.md"))) === undefined, + "CLI rename did not converge" + ); + await until( + async () => + (await persistence(a).load()).database!.local![documentId] === + "renamed.md", + "Rename changed document identity" + ); + const saved = await persistence(a).load(); + assert( + saved.historyCheckpoint?.checkpoint, + "CLI dropped the history checkpoint" + ); + await stop(first, "SIGKILL"); + await fs.writeFile(path.join(a, "renamed.md"), "edited after kill\n"); + first = await launch(a); + await until( + async () => + (await contents(path.join(b, "renamed.md"))) === + "edited after kill\n" && + (await contents(path.join(c, "renamed.md"))) === + "edited after kill\n", + "Restart failed to reconcile current files" + ); + const snapshot = await getJson( + `${server.remoteUri}/vaults/production/vault-snapshot`, + token + ); + assert.deepEqual(snapshot.fileManifest.entries, { + [documentId]: "renamed.md" + }); + await fs.unlink(path.join(b, "renamed.md")); + await until( + async () => + (await contents(path.join(a, "renamed.md"))) === undefined && + (await contents(path.join(c, "renamed.md"))) === undefined, + "Delete did not converge" + ); + await stop(first, "SIGTERM"); + assert.equal(first.exitCode, 0, "CLI shutdown failed"); + } +); diff --git a/frontend/test-support/real-disk.test.ts b/frontend/test-support/real-disk.test.ts index 1819f632..63e5c0de 100644 --- a/frontend/test-support/real-disk.test.ts +++ b/frontend/test-support/real-disk.test.ts @@ -3,9 +3,7 @@ import { test } from "node:test"; import * as fs from "node:fs/promises"; import * as path from "node:path"; import { tmpdir } from "node:os"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { RealDisk } from "./real-disk"; +import { NodeFileSystemOperations } from "../local-client-cli/src/node-filesystem"; import { Database, emptyState } from "../sync-client/src/persistence/database"; import { FileOperations } from "../sync-client/src/file-operations/file-operations"; import type { ServerConfig } from "../sync-client/src/services/server-config"; @@ -21,16 +19,7 @@ test( try { const root = path.join(directory, "vault"); await fs.mkdir(root); - const helper = path.join(directory, "native-fs"); - await promisify(execFile)("cc", [ - "-Wall", - "-Wextra", - "-O2", - path.join(__dirname, "native-fs.c"), - "-o", - helper - ]); - const disk = new RealDisk(root, helper); + const disk = new NodeFileSystemOperations(root); const snapshot = (text: string) => ({ content: new TextEncoder().encode(text) }); @@ -50,6 +39,32 @@ test( ).toString() ) ); + for (const unsafe of [ + "../outside", + "/absolute", + "folder/../outside" + ]) { + await assert.rejects( + disk.write(unsafe, snapshot("blocked")), + /Unsafe path/ + ); + await assert.rejects(disk.readSnapshot(unsafe), /Unsafe path/); + } + // POSIX filenames can contain colons. The engine, not the adapter, + // allocates their portable names for sync. + if (process.platform !== "win32") { + await fs.writeFile( + path.join(root, "local:name.md"), + "local name" + ); + assert.equal( + Buffer.from( + (await disk.readSnapshot("local:name.md"))!.content + ).toString(), + "local name" + ); + await disk.rename("local:name.md", "portable-name.md"); + } await disk.write("a.md", snapshot("A")); await disk.write("b.md", snapshot("B")); // A file in an ancestor position makes descendants absent. diff --git a/frontend/test-support/real-disk.ts b/frontend/test-support/real-disk.ts deleted file mode 100644 index ffe05c40..00000000 --- a/frontend/test-support/real-disk.ts +++ /dev/null @@ -1,171 +0,0 @@ -import assert from "node:assert/strict"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; -import { constants } from "node:fs"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import type { FileSystemOperations, FileSnapshot } from "sync-client"; - -const exec = promisify(execFile); -/** Small integration fixture, NOT a supported application adapter. Safety - * preflights detect static symlinks/hardlinks; they are not an openat sandbox - * against a hostile process replacing ancestors concurrently. */ -export class RealDisk implements FileSystemOperations { - public constructor( - private readonly root: string, - private readonly helper: string - ) {} - private async resolve(relative: string): Promise { - assert( - relative === "" || - relative - .split("/") - .every( - (p) => p && p !== "." && p !== ".." && !p.includes("\\") - ), - "Unsafe path" - ); - let current = this.root; - for (const component of relative.split("/").filter(Boolean)) { - current = path.join(current, component); - const entry = await this.info(current); - if (entry) - assert( - entry.isDirectory() || - (entry.isFile() && entry.nlink === 1), - "Symlink, hardlink or special file" - ); - } - return current; - } - private async info(absolute: string) { - try { - return await fs.lstat(absolute); - } catch (error) { - if ( - ["ENOENT", "ENOTDIR"].includes( - (error as NodeJS.ErrnoException).code ?? "" - ) - ) - return undefined; - throw error; - } - } - public async stat(relative: string) { - const info = await this.info(await this.resolve(relative)); - return info - ? { - kind: info.isFile() - ? ("file" as const) - : ("directory" as const), - size: info.size - } - : undefined; - } - public async exists(relative: string) { - return (await this.stat(relative)) !== undefined; - } - public async readSnapshot( - relative: string - ): Promise { - const absolute = await this.resolve(relative); - let handle; - try { - handle = await fs.open( - absolute, - constants.O_RDONLY | constants.O_NOFOLLOW - ); - } catch (error) { - if ( - ["ENOENT", "ENOTDIR"].includes( - (error as NodeJS.ErrnoException).code ?? "" - ) - ) - return undefined; - throw error; - } - try { - const before = await handle.stat({ bigint: true }); - assert( - before.isFile() && before.nlink === 1n, - "Not an independent regular file" - ); - const content = new Uint8Array(await handle.readFile()); - const after = await handle.stat({ bigint: true }); - const atPath = await fs.lstat(absolute, { bigint: true }); - assert( - before.ino === after.ino && - after.ino === atPath.ino && - before.size === after.size && - before.mtimeNs === after.mtimeNs && - before.ctimeNs === after.ctimeNs, - "File changed during read" - ); - return { content }; - } finally { - await handle.close(); - } - } - public async listFilesRecursively(root = ""): Promise { - const result: string[] = []; - for (const name of await fs.readdir(await this.resolve(root))) { - const relative = root ? `${root}/${name}` : name; - if ((await this.stat(relative))!.kind === "directory") - result.push(...(await this.listFilesRecursively(relative))); - else result.push(relative); - } - return result.sort(); - } - public async createDirectory(relative: string): Promise { - await this.resolve(relative); - let current = this.root; - for (const component of relative.split("/").filter(Boolean)) { - const next = path.join(current, component); - try { - await fs.mkdir(next); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") - throw error; - } - assert((await fs.lstat(next)).isDirectory()); - current = next; - } - } - public async write( - relative: string, - snapshot: FileSnapshot - ): Promise { - assert( - !snapshot.cursors?.length, - "This disk-only fixture has no editor selections" - ); - const destination = await this.resolve(relative); - await fs.writeFile(destination, snapshot.content, { - flag: "wx", - mode: 0o600 - }); - } - public async rename(from: string, to: string): Promise { - const source = await this.resolve(from), - destination = await this.resolve(to); - assert((await this.info(source))?.isFile()); - await exec(this.helper, ["rename", source, destination]); - } - public async delete(relative: string): Promise { - assert(relative, "Cannot prune root"); - const absolute = await this.resolve(relative); - const entry = await this.info(absolute); - if (!entry) return; - assert(entry.isDirectory(), "Cannot delete regular file"); - for (const child of await fs.readdir(absolute)) - await this.delete(`${relative}/${child}`); - await fs.rmdir(absolute); - } - public async deleteFile(relative: string): Promise { - const absolute = await this.resolve(relative); - const entry = await this.info(absolute); - if (!entry) return; - assert(entry.isFile() && entry.nlink === 1, "Cannot unlink non-file"); - await fs.unlink(absolute); - } -} diff --git a/frontend/test-support/storage.ts b/frontend/test-support/storage.ts index 09e339a3..a7db333e 100644 --- a/frontend/test-support/storage.ts +++ b/frontend/test-support/storage.ts @@ -2,14 +2,11 @@ import assert from "node:assert/strict"; import type { FileSnapshot, FileSystemOperations, - StoredDatabase, - SyncSettings + StoredDatabase } from "sync-client"; -export type StoredClient = Partial<{ - settings: Partial; - database: Partial; -}>; +import type { StoredClient } from "sync-client"; +export type { StoredClient } from "sync-client"; export type Boundary = (label: string) => void | Promise; type Entry = FileSnapshot | null; // null is a directory; absence is not a directory. export interface DiskImage { diff --git a/scripts/e2e-runner.mjs b/scripts/e2e-runner.mjs index d38fe17b..89b2d342 100644 --- a/scripts/e2e-runner.mjs +++ b/scripts/e2e-runner.mjs @@ -176,7 +176,7 @@ async function main() { }; const frontend = join(root, "frontend"); - // Build only packages in scope and always rebuild the protocol peers. + // Build production adapters as well as the protocol peers and test clients. if ( !(await run( "build-server", @@ -201,6 +201,10 @@ async function main() { "deterministic-tests", "--workspace", "test-client", + "--workspace", + "local-client-cli", + "--workspace", + "vault-link-obsidian-plugin", ], frontend, )) @@ -222,6 +226,7 @@ async function main() { ["run", "test", "--workspace", "sync-client"], frontend, ); + await run("adapter-unit-tests", "npm", ["run", "test", "--workspace", "local-client-cli", "--workspace", "vault-link-obsidian-plugin"], frontend); await run("harness-self-tests", "npm", ["run", "test:harness"], frontend); await run("crash-and-protocol", "npm", ["run", "test:protocol"], frontend); -- 2.54.0