andras/vault-link
andras/vault-link
Watch
1
0
Fork
You've already forked vault-link
 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 35

Add proper shutdown, rate limits, config validation, cors config, fix dangling cursors, cache regex, merge created texts

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-28 09:49:46 +00:00
commit e15b0f9903
28 changed files with 1278 additions and 465 deletions
Download patch file Download diff file Expand all files Collapse all files

7
scripts/check.sh
View file

@ -30,8 +30,11 @@ fi
which cargo-machete || cargo install cargo-machete which cargo-machete || cargo install cargo-machete
cargo machete --with-metadata cargo machete --with-metadata
cd ..
scripts/update-api-types.sh # this will dirty up the git state if not up-to-date
echo "Running checks in frontend" echo "Running checks in frontend"
cd ../frontend cd frontend
if [[ "$FIX_MODE" == true ]]; then if [[ "$FIX_MODE" == true ]]; then
npm install npm install
@ -57,6 +60,4 @@ if [[ "$FIX_MODE" == false ]] && [[ $(git status --porcelain) ]]; then
exit 1 exit 1
fi fi
cd ..
echo "Success" echo "Success"

4
scripts/utils/wait-for-server.sh
View file

@ -2,14 +2,14 @@
set -e set -e
SERVER_URL="http://localhost:3000" SERVER_URL="http://localhost:3010"
MAX_RETRIES=30 MAX_RETRIES=30
RETRY_INTERVAL_IN_SECONDS=5 RETRY_INTERVAL_IN_SECONDS=5
echo "Waiting for $SERVER_URL to become available..." echo "Waiting for $SERVER_URL to become available..."
count=0 count=0
while [ $count -lt $MAX_RETRIES ]; do while [ $count -lt $MAX_RETRIES ]; do
if curl -s -f -o /dev/null $SERVER_URL; then if curl -s -o /dev/null $SERVER_URL; then
echo "$SERVER_URL is now available!" echo "$SERVER_URL is now available!"
break break
fi fi

112
sync-server/Cargo.lock generated
View file

@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.2" version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [ dependencies = [
"find-msvc-tools",
"shlex", "shlex",
] ]
@ -624,6 +625,12 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.11.1" version = "0.11.1"
@ -1272,6 +1279,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.8.0" version = "0.8.0"
@ -1582,12 +1599,12 @@ dependencies = [
[[package]] [[package]]
name = "reconcile-text" name = "reconcile-text"
version = "0.8.0" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5" checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557"
dependencies = [ dependencies = [
"serde", "serde",
"thiserror 2.0.17", "thiserror 2.0.18",
] ]
[[package]] [[package]]
@ -1648,6 +1665,40 @@ dependencies = [
"zeroize", "zeroize",
] ]
[[package]]
name = "rust-embed"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
"walkdir",
]
[[package]]
name = "rust-embed-impl"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.90",
"walkdir",
]
[[package]]
name = "rust-embed-utils"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
dependencies = [
"sha2",
"walkdir",
]
[[package]] [[package]]
name = "rustc-demangle" name = "rustc-demangle"
version = "0.1.24" version = "0.1.24"
@ -1679,6 +1730,15 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]] [[package]]
name = "sanitize-filename" name = "sanitize-filename"
version = "0.6.0" version = "0.6.0"
@ -1916,7 +1976,7 @@ dependencies = [
"serde_json", "serde_json",
"sha2", "sha2",
"smallvec", "smallvec",
"thiserror 2.0.17", "thiserror 2.0.18",
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"tracing", "tracing",
@ -2000,7 +2060,7 @@ dependencies = [
"smallvec", "smallvec",
"sqlx-core", "sqlx-core",
"stringprep", "stringprep",
"thiserror 2.0.17", "thiserror 2.0.18",
"tracing", "tracing",
"uuid", "uuid",
"whoami", "whoami",
@ -2039,7 +2099,7 @@ dependencies = [
"smallvec", "smallvec",
"sqlx-core", "sqlx-core",
"stringprep", "stringprep",
"thiserror 2.0.17", "thiserror 2.0.18",
"tracing", "tracing",
"uuid", "uuid",
"whoami", "whoami",
@ -2065,7 +2125,7 @@ dependencies = [
"serde", "serde",
"serde_urlencoded", "serde_urlencoded",
"sqlx-core", "sqlx-core",
"thiserror 2.0.17", "thiserror 2.0.18",
"tracing", "tracing",
"url", "url",
"uuid", "uuid",
@ -2136,15 +2196,18 @@ dependencies = [
"futures", "futures",
"humantime-serde", "humantime-serde",
"log", "log",
"mime_guess",
"rand 0.9.0", "rand 0.9.0",
"reconcile-text", "reconcile-text",
"regex", "regex",
"rust-embed",
"sanitize-filename", "sanitize-filename",
"serde", "serde",
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"sqlx", "sqlx",
"thiserror 2.0.17", "subtle",
"thiserror 2.0.18",
"tokio", "tokio",
"tower-http", "tower-http",
"tracing", "tracing",
@ -2203,11 +2266,11 @@ dependencies = [
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "2.0.17" version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [ dependencies = [
"thiserror-impl 2.0.17", "thiserror-impl 2.0.18",
] ]
[[package]] [[package]]
@ -2223,9 +2286,9 @@ dependencies = [
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "2.0.17" version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2276,7 +2339,6 @@ dependencies = [
"bytes", "bytes",
"libc", "libc",
"mio", "mio",
"parking_lot",
"pin-project-lite", "pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"socket2", "socket2",
@ -2434,7 +2496,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
dependencies = [ dependencies = [
"chrono", "chrono",
"lazy_static", "lazy_static",
"thiserror 2.0.17", "thiserror 2.0.18",
"ts-rs-macros", "ts-rs-macros",
"uuid", "uuid",
] ]
@ -2481,6 +2543,12 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea" checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.17" version = "0.3.17"
@ -2577,6 +2645,16 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"

5
sync-server/Cargo.toml
View file

@ -33,7 +33,10 @@ serde_json = "1.0.140"
bimap = "0.6.3" bimap = "0.6.3"
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] } ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
base64 = "0.22.1" base64 = "0.22.1"
reconcile-text = { version = "0.8.0", features = ["serde"] } reconcile-text = { version = "0.11.0", features = ["serde"] }
rust-embed = "8.5"
mime_guess = "2.0"
subtle = "2.6.1"
[profile.release] [profile.release]
codegen-units = 1 codegen-units = 1

5
sync-server/config-e2e.yml
View file

@ -1,12 +1,13 @@
database: database:
databases_directory_path: databases databases_directory_path: databases
max_connections_per_vault: 12 max_connections_per_vault: 8
cursor_timeout: 1m cursor_timeout: 1m
server: server:
host: 0.0.0.0 host: 0.0.0.0
port: 3000 port: 3010
max_body_size_mb: 512 max_body_size_mb: 512
max_clients_per_vault: 256 max_clients_per_vault: 256
broadcast_channel_capacity: 1024
response_timeout: 30m response_timeout: 30m
mergeable_file_extensions: mergeable_file_extensions:
- md - md

27
sync-server/src/app_state.rs
View file

@ -2,6 +2,8 @@ pub mod cursors;
pub mod database; pub mod database;
pub mod websocket; pub mod websocket;
use std::sync::{Arc, atomic::AtomicUsize};
use anyhow::Result; use anyhow::Result;
use cursors::Cursors; use cursors::Cursors;
use database::Database; use database::Database;
@ -15,21 +17,42 @@ pub struct AppState {
pub database: Database, pub database: Database,
pub cursors: Cursors, pub cursors: Cursors,
pub broadcasts: Broadcasts, pub broadcasts: Broadcasts,
/// Tracks WebSocket connections that have upgraded but not yet completed
/// the authentication handshake
pub pending_ws_connections: Arc<AtomicUsize>,
/// Send on this channel to stop background tasks (cursor cleanup,
/// idle-pool cleanup)
shutdown_tx: Arc<tokio::sync::watch::Sender<()>>,
} }
impl AppState { impl AppState {
pub async fn try_new(config: Config) -> Result<Self> { pub async fn try_new(config: Config) -> Result<Self> {
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
let broadcasts = Broadcasts::new(&config.server); let broadcasts = Broadcasts::new(&config.server);
let database = Database::try_new(&config.database, &broadcasts).await?; let database =
Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?;
let cursors: Cursors = Cursors::new(&config.database, &broadcasts); let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
Cursors::start_background_task(cursors.clone()); Cursors::start_background_task(cursors.clone(), shutdown_rx);
Ok(Self { Ok(Self {
config, config,
database, database,
cursors, cursors,
broadcasts, broadcasts,
pending_ws_connections: Arc::new(AtomicUsize::new(0)),
shutdown_tx: Arc::new(shutdown_tx),
}) })
} }
/// Signal all background tasks (idle pool cleanup, cursor cleanup) to stop
pub fn shutdown(&self) {
let _ = self.shutdown_tx.send(());
}
/// Get a receiver to be notified when shutdown is triggered
pub fn subscribe_shutdown(&self) -> tokio::sync::watch::Receiver<()> {
self.shutdown_tx.subscribe()
}
} }

92
sync-server/src/app_state/cursors.rs
View file

@ -42,7 +42,9 @@ impl Cursors {
) { ) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await; let mut vault_to_cursors = self.vault_to_cursors.lock().await;
let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new); let all_device_cursors = vault_to_cursors
.entry(vault_id.clone())
.or_insert_with(Vec::new);
all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id); all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id);
all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors { all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors {
@ -52,7 +54,7 @@ impl Cursors {
})); }));
drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock
self.broadcast_cursors().await; self.broadcast_cursors_for_vault(&vault_id).await;
} }
pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> { pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> {
@ -69,45 +71,83 @@ impl Cursors {
.unwrap_or_default() .unwrap_or_default()
} }
pub fn start_background_task(self) { pub fn start_background_task(self, mut shutdown: tokio::sync::watch::Receiver<()>) {
tokio::spawn(async move { tokio::spawn(async move {
loop { loop {
self.remove_expired_cursors().await; tokio::select! {
tokio::time::sleep(Duration::from_secs(1)).await; () = tokio::time::sleep(Duration::from_secs(1)) => {
self.remove_expired_cursors().await;
}
Ok(()) = shutdown.changed() => break,
}
} }
}); });
} }
async fn remove_expired_cursors(&self) { async fn remove_expired_cursors(&self) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await; let changed_vaults: Vec<VaultId> = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
for (_vault_id, cursors) in vault_to_cursors.iter_mut() { let mut changed = Vec::new();
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout)); for (vault_id, cursors) in vault_to_cursors.iter_mut() {
let before = cursors.len();
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
if cursors.len() != before {
changed.push(vault_id.clone());
}
}
// Remove empty vault entries to prevent unbounded growth
vault_to_cursors.retain(|_, cursors| !cursors.is_empty());
changed
};
for vault_id in &changed_vaults {
self.broadcast_cursors_for_vault(vault_id).await;
} }
} }
async fn broadcast_cursors(&self) { async fn broadcast_cursors_for_vault(&self, vault_id: &VaultId) {
let vault_to_cursors = self.vault_to_cursors.lock().await; let client_cursors: Vec<ClientCursors> = {
let vault_to_cursors = self.vault_to_cursors.lock().await;
vault_to_cursors
.get(vault_id)
.map(|cursors| cursors.iter().map(|c| c.client_cursors.clone()).collect())
.unwrap_or_default()
};
for (vault_id, cursors) in vault_to_cursors.iter() { self.broadcasts
self.broadcasts .send_document_update(
.send_document_update( vault_id.clone(),
vault_id.clone(), WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions( CursorPositionFromServer {
CursorPositionFromServer { clients: client_cursors,
clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(), },
}, )),
)), )
) .await;
.await;
}
} }
pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) { pub async fn remove_cursors_of_device(&self, vault_id: &VaultId, device_id: &DeviceId) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await; let changed = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) { if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
cursors.retain(|c| c.client_cursors.device_id != device_id); let before = cursors.len();
cursors.retain(|c| c.client_cursors.device_id != *device_id);
let changed = cursors.len() != before;
if cursors.is_empty() {
vault_to_cursors.remove(vault_id);
}
changed
} else {
false
}
};
if changed {
self.broadcast_cursors_for_vault(vault_id).await;
} }
} }
} }

490
sync-server/src/app_state/database.rs
View file

@ -9,8 +9,17 @@ use models::{
use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions, types::chrono::Utc}; use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions, types::chrono::Utc};
pub mod models; pub mod models;
use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions};
use tokio::sync::Mutex; /// Sentinel error indicating the SQLite database is busy (SQLITE_BUSY).
/// Handlers can downcast to this to return 429 instead of 500.
#[derive(Debug, thiserror::Error)]
#[error("Database is busy")]
pub struct WriteBusyError;
use sqlx::{
Pool, Sqlite, pool::PoolConnection, sqlite::SqliteConnection, sqlite::SqlitePoolOptions,
};
use tokio::sync::{Mutex, OnceCell};
use tokio::time::Instant; use tokio::time::Instant;
use uuid::fmt::Hyphenated; use uuid::fmt::Hyphenated;
@ -19,33 +28,154 @@ use super::websocket::{
models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate}, models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate},
}; };
use crate::config::database_config::DatabaseConfig; use crate::config::database_config::DatabaseConfig;
use crate::consts::IDLE_POOL_TIMEOUT;
#[derive(Clone)] /// Holds separate reader and writer pools for a single vault.
struct PoolWithTimestamp { /// The writer pool has exactly 1 connection so writes never compete
pool: Pool<Sqlite>, /// with reads for pool slots.
last_accessed: Instant, #[derive(Debug, Clone)]
struct VaultPools {
reader: Pool<Sqlite>,
writer: Pool<Sqlite>,
} }
impl std::fmt::Debug for PoolWithTimestamp { #[derive(Debug)]
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { struct VaultPool {
f.debug_struct("PoolWithTimestamp") cell: Arc<OnceCell<VaultPools>>,
.field("pool", &"Pool<Sqlite>") last_accessed: Mutex<Instant>,
.field("last_accessed", &self.last_accessed)
.finish()
}
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct Database { pub struct Database {
config: DatabaseConfig, config: DatabaseConfig,
broadcasts: Broadcasts, broadcasts: Broadcasts,
connection_pools: Arc<Mutex<HashMap<VaultId, PoolWithTimestamp>>>, connection_pools: Arc<Mutex<HashMap<VaultId, Arc<VaultPool>>>>,
/// Per-vault write serialization. SQLite allows only one writer at a
/// time; `BEGIN IMMEDIATE` on a second connection blocks until the first
/// commits (up to `busy_timeout`). Under concurrent load the blocked
/// connections consume the pool, starving even read-only requests.
/// This mutex moves the wait from the SQLite layer (where it holds a
/// pool connection) to the Tokio layer (where it holds nothing).
write_locks: Arc<Mutex<HashMap<VaultId, Arc<tokio::sync::Mutex<()>>>>>,
} }
pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>; /// A write transaction backed by a raw `BEGIN IMMEDIATE` instead of sqlx's
/// savepoint-based `Transaction`. This avoids the savepoint mismatch caused
/// by the old `END; BEGIN IMMEDIATE;` workaround.
///
/// Holds an `OwnedMutexGuard` that serializes write transactions per vault
/// at the application level (see `Database::write_locks`). The guard is
/// released when the transaction is committed, rolled back, or dropped.
pub struct WriteTransaction {
conn: Option<PoolConnection<Sqlite>>,
_write_guard: tokio::sync::OwnedMutexGuard<()>,
}
impl WriteTransaction {
async fn new(pool: &Pool<Sqlite>, write_guard: tokio::sync::OwnedMutexGuard<()>) -> Result<Self> {
let mut conn = pool
.acquire()
.await
.context("Cannot acquire connection for write transaction")?;
if let Err(e) = sqlx::query("BEGIN IMMEDIATE")
.execute(&mut *conn)
.await
{
let is_busy = match &e {
sqlx::Error::Database(db_err) => {
// SQLITE_BUSY base code is 5. Extended codes share base 5.
let busy_by_code = db_err.code().is_some_and(|c| {
c.parse::<u32>().is_ok_and(|n| n & 0xFF == 5)
});
busy_by_code || db_err.message().contains("database is locked")
}
_ => false,
};
if is_busy {
return Err(WriteBusyError.into());
}
return Err(e).context("Cannot begin immediate transaction");
}
Ok(Self { conn: Some(conn), _write_guard: write_guard })
}
pub async fn commit(mut self) -> Result<()> {
if let Some(mut conn) = self.conn.take() {
sqlx::query("COMMIT")
.execute(&mut *conn)
.await
.context("Failed to commit transaction")?;
}
Ok(())
}
pub async fn rollback(mut self) -> Result<()> {
if let Some(mut conn) = self.conn.take() {
sqlx::query("ROLLBACK")
.execute(&mut *conn)
.await
.context("Failed to rollback transaction")?;
}
Ok(())
}
}
impl Drop for WriteTransaction {
fn drop(&mut self) {
if self.conn.is_some() {
// The connection is returned to the pool with an open transaction.
// The pool's `before_acquire` hook issues a ROLLBACK before
// handing it to the next consumer, so no async work is needed
// here. If the pool is being shut down, SQLite itself rolls back
// uncommitted transactions when the connection closes.
log::warn!("WriteTransaction dropped without commit or rollback");
}
}
}
impl std::ops::Deref for WriteTransaction {
type Target = SqliteConnection;
fn deref(&self) -> &Self::Target {
self.conn
.as_ref()
.expect("BUG: WriteTransaction dereferenced after being consumed")
.deref()
}
}
impl std::ops::DerefMut for WriteTransaction {
fn deref_mut(&mut self) -> &mut Self::Target {
self.conn
.as_mut()
.expect("BUG: WriteTransaction dereferenced after being consumed")
.deref_mut()
}
}
/// Ensure the connection has no leftover open transaction (e.g. from a
/// `WriteTransaction` that was dropped without commit/rollback). ROLLBACK
/// is a harmless no-op if no transaction is active.
fn rollback_before_acquire(
conn: &mut SqliteConnection,
_meta: sqlx::pool::PoolConnectionMetadata,
) -> futures::future::BoxFuture<'_, Result<bool, sqlx::Error>> {
Box::pin(async move {
if let Err(e) = sqlx::query("ROLLBACK").execute(&mut *conn).await {
// "cannot rollback - no transaction is active" is the common
// case (connection returned cleanly). Only unexpected errors
// deserve attention.
log::debug!("before_acquire ROLLBACK failed: {e}");
}
Ok(true)
})
}
impl Database { impl Database {
pub async fn try_new(config: &DatabaseConfig, broadcasts: &Broadcasts) -> Result<Self> { pub async fn try_new(
config: &DatabaseConfig,
broadcasts: &Broadcasts,
shutdown: tokio::sync::watch::Receiver<()>,
) -> Result<Self> {
tokio::fs::create_dir_all(&config.databases_directory_path) tokio::fs::create_dir_all(&config.databases_directory_path)
.await .await
.with_context(|| { .with_context(|| {
@ -70,24 +200,29 @@ impl Database {
.trim_end_matches(".sqlite") .trim_end_matches(".sqlite")
.to_owned(); .to_owned();
let pool = Self::create_vault_database(config, &vault).await?; Self::validate_vault_id(&vault)?;
let pools = Self::create_vault_database(config, &vault).await?;
let cell = Arc::new(OnceCell::new());
cell.set(pools).expect("cell is new");
connection_pools.insert( connection_pools.insert(
vault.clone(), vault.clone(),
PoolWithTimestamp { Arc::new(VaultPool {
pool, cell,
last_accessed: Instant::now(), last_accessed: Mutex::new(Instant::now()),
}, }),
); );
} }
info!("Database migrations applied");
let database = Self { let database = Self {
config: config.clone(), config: config.clone(),
connection_pools: Arc::new(Mutex::new(connection_pools)), connection_pools: Arc::new(Mutex::new(connection_pools)),
broadcasts: broadcasts.clone(), broadcasts: broadcasts.clone(),
write_locks: Arc::new(Mutex::new(HashMap::new())),
}; };
// Start background task to cleanup idle connection pools database.start_idle_pool_cleanup(shutdown);
database.start_idle_pool_cleanup();
Ok(database) Ok(database)
} }
@ -95,92 +230,167 @@ impl Database {
async fn create_vault_database( async fn create_vault_database(
config: &DatabaseConfig, config: &DatabaseConfig,
vault: &VaultId, vault: &VaultId,
) -> Result<Pool<Sqlite>> { ) -> Result<VaultPools> {
let file_name = config let file_name = config
.databases_directory_path .databases_directory_path
.join(format!("{vault}.sqlite")); .join(format!("{vault}.sqlite"));
let connection_options = SqliteConnectOptions::new() // Database-level PRAGMAs (auto_vacuum, journal_mode) require a write
// lock and persist across connections. Set them once with a dedicated
// init connection so pool connections never need the write lock just to
// open.
let init_options = SqliteConnectOptions::new()
.filename(file_name.clone()) .filename(file_name.clone())
.create_if_missing(true) .create_if_missing(true)
.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Full) .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
.busy_timeout(Duration::from_secs(3600)) .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal);
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30));
let pool = SqlitePoolOptions::new() // Run migrations on a dedicated connection, NOT through the pool.
// The pool's `before_acquire` hook issues ROLLBACK on every checkout,
// which can roll back the migration's bookkeeping transaction (the
// _sqlx_migrations INSERT) while the DDL (ALTER TABLE) has already
// auto-committed — leaving the migration in a dirty state.
//
// Uses `run_direct` instead of `run` because `run` takes
// `impl Acquire<'_>`, whose lifetime bound prevents the enclosing
// future from satisfying the `Send` requirement of axum handlers.
let mut init_conn = sqlx::SqliteConnection::connect_with(&init_options).await?;
sqlx::migrate!("src/app_state/database/migrations")
.run_direct(&mut init_conn)
.await
.context("Cannot run pending migrations")?;
drop(init_conn);
// Per-connection PRAGMAs shared by both reader and writer pools.
// journal_mode = WAL is a no-op on an already-WAL database.
let base_options = SqliteConnectOptions::new()
.filename(file_name.clone())
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(30))
.log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30))
// In WAL mode, NORMAL is safe: data survives OS crashes, only the
// last transaction can be lost on power failure. The default FULL
// forces an extra fsync() per commit, roughly halving write throughput.
.pragma("synchronous", "NORMAL")
// 16 MB page cache per connection (negative = KiB). Reduces disk
// reads for the latest_document_versions GROUP BY view.
.pragma("cache_size", "-16384")
// Memory-mapped I/O avoids read() syscalls. SQLite falls back to
// regular I/O for writes and beyond the mapped region. 256 MB is
// conservative; the OS handles actual memory pressure.
.pragma("mmap_size", "268435456")
// Keep temp tables and sort spillovers in memory instead of temp files.
.pragma("temp_store", "MEMORY")
// Cap WAL file growth at 64 MB. Without this, the WAL can grow
// unbounded during heavy write bursts (e.g. E2E tests with many
// concurrent clients). SQLite truncates to this size on checkpoint.
.pragma("journal_size_limit", "67108864");
// Reader pool: multiple connections for concurrent reads.
let reader = SqlitePoolOptions::new()
.max_connections(config.max_connections_per_vault) .max_connections(config.max_connections_per_vault)
.acquire_slow_threshold(Duration::from_secs(30)) .acquire_slow_threshold(Duration::from_secs(30))
.test_before_acquire(true) // Disabled: the health-check query is subject to busy_timeout
.connect_with(connection_options) // and blocks all connection checkouts when a write is active,
// starving the pool for up to 30s even for simple reads.
// The before_acquire ROLLBACK hook is sufficient for cleanup.
.test_before_acquire(false)
.before_acquire(rollback_before_acquire)
.connect_with(base_options.clone())
.await .await
.with_context(|| format!("Cannot open database at `{}`", file_name.display()))?; .with_context(|| format!("Cannot open reader pool at `{}`", file_name.display()))?;
Self::run_migrations(&pool).await?; // Writer pool: exactly 1 connection, dedicated to writes.
// Since the Tokio mutex already serializes writers per vault, this
// single connection is never contended. Separating it from the
// reader pool ensures writes never compete with reads for pool slots.
let writer = SqlitePoolOptions::new()
.max_connections(1)
.acquire_slow_threshold(Duration::from_secs(30))
.test_before_acquire(false)
.before_acquire(rollback_before_acquire)
.connect_with(base_options)
.await
.with_context(|| format!("Cannot open writer pool at `{}`", file_name.display()))?;
Ok(pool) Ok(VaultPools { reader, writer })
} }
async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
sqlx::migrate!("src/app_state/database/migrations")
.run(pool)
.await
.context("Cannot check for pending migrations")
}
async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> { fn validate_vault_id(vault: &VaultId) -> Result<()> {
let mut pools = self.connection_pools.lock().await; if vault.is_empty() {
anyhow::bail!("Vault ID must not be empty");
if !pools.contains_key(vault) { }
let pool = Self::create_vault_database(&self.config, vault).await?; if vault.contains('/')
pools.insert( || vault.contains('\\')
vault.clone(), || vault.contains("..")
PoolWithTimestamp { || vault.contains('\0')
pool, {
last_accessed: Instant::now(), anyhow::bail!(
}, "Invalid vault ID: must not contain path separators, '..', or null bytes"
); );
} }
Ok(())
let pool_with_timestamp = pools
.get_mut(vault)
.expect("Pool was just inserted or already exists");
// Update last accessed time
pool_with_timestamp.last_accessed = Instant::now();
Ok(pool_with_timestamp.pool.clone())
} }
/// Attempting to write from this transaction might result in a async fn get_vault_pools(&self, vault: &VaultId) -> Result<VaultPools> {
/// database locked error. Use this transaction for read-only operations. Self::validate_vault_id(vault)?;
pub async fn create_readonly_transaction(
&self,
vault: &VaultId,
) -> Result<Transaction<'static>> {
self.get_connection_pool(vault)
.await?
.begin()
.await
.context("Cannot create transaction")
}
pub async fn create_write_transaction(&self, vault: &VaultId) -> Result<Transaction<'static>> { // Get or create the VaultPool entry. The global lock is held only
let mut transaction = self.create_readonly_transaction(vault).await?; // long enough for a HashMap lookup/insert — never across
// create_vault_database.
let vault_pool = {
let mut pools = self.connection_pools.lock().await;
pools
.entry(vault.clone())
.or_insert_with(|| {
Arc::new(VaultPool {
cell: Arc::new(OnceCell::new()),
last_accessed: Mutex::new(Instant::now()),
})
})
.clone()
};
// sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481 // OnceCell::get_or_try_init guarantees exactly-once
sqlx::query!("END; BEGIN IMMEDIATE;") // initialization: concurrent callers for the same vault wait
.execute(&mut *transaction) // here; callers for other vaults are not blocked.
let config = self.config.clone();
let vault_clone = vault.clone();
let pools = vault_pool
.cell
.get_or_try_init(|| async {
Self::create_vault_database(&config, &vault_clone).await
})
.await?; .await?;
Ok(transaction) *vault_pool.last_accessed.lock().await = Instant::now();
Ok(pools.clone())
}
/// Return the reader pool for read-only queries.
async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> {
Ok(self.get_vault_pools(vault).await?.reader)
}
pub async fn create_write_transaction(&self, vault: &VaultId) -> Result<WriteTransaction> {
let write_lock = {
let mut locks = self.write_locks.lock().await;
locks
.entry(vault.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
};
let write_guard = write_lock.lock_owned().await;
let pools = self.get_vault_pools(vault).await?;
WriteTransaction::new(&pools.writer, write_guard).await
} }
/// Return the latest state of all documents in the vault /// Return the latest state of all documents in the vault
pub async fn get_latest_documents( pub async fn get_latest_documents(
&self, &self,
vault: &VaultId, vault: &VaultId,
transaction: Option<&mut Transaction<'_>>, connection: Option<&mut SqliteConnection>,
) -> Result<Vec<DocumentVersionWithoutContent>> { ) -> Result<Vec<DocumentVersionWithoutContent>> {
let query = sqlx::query!( let query = sqlx::query!(
r#" r#"
@ -198,8 +408,8 @@ impl Database {
"#, "#,
); );
if let Some(transaction) = transaction { if let Some(conn) = connection {
query.fetch_all(&mut **transaction).await query.fetch_all(&mut *conn).await
} else { } else {
query query
.fetch_all(&self.get_connection_pool(vault).await?) .fetch_all(&self.get_connection_pool(vault).await?)
@ -216,9 +426,7 @@ impl Database {
is_deleted: row.is_deleted, is_deleted: row.is_deleted,
user_id: row.user_id, user_id: row.user_id,
device_id: row.device_id, device_id: row.device_id,
content_size: row content_size: row.content_size.unwrap_or(0),
.content_size
.expect("Content size can't be null but sqlx can't infer it"),
}) })
.collect() .collect()
}) })
@ -230,7 +438,7 @@ impl Database {
&self, &self,
vault: &VaultId, vault: &VaultId,
vault_update_id: VaultUpdateId, vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>, connection: Option<&mut SqliteConnection>,
) -> Result<Vec<DocumentVersionWithoutContent>> { ) -> Result<Vec<DocumentVersionWithoutContent>> {
let query = sqlx::query!( let query = sqlx::query!(
r#" r#"
@ -250,8 +458,8 @@ impl Database {
vault_update_id vault_update_id
); );
if let Some(transaction) = transaction { if let Some(conn) = connection {
query.fetch_all(&mut **transaction).await query.fetch_all(&mut *conn).await
} else { } else {
query query
.fetch_all(&self.get_connection_pool(vault).await?) .fetch_all(&self.get_connection_pool(vault).await?)
@ -270,9 +478,7 @@ impl Database {
is_deleted: row.is_deleted, is_deleted: row.is_deleted,
user_id: row.user_id, user_id: row.user_id,
device_id: row.device_id, device_id: row.device_id,
content_size: row content_size: row.content_size.unwrap_or(0),
.content_size
.expect("Content size can't be null but sqlx can't infer it"),
}) })
.collect() .collect()
}) })
@ -281,7 +487,7 @@ impl Database {
pub async fn get_max_update_id_in_vault( pub async fn get_max_update_id_in_vault(
&self, &self,
vault: &VaultId, vault: &VaultId,
transaction: Option<&mut Transaction<'_>>, connection: Option<&mut SqliteConnection>,
) -> Result<i64> { ) -> Result<i64> {
let query = sqlx::query!( let query = sqlx::query!(
r#" r#"
@ -290,8 +496,8 @@ impl Database {
"#, "#,
); );
if let Some(transaction) = transaction { if let Some(conn) = connection {
query.fetch_one(&mut **transaction).await query.fetch_one(&mut *conn).await
} else { } else {
query query
.fetch_one(&self.get_connection_pool(vault).await?) .fetch_one(&self.get_connection_pool(vault).await?)
@ -301,11 +507,11 @@ impl Database {
.context("Cannot fetch max update id in vault") .context("Cannot fetch max update id in vault")
} }
pub async fn get_latest_document_by_path( pub async fn get_latest_non_deleted_document_by_path(
&self, &self,
vault: &VaultId, vault: &VaultId,
relative_path: &str, relative_path: &str,
transaction: Option<&mut Transaction<'_>>, connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> { ) -> Result<Option<StoredDocumentVersion>> {
let query = sqlx::query_as!( let query = sqlx::query_as!(
StoredDocumentVersion, StoredDocumentVersion,
@ -330,8 +536,8 @@ impl Database {
relative_path relative_path
); );
if let Some(transaction) = transaction { if let Some(conn) = connection {
query.fetch_optional(&mut **transaction).await query.fetch_optional(&mut *conn).await
} else { } else {
query query
.fetch_optional(&self.get_connection_pool(vault).await?) .fetch_optional(&self.get_connection_pool(vault).await?)
@ -344,7 +550,7 @@ impl Database {
&self, &self,
vault: &VaultId, vault: &VaultId,
document_id: &DocumentId, document_id: &DocumentId,
transaction: Option<&mut Transaction<'_>>, connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> { ) -> Result<Option<StoredDocumentVersion>> {
let document_id = document_id.as_hyphenated(); let document_id = document_id.as_hyphenated();
let query = sqlx::query_as!( let query = sqlx::query_as!(
@ -366,8 +572,8 @@ impl Database {
document_id document_id
); );
if let Some(transaction) = transaction { if let Some(conn) = connection {
query.fetch_optional(&mut **transaction).await query.fetch_optional(&mut *conn).await
} else { } else {
query query
.fetch_optional(&self.get_connection_pool(vault).await?) .fetch_optional(&self.get_connection_pool(vault).await?)
@ -380,7 +586,7 @@ impl Database {
&self, &self,
vault: &VaultId, vault: &VaultId,
vault_update_id: VaultUpdateId, vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>, connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> { ) -> Result<Option<StoredDocumentVersion>> {
let query = sqlx::query_as!( let query = sqlx::query_as!(
StoredDocumentVersion, StoredDocumentVersion,
@ -400,8 +606,8 @@ impl Database {
vault_update_id vault_update_id
); );
if let Some(transaction) = transaction { if let Some(conn) = connection {
query.fetch_optional(&mut **transaction).await query.fetch_optional(&mut *conn).await
} else { } else {
query query
.fetch_optional(&self.get_connection_pool(vault).await?) .fetch_optional(&self.get_connection_pool(vault).await?)
@ -415,7 +621,7 @@ impl Database {
&self, &self,
vault_id: &VaultId, vault_id: &VaultId,
version: &StoredDocumentVersion, version: &StoredDocumentVersion,
transaction: Option<Transaction<'_>>, transaction: Option<WriteTransaction>,
) -> Result<()> { ) -> Result<()> {
let document_id = version.document_id.as_hyphenated(); let document_id = version.document_id.as_hyphenated();
let query = sqlx::query!( let query = sqlx::query!(
@ -428,9 +634,10 @@ impl Database {
content, content,
is_deleted, is_deleted,
user_id, user_id,
device_id device_id,
has_been_merged
) )
values (?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?)
"#, "#,
version.vault_update_id, version.vault_update_id,
document_id, document_id,
@ -439,7 +646,8 @@ impl Database {
version.content, version.content,
version.is_deleted, version.is_deleted,
version.user_id, version.user_id,
version.device_id version.device_id,
version.has_been_merged
); );
if let Some(mut transaction) = transaction { if let Some(mut transaction) = transaction {
@ -477,38 +685,66 @@ impl Database {
/// Cleanup idle connection pools that haven't been accessed in more than 5 minutes /// Cleanup idle connection pools that haven't been accessed in more than 5 minutes
async fn cleanup_idle_pools(&self) { async fn cleanup_idle_pools(&self) {
let mut pools = self.connection_pools.lock().await; // Collect idle vaults and remove them from the map while holding
let now = Instant::now(); // the lock briefly. Close pools OUTSIDE the lock so that
let idle_timeout = Duration::from_secs(5 * 60); // 5 minutes // pool.close().await doesn't block other get_connection_pool calls.
let idle_pools: Vec<(VaultId, Arc<VaultPool>)> = {
let mut pools = self.connection_pools.lock().await;
let now = Instant::now();
// Collect vaults to remove let vaults_to_remove: Vec<VaultId> = pools
let vaults_to_remove: Vec<VaultId> = pools .iter()
.iter() .filter(|(_, vp)| {
.filter(|(_, pool_with_timestamp)| { // If the lock is contested, the pool is actively used — not idle.
now.duration_since(pool_with_timestamp.last_accessed) > idle_timeout let Ok(last) = vp.last_accessed.try_lock() else {
}) return false;
.map(|(vault_id, _)| vault_id.clone()) };
.collect(); now.duration_since(*last) > IDLE_POOL_TIMEOUT
})
.map(|(vault_id, _)| vault_id.clone())
.collect();
// Close and remove idle pools vaults_to_remove
for vault_id in &vaults_to_remove { .into_iter()
if let Some(pool_with_timestamp) = pools.remove(vault_id) { .filter_map(|id| pools.remove(&id).map(|vp| (id, vp)))
info!("Closing idle database connection pool for vault `{vault_id}`"); .collect()
pool_with_timestamp.pool.close().await; };
for (vault_id, vault_pool) in idle_pools {
if let Some(pools) = vault_pool.cell.get() {
// Checkpoint the WAL before closing to reclaim disk space
// and ensure the next open doesn't need a large WAL replay.
// TRUNCATE mode resets the WAL file to zero bytes.
if let Err(e) = sqlx::query("PRAGMA wal_checkpoint(TRUNCATE)")
.execute(&pools.writer)
.await
{
log::warn!("WAL checkpoint failed for vault `{vault_id}`: {e}");
}
info!("Closing idle database connection pools for vault `{vault_id}`");
pools.reader.close().await;
pools.writer.close().await;
} }
} }
} }
/// Start a background task that periodically cleans up idle connection pools /// Start a background task that periodically cleans up idle connection pools
fn start_idle_pool_cleanup(&self) { fn start_idle_pool_cleanup(&self, mut shutdown: tokio::sync::watch::Receiver<()>) {
let database = self.clone(); let database = self.clone();
tokio::spawn(async move { tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60)); // Check every minute let mut interval = tokio::time::interval(Duration::from_secs(60)); // Check every minute
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop { loop {
interval.tick().await; tokio::select! {
database.cleanup_idle_pools().await; _ = interval.tick() => {
database.cleanup_idle_pools().await;
}
_ = shutdown.changed() => {
info!("Idle pool cleanup task shutting down");
break;
}
}
} }
}); });
} }

1
sync-server/src/app_state/database/models.rs
View file

@ -22,7 +22,6 @@ pub struct StoredDocumentVersion {
pub device_id: DeviceId, pub device_id: DeviceId,
#[allow(dead_code)] // This is for manual analysis #[allow(dead_code)] // This is for manual analysis
pub has_been_merged: bool, pub has_been_merged: bool,
pub idempotency_key: Option<String>,
} }
impl PartialEq<Self> for StoredDocumentVersion { impl PartialEq<Self> for StoredDocumentVersion {

64
sync-server/src/app_state/websocket/broadcasts.rs
View file

@ -1,35 +1,52 @@
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
use anyhow::Context;
use log::{debug, warn}; use log::{debug, warn};
use tokio::sync::{Mutex, broadcast}; use tokio::sync::{Mutex, broadcast};
use super::models::WebSocketServerMessageWithOrigin; use super::models::WebSocketServerMessageWithOrigin;
use crate::{ use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig};
app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error,
};
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Broadcasts { pub struct Broadcasts {
max_clients_per_vault: usize, broadcast_channel_capacity: usize,
tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>, tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>,
} }
type TxMap = HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>;
impl Broadcasts { impl Broadcasts {
pub fn new(server_config: &ServerConfig) -> Self { pub fn new(server_config: &ServerConfig) -> Self {
Self { Self {
max_clients_per_vault: server_config.max_clients_per_vault, broadcast_channel_capacity: server_config.broadcast_channel_capacity,
tx: Arc::new(Mutex::new(HashMap::new())), tx: Arc::new(Mutex::new(HashMap::new())),
} }
} }
/// Remove senders for vaults with no active receivers
fn prune_inactive_vaults(tx_map: &mut TxMap) {
tx_map.retain(|_, sender| sender.receiver_count() > 0);
}
pub async fn get_receiver( pub async fn get_receiver(
&self, &self,
vault: VaultId, vault: VaultId,
) -> broadcast::Receiver<WebSocketServerMessageWithOrigin> { max_clients: usize,
let tx = self.get_or_create(vault).await; ) -> Result<broadcast::Receiver<WebSocketServerMessageWithOrigin>, crate::errors::SyncServerError>
{
let mut tx_map = self.tx.lock().await;
Self::prune_inactive_vaults(&mut tx_map);
tx.subscribe() let sender = tx_map
.entry(vault)
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
if sender.receiver_count() >= max_clients {
return Err(crate::errors::client_error(anyhow::anyhow!(
"Vault has reached the maximum number of clients ({max_clients})"
)));
}
Ok(sender.subscribe())
} }
/// Notify all clients (who are subscribed to the vault) about an update. /// Notify all clients (who are subscribed to the vault) about an update.
@ -39,31 +56,20 @@ impl Broadcasts {
vault: VaultId, vault: VaultId,
document: WebSocketServerMessageWithOrigin, document: WebSocketServerMessageWithOrigin,
) { ) {
let tx = self.get_or_create(vault.clone()).await; let mut tx_map = self.tx.lock().await;
Self::prune_inactive_vaults(&mut tx_map);
if tx.receiver_count() == 0 { let sender = tx_map
.entry(vault.clone())
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
if sender.receiver_count() == 0 {
debug!("Skipping broadcast, no clients connected for vault `{vault}`"); debug!("Skipping broadcast, no clients connected for vault `{vault}`");
return; return;
} }
let result = tx if let Err(e) = sender.send(document) {
.send(document) warn!("Failed to broadcast to vault `{vault}`: {e}");
.context("Cannot broadcast server message to websocket listeners")
.map_err(server_error);
if result.is_err() {
warn!("Failed to send message: {result:?}");
} }
} }
async fn get_or_create(
&self,
vault: VaultId,
) -> broadcast::Sender<WebSocketServerMessageWithOrigin> {
let mut tx = self.tx.lock().await;
tx.entry(vault)
.or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone())
.clone()
}
} }

13
sync-server/src/config.rs
View file

@ -27,6 +27,19 @@ pub struct Config {
} }
impl Config { impl Config {
pub fn validate(&self) -> Result<()> {
self.server
.validate()
.context("Invalid server configuration")?;
self.logging
.validate()
.context("Invalid logging configuration")?;
self.database
.validate()
.context("Invalid database configuration")?;
Ok(())
}
pub async fn read_or_create(path: &Path) -> Result<Self> { pub async fn read_or_create(path: &Path) -> Result<Self> {
let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());

19
sync-server/src/config/database_config.rs
View file

@ -1,5 +1,6 @@
use std::{path::PathBuf, time::Duration}; use std::{path::PathBuf, time::Duration};
use anyhow::{Result, ensure};
use log::debug; use log::debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -34,6 +35,24 @@ fn default_cursor_timeout() -> Duration {
DEFAULT_CURSOR_TIMEOUT DEFAULT_CURSOR_TIMEOUT
} }
impl DatabaseConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
self.databases_directory_path.as_os_str().len() > 0,
"databases_directory_path must not be empty"
);
ensure!(
self.max_connections_per_vault > 0,
"max_connections_per_vault must be greater than 0"
);
ensure!(
!self.cursor_timeout.is_zero(),
"cursor_timeout must be greater than 0"
);
Ok(())
}
}
impl Default for DatabaseConfig { impl Default for DatabaseConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {

12
sync-server/src/config/logging_config.rs
View file

@ -1,5 +1,6 @@
use std::time::Duration; use std::time::Duration;
use anyhow::{Result, ensure};
use log::debug; use log::debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -20,6 +21,17 @@ pub struct LoggingConfig {
pub log_level: LogLevel, pub log_level: LogLevel,
} }
impl LoggingConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
!self.log_directory.is_empty(),
"log_directory must not be an empty string"
);
ensure!(self.log_rotation > 0, "log_rotation must be greater than 0");
Ok(())
}
}
impl Default for LoggingConfig { impl Default for LoggingConfig {
fn default() -> Self { fn default() -> Self {
Self { Self {

75
sync-server/src/config/server_config.rs
View file

@ -1,10 +1,13 @@
use anyhow::{Result, ensure};
use log::debug; use log::debug;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::time::Duration; use std::time::Duration;
use crate::consts::{ use crate::consts::{
DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_ALLOWED_ORIGINS, DEFAULT_BROADCAST_CHANNEL_CAPACITY, DEFAULT_HOST,
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_MAX_PENDING_WS_CONNECTIONS,
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND,
DEFAULT_RESPONSE_TIMEOUT_SECONDS,
}; };
#[derive(Debug, Deserialize, Serialize, Clone, Default)] #[derive(Debug, Deserialize, Serialize, Clone, Default)]
@ -21,11 +24,56 @@ pub struct ServerConfig {
#[serde(default = "default_max_clients_per_vault")] #[serde(default = "default_max_clients_per_vault")]
pub max_clients_per_vault: usize, pub max_clients_per_vault: usize,
#[serde(default = "default_broadcast_channel_capacity")]
pub broadcast_channel_capacity: usize,
#[serde(default = "default_response_timeout", with = "humantime_serde")] #[serde(default = "default_response_timeout", with = "humantime_serde")]
pub response_timeout: Duration, pub response_timeout: Duration,
#[serde(default = "default_mergeable_file_extensions")] #[serde(default = "default_mergeable_file_extensions")]
pub mergeable_file_extensions: Vec<String>, pub mergeable_file_extensions: Vec<String>,
/// Per-user maximum requests per second (keyed by bearer token).
/// `None` disables rate limiting.
#[serde(default = "DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND")]
pub rate_limit_per_user_per_second: Option<u64>,
/// Allowed CORS origins. Default: `["*"]` (allow all).
#[serde(default = "default_allowed_origins")]
pub allowed_origins: Vec<String>,
/// Maximum concurrent unauthenticated WebSocket connections waiting for
/// handshake. Limits resource consumption from clients that connect but
/// never authenticate.
#[serde(default = "default_max_pending_websocket_connections")]
pub max_pending_websocket_connections: usize,
}
impl ServerConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
self.response_timeout > 0,
"response_timeout must be greater than 0"
);
ensure!(
self.max_body_size_mb > 0,
"max_body_size_mb must be greater than 0"
);
ensure!(
self.max_clients_per_vault > 0,
"max_clients_per_vault must be greater than 0"
);
ensure!(
self.broadcast_channel_capacity > 0,
"broadcast_channel_capacity must be greater than 0"
);
ensure!(
self.max_pending_websocket_connections > 0,
"max_pending_websocket_connections must be greater than 0"
);
Ok(())
}
} }
fn default_host() -> String { fn default_host() -> String {
@ -48,6 +96,11 @@ fn default_max_clients_per_vault() -> usize {
DEFAULT_MAX_CLIENTS_PER_VAULT DEFAULT_MAX_CLIENTS_PER_VAULT
} }
fn default_broadcast_channel_capacity() -> usize {
debug!("Using default broadcast channel capacity: {DEFAULT_BROADCAST_CHANNEL_CAPACITY}");
DEFAULT_BROADCAST_CHANNEL_CAPACITY
}
fn default_response_timeout() -> Duration { fn default_response_timeout() -> Duration {
debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}"); debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}");
DEFAULT_RESPONSE_TIMEOUT_SECONDS DEFAULT_RESPONSE_TIMEOUT_SECONDS
@ -60,3 +113,21 @@ fn default_mergeable_file_extensions() -> Vec<String> {
.map(|s| (*s).to_owned()) .map(|s| (*s).to_owned())
.collect() .collect()
} }
fn DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND() -> Option<u64> {
debug!("Using default rate limit per second: {DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND:?}");
DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND
}
fn default_allowed_origins() -> Vec<String> {
debug!("Using default allowed origins: {DEFAULT_ALLOWED_ORIGINS:?}");
DEFAULT_ALLOWED_ORIGINS
.iter()
.map(|s| (*s).to_owned())
.collect()
}
fn default_max_pending_websocket_connections() -> usize {
debug!("Using default max pending WebSocket connections: {DEFAULT_MAX_PENDING_WS_CONNECTIONS}");
DEFAULT_MAX_PENDING_WS_CONNECTIONS
}

20
sync-server/src/consts.rs
View file

@ -5,19 +5,31 @@ use crate::utils::log_level::LogLevel;
pub const DEFAULT_CONFIG_PATH: &str = "config.yml"; pub const DEFAULT_CONFIG_PATH: &str = "config.yml";
pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases"; pub const DEFAULT_DATABASES_DIRECTORY_PATH: &str = "databases";
pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 12; pub const DEFAULT_MAX_CONNECTIONS_PER_VAULT: u32 = 6;
pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60); pub const DEFAULT_CURSOR_TIMEOUT: Duration = Duration::from_secs(60);
pub const DEFAULT_HOST: &str = "127.0.0.1"; pub const DEFAULT_HOST: &str = "127.0.0.1";
pub const DEFAULT_PORT: u16 = 3000; pub const DEFAULT_PORT: u16 = 3000;
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096; pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_secs(1800); pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_mins(30);
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256; pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;
pub const DEFAULT_BROADCAST_CHANNEL_CAPACITY: usize = 4096;
pub const DEFAULT_MAX_PENDING_WS_CONNECTIONS: usize = 128;
pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; pub const DEFAULT_LOG_DIRECTORY: &str = "logs";
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); // 1 day pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_hours(24);
pub const IDLE_POOL_TIMEOUT: Duration = Duration::from_mins(5);
pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
pub const MAX_CURSOR_DOCUMENTS: usize = 1000;
pub const MAX_CURSORS_PER_DOCUMENT: usize = 100;
pub const MAX_RELATIVE_PATH_LEN: usize = 4096;
pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info; pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"]; pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
pub const SUPPORTED_API_VERSION: u32 = 2; pub const DEFAULT_RATE_LIMIT_PER_USER_PER_SECOND: Option<u64> = None;
pub const DEFAULT_ALLOWED_ORIGINS: &[&str] = &["*"];
pub const SUPPORTED_API_VERSION: u32 = 3;

9
sync-server/src/main.rs
View file

@ -41,11 +41,12 @@ async fn main() -> ExitCode {
} }
}; };
let mut result = set_up_logging(&args, &config.logging); let result = async {
config.validate().map_err(init_error)?;
if result.is_ok() { set_up_logging(&args, &config.logging)?;
result = start_server(config).await; start_server(config).await
} }
.await;
match result { match result {
Ok(()) => ExitCode::SUCCESS, Ok(()) => ExitCode::SUCCESS,

123
sync-server/src/server.rs
View file

@ -8,6 +8,7 @@ mod fetch_latest_document_version;
mod fetch_latest_documents; mod fetch_latest_documents;
mod index; mod index;
mod ping; mod ping;
mod rate_limit;
mod requests; mod requests;
mod responses; mod responses;
mod update_document; mod update_document;
@ -24,7 +25,7 @@ use axum::{
routing::{IntoMakeService, delete, get, post, put}, routing::{IntoMakeService, delete, get, post, put},
}; };
use device_id_header::DEVICE_ID_HEADER_NAME; use device_id_header::DEVICE_ID_HEADER_NAME;
use log::info; use log::{info, warn};
use tokio::signal; use tokio::signal;
use tower_http::{ use tower_http::{
LatencyUnit, LatencyUnit,
@ -41,7 +42,7 @@ use tracing::{Level, info_span};
use crate::{ use crate::{
app_state::AppState, app_state::AppState,
config::{Config, server_config::ServerConfig}, config::{Config, server_config::ServerConfig},
errors::{client_error, not_found_error}, consts::GRACEFUL_SHUTDOWN_TIMEOUT,
}; };
pub async fn create_server(config: Config) -> Result<()> { pub async fn create_server(config: Config) -> Result<()> {
@ -56,21 +57,26 @@ pub async fn create_server(config: Config) -> Result<()> {
.route("/", get(index::index)) .route("/", get(index::index))
.route("/vaults/:vault_id/ping", get(ping::ping)) .route("/vaults/:vault_id/ping", get(ping::ping))
.route("/vaults/:vault_id/ws", get(websocket::websocket_handler)) .route("/vaults/:vault_id/ws", get(websocket::websocket_handler))
.fallback(index::spa_fallback);
let cors_layer = build_cors_layer(&server_config).context("Invalid CORS configuration")?;
if let Some(rate_limit) = server_config.rate_limit_per_user_per_second {
info!("Rate limiting enabled: {rate_limit} requests/second per user");
let limiter = rate_limit::RateLimiter::new(rate_limit);
app = app.layer(middleware::from_fn_with_state(
limiter,
rate_limit::rate_limit_middleware,
));
}
let app = app
.layer(DefaultBodyLimit::disable()) .layer(DefaultBodyLimit::disable())
.layer(RequestBodyLimitLayer::new( .layer(RequestBodyLimitLayer::new(
app_state.config.server.max_body_size_mb * 1024 * 1024, app_state.config.server.max_body_size_mb * 1024 * 1024,
)) ))
.layer(TimeoutLayer::new(server_config.response_timeout)) .layer(TimeoutLayer::new(server_config.response_timeout))
.layer( .layer(cors_layer)
CorsLayer::new()
.allow_origin("*".parse::<HeaderValue>().expect("Failed to parse origin"))
.allow_headers([
http::header::CONTENT_TYPE,
http::header::AUTHORIZATION,
DEVICE_ID_HEADER_NAME.clone(),
])
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]),
)
.layer( .layer(
TraceLayer::new_for_http() TraceLayer::new_for_http()
.make_span_with(|request: &Request<_>| { .make_span_with(|request: &Request<_>| {
@ -90,12 +96,39 @@ pub async fn create_server(config: Config) -> Result<()> {
.on_eos(DefaultOnEos::new()) .on_eos(DefaultOnEos::new())
.on_failure(DefaultOnFailure::new().level(Level::ERROR)), .on_failure(DefaultOnFailure::new().level(Level::ERROR)),
) )
.with_state(app_state) .with_state(app_state.clone())
.fallback(handle_404)
.fallback(handle_405)
.into_make_service(); .into_make_service();
start_server(app, &server_config).await start_server(app, &server_config, app_state).await
}
fn build_cors_layer(server_config: &ServerConfig) -> Result<CorsLayer> {
let origins = &server_config.allowed_origins;
let cors = if origins.len() == 1 && origins[0] == "*" {
info!("CORS: allowing all origins (wildcard)");
let header: HeaderValue = "*"
.parse()
.context("Failed to parse wildcard CORS origin")?;
CorsLayer::new().allow_origin(header)
} else {
let parsed: Vec<HeaderValue> = origins
.iter()
.map(|o| {
o.parse::<HeaderValue>()
.with_context(|| format!("Failed to parse CORS origin: `{o}`"))
})
.collect::<Result<Vec<_>>>()?;
CorsLayer::new().allow_origin(parsed)
};
Ok(cors
.allow_headers([
http::header::CONTENT_TYPE,
http::header::AUTHORIZATION,
DEVICE_ID_HEADER_NAME.clone(),
])
.allow_methods([Method::GET, Method::POST, Method::PUT, Method::DELETE]))
} }
fn get_authed_routes(app_state: AppState) -> Router<AppState> { fn get_authed_routes(app_state: AppState) -> Router<AppState> {
@ -135,7 +168,11 @@ fn get_authed_routes(app_state: AppState) -> Router<AppState> {
.layer(middleware::from_fn_with_state(app_state, auth_middleware)) .layer(middleware::from_fn_with_state(app_state, auth_middleware))
} }
async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig) -> Result<()> { async fn start_server(
app: IntoMakeService<axum::Router>,
config: &ServerConfig,
app_state: AppState,
) -> Result<()> {
let address = format!("{}:{}", config.host, config.port); let address = format!("{}:{}", config.host, config.port);
let listener = tokio::net::TcpListener::bind(address.clone()) let listener = tokio::net::TcpListener::bind(address.clone())
.await .await
@ -148,26 +185,46 @@ async fn start_server(app: IntoMakeService<axum::Router>, config: &ServerConfig)
.context("Failed to get local address")? .context("Failed to get local address")?
); );
axum::serve(listener, app) let mut shutdown_rx = app_state.subscribe_shutdown();
.with_graceful_shutdown(shutdown_signal())
.tcp_nodelay(true) let server = axum::serve(listener, app)
.await .with_graceful_shutdown(async move {
.context("Failed to start server") shutdown_signal().await;
app_state.shutdown();
})
.tcp_nodelay(true);
tokio::select! {
result = server => result.context("Failed to start server"),
() = async {
let _ = shutdown_rx.changed().await;
info!(
"Shutdown signal received, waiting up to {}s for in-flight requests to complete...",
GRACEFUL_SHUTDOWN_TIMEOUT.as_secs()
);
tokio::time::sleep(GRACEFUL_SHUTDOWN_TIMEOUT).await;
warn!("Graceful shutdown timed out, forcing exit");
} => Ok(()),
}
} }
async fn shutdown_signal() { async fn shutdown_signal() {
let ctrl_c = async { let ctrl_c = async {
signal::ctrl_c() if let Err(e) = signal::ctrl_c().await {
.await log::error!("Failed to install Ctrl+C handler: {e}");
.expect("failed to install Ctrl+C handler"); }
}; };
#[cfg(unix)] #[cfg(unix)]
let terminate = async { let terminate = async {
signal::unix::signal(signal::unix::SignalKind::terminate()) match signal::unix::signal(signal::unix::SignalKind::terminate()) {
.expect("failed to install signal handler") Ok(mut signal) => {
.recv() signal.recv().await;
.await; }
Err(e) => {
log::error!("Failed to install SIGTERM handler: {e}");
}
}
}; };
#[cfg(not(unix))] #[cfg(not(unix))]
@ -178,11 +235,3 @@ async fn shutdown_signal() {
() = terminate => {}, () = terminate => {},
} }
} }
async fn handle_404() -> impl IntoResponse {
not_found_error(anyhow!("Page not found"))
}
async fn handle_405() -> impl IntoResponse {
client_error(anyhow!("Method not allowed"))
}

89
sync-server/src/server/create_document.rs
View file

@ -1,3 +1,4 @@
use anyhow::Context as _;
use axum::{ use axum::{
Extension, Json, Extension, Json,
extract::{Path, State}, extract::{Path, State},
@ -5,18 +6,21 @@ use axum::{
use axum_extra::TypedHeader; use axum_extra::TypedHeader;
use axum_typed_multipart::TypedMultipart; use axum_typed_multipart::TypedMultipart;
use log::{debug, info}; use log::{debug, info};
use reconcile_text::{BuiltinTokenizer, reconcile};
use serde::Deserialize; use serde::Deserialize;
use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion}; use super::{device_id_header::DeviceIdHeader, requests::CreateDocumentVersion};
use crate::{ use crate::{
app_state::{ app_state::{
AppState, AppState,
database::models::{DocumentVersionWithoutContent, StoredDocumentVersion, VaultId}, database::models::{StoredDocumentVersion, VaultId},
}, },
config::user_config::User, config::user_config::User,
errors::{SyncServerError, client_error, server_error}, errors::{SyncServerError, client_error, server_error, write_transaction_error},
server::{responses::DocumentUpdateResponse, update_document},
utils::{ utils::{
find_first_available_path::find_first_available_path, normalize::normalize, find_first_available_path::find_first_available_path, is_binary::is_binary,
is_file_type_mergable::is_file_type_mergable, normalize::normalize,
sanitize_path::sanitize_path, sanitize_path::sanitize_path,
}, },
}; };
@ -30,48 +34,75 @@ pub struct CreateDocumentPathParams {
/// Create a new document in case a document with the same doesn't exist /// Create a new document in case a document with the same doesn't exist
/// already. If a document with the same path exists, a new version is created /// already. If a document with the same path exists, a new version is created
/// with their content merged. /// with their content merged.
///
/// Text content must be UTF-8 encoded. Clients are responsible for
/// transcoding other encodings (e.g. UTF-16) to UTF-8 before sending.
#[axum::debug_handler] #[axum::debug_handler]
#[allow(clippy::too_many_lines)]
pub async fn create_document( pub async fn create_document(
Path(CreateDocumentPathParams { vault_id }): Path<CreateDocumentPathParams>, Path(CreateDocumentPathParams { vault_id }): Path<CreateDocumentPathParams>,
Extension(user): Extension<User>, Extension(user): Extension<User>,
TypedHeader(device_id): TypedHeader<DeviceIdHeader>, TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
State(state): State<AppState>, State(state): State<AppState>,
TypedMultipart(request): TypedMultipart<CreateDocumentVersion>, TypedMultipart(request): TypedMultipart<CreateDocumentVersion>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> { ) -> Result<Json<DocumentUpdateResponse>, SyncServerError> {
debug!("Creating document in vault `{vault_id}`"); debug!("Creating document in vault `{vault_id}`");
let mut transaction = state let mut transaction = state
.database .database
.create_write_transaction(&vault_id) .create_write_transaction(&vault_id)
.await .await
.map_err(server_error)?; .map_err(write_transaction_error)?;
let document_id = match request.document_id { let sanitized_relative_path = sanitize_path(&request.relative_path).map_err(client_error)?;
Some(document_id) => { let new_content = request.content.contents.to_vec();
let existing_version = state
.database
.get_latest_document(&vault_id, &document_id, Some(&mut transaction))
.await
.map_err(server_error)?;
if existing_version.is_some() { let latest_version = state
return Err(client_error(anyhow::anyhow!(
"Document with the same ID `{document_id}` already exists"
)));
}
document_id
}
None => uuid::Uuid::new_v4(),
};
let last_update_id = state
.database .database
.get_max_update_id_in_vault(&vault_id, Some(&mut transaction)) .get_latest_non_deleted_document_by_path(
&vault_id,
&sanitized_relative_path,
Some(&mut *transaction),
)
.await
.map_err(server_error)?;
if let Some(latest_version) = latest_version {
let is_mergeable_text = is_file_type_mergable(
&sanitized_relative_path,
&state.config.server.mergeable_file_extensions,
) && !is_binary(&latest_version.content)
&& !is_binary(&new_content);
if is_mergeable_text || new_content == latest_version.content {
return update_document::update_document(
&sanitized_relative_path,
Vec::new(),
vault_id,
latest_version.document_id,
&request.relative_path,
new_content,
user,
device_id,
state,
transaction,
)
.await;
}
// For non-mergeable (binary) files with different content, don't
// merge, create a separate document at a deconflicted path so
// neither client's data is silently overwritten.
}
let document_id = uuid::Uuid::new_v4();
let last_update_id = state
.database
.get_max_update_id_in_vault(&vault_id, Some(&mut *transaction))
.await .await
.map_err(server_error)?; .map_err(server_error)?;
let sanitized_relative_path = sanitize_path(&request.relative_path);
let deduped_path = find_first_available_path( let deduped_path = find_first_available_path(
&vault_id, &vault_id,
&sanitized_relative_path, &sanitized_relative_path,
@ -91,7 +122,7 @@ pub async fn create_document(
vault_update_id: last_update_id + 1, vault_update_id: last_update_id + 1,
document_id, document_id,
relative_path: deduped_path, relative_path: deduped_path,
content: request.content.contents.to_vec(), content: new_content,
updated_date: chrono::Utc::now(), updated_date: chrono::Utc::now(),
is_deleted: false, is_deleted: false,
user_id: user.name, user_id: user.name,
@ -105,5 +136,7 @@ pub async fn create_document(
.await .await
.map_err(server_error)?; .map_err(server_error)?;
Ok(Json(new_version.into())) Ok(Json(DocumentUpdateResponse::FastForwardUpdate(
new_version.into(),
)))
} }

10
sync-server/src/server/delete_document.rs
View file

@ -1,4 +1,4 @@
use anyhow::Context; use anyhow::{Context, anyhow};
use axum::{ use axum::{
Extension, Json, Extension, Json,
extract::{Path, State}, extract::{Path, State},
@ -16,7 +16,7 @@ use crate::{
}, },
}, },
config::user_config::User, config::user_config::User,
errors::{SyncServerError, server_error}, errors::{SyncServerError, client_error, not_found_error, server_error, write_transaction_error},
utils::{normalize::normalize, sanitize_path::sanitize_path}, utils::{normalize::normalize, sanitize_path::sanitize_path},
}; };
@ -37,7 +37,7 @@ pub async fn delete_document(
Extension(user): Extension<User>, Extension(user): Extension<User>,
TypedHeader(device_id): TypedHeader<DeviceIdHeader>, TypedHeader(device_id): TypedHeader<DeviceIdHeader>,
State(state): State<AppState>, State(state): State<AppState>,
Json(request): Json<DeleteDocumentVersion>, Json(_request): Json<DeleteDocumentVersion>,
) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> { ) -> Result<Json<DocumentVersionWithoutContent>, SyncServerError> {
debug!("Deleting document `{document_id}` in vault `{vault_id}`"); debug!("Deleting document `{document_id}` in vault `{vault_id}`");
@ -45,7 +45,7 @@ pub async fn delete_document(
.database .database
.create_write_transaction(&vault_id) .create_write_transaction(&vault_id)
.await .await
.map_err(server_error)?; .map_err(write_transaction_error)?;
let last_update_id = state let last_update_id = state
.database .database
@ -77,7 +77,7 @@ pub async fn delete_document(
let new_version = StoredDocumentVersion { let new_version = StoredDocumentVersion {
vault_update_id: last_update_id + 1, vault_update_id: last_update_id + 1,
document_id, document_id,
relative_path: sanitize_path(&request.relative_path), relative_path: sanitize_path(&request.relative_path).map_err(client_error)?,
content: latest_content, // copy the content from the latest version content: latest_content, // copy the content from the latest version
updated_date: chrono::Utc::now(), updated_date: chrono::Utc::now(),
is_deleted: true, is_deleted: true,

4
sync-server/src/server/fetch_document_version.rs
View file

@ -11,7 +11,7 @@ use crate::{
AppState, AppState,
database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId}, database::models::{DocumentId, DocumentVersion, VaultId, VaultUpdateId},
}, },
errors::{SyncServerError, not_found_error, server_error}, errors::{SyncServerError, client_error, not_found_error, server_error},
utils::normalize::normalize, utils::normalize::normalize,
}; };
@ -52,7 +52,7 @@ pub async fn fetch_document_version(
)?; )?;
if result.document_id != document_id { if result.document_id != document_id {
return Err(not_found_error(anyhow!( return Err(client_error(anyhow!(
"Document with document id `{document_id}` does not have a version with id \ "Document with document id `{document_id}` does not have a version with id \
`{vault_update_id}`", `{vault_update_id}`",
))); )));

Some files were not shown because too many files have changed in this diff Show more