This commit is contained in:
Andras Schmelczer 2026-03-21 12:47:39 +00:00
commit a20264bcaf
43 changed files with 13567 additions and 3694 deletions

433
sync-server/Cargo.lock generated
View file

@ -337,10 +337,11 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]]
name = "cc"
version = "1.2.2"
version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [
"find-msvc-tools",
"shlex",
]
@ -350,6 +351,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.41"
@ -624,6 +631,12 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "flume"
version = "0.11.1"
@ -773,8 +786,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
]
[[package]]
@ -957,6 +972,24 @@ dependencies = [
"pin-project-lite",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
dependencies = [
"http",
"hyper",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots 1.0.6",
]
[[package]]
@ -966,13 +999,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
"hyper",
"pin-project-lite",
"socket2 0.5.10",
"tokio",
"tower-service",
"tracing",
]
[[package]]
@ -1153,6 +1189,12 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "ipnet"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
@ -1272,6 +1314,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "miniz_oxide"
version = "0.8.0"
@ -1505,6 +1557,58 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quinn"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls",
"socket2 0.5.10",
"thiserror 2.0.18",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom 0.2.15",
"rand 0.8.5",
"ring",
"rustc-hash",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.0",
"tracing",
"windows-sys 0.59.0",
]
[[package]]
name = "quote"
version = "1.0.37"
@ -1582,12 +1686,12 @@ dependencies = [
[[package]]
name = "reconcile-text"
version = "0.8.0"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "599cf9539996a2a19e501110404c59ba62f4974009f8fb864a8b7151c15ee5a5"
checksum = "52e0cf361887ea64c479ca871c1170dda761f84e122f2616b5579906a38d7557"
dependencies = [
"serde",
"thiserror 2.0.17",
"thiserror 2.0.18",
]
[[package]]
@ -1628,6 +1732,63 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "reqwest"
version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"once_cell",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-pemfile",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tower",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots 0.26.11",
"windows-registry",
]
[[package]]
name = "ring"
version = "0.17.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.15",
"libc",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "rsa"
version = "0.9.7"
@ -1648,12 +1809,52 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rust-embed"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
"walkdir",
]
[[package]]
name = "rust-embed-impl"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa"
dependencies = [
"proc-macro2",
"quote",
"rust-embed-utils",
"syn 2.0.90",
"walkdir",
]
[[package]]
name = "rust-embed-utils"
version = "8.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1"
dependencies = [
"sha2",
"walkdir",
]
[[package]]
name = "rustc-demangle"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustix"
version = "0.38.41"
@ -1667,6 +1868,50 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rustls"
version = "0.23.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
dependencies = [
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-pemfile"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "rustls-pki-types"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
dependencies = [
"web-time",
"zeroize",
]
[[package]]
name = "rustls-webpki"
version = "0.103.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
[[package]]
name = "rustversion"
version = "1.0.18"
@ -1679,6 +1924,15 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "sanitize-filename"
version = "0.6.0"
@ -1846,6 +2100,16 @@ dependencies = [
"serde",
]
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "socket2"
version = "0.6.0"
@ -1916,7 +2180,7 @@ dependencies = [
"serde_json",
"sha2",
"smallvec",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tracing",
@ -2000,7 +2264,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"uuid",
"whoami",
@ -2039,7 +2303,7 @@ dependencies = [
"smallvec",
"sqlx-core",
"stringprep",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"uuid",
"whoami",
@ -2065,7 +2329,7 @@ dependencies = [
"serde",
"serde_urlencoded",
"sqlx-core",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tracing",
"url",
"uuid",
@ -2136,15 +2400,18 @@ dependencies = [
"futures",
"humantime-serde",
"log",
"mime_guess",
"rand 0.9.0",
"reconcile-text",
"regex",
"reqwest",
"rust-embed",
"sanitize-filename",
"serde",
"serde_json",
"serde_yaml",
"sqlx",
"thiserror 2.0.17",
"thiserror 2.0.18",
"tokio",
"tower-http",
"tracing",
@ -2158,6 +2425,9 @@ name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
@ -2203,11 +2473,11 @@ dependencies = [
[[package]]
name = "thiserror"
version = "2.0.17"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
"thiserror-impl 2.0.17",
"thiserror-impl 2.0.18",
]
[[package]]
@ -2223,9 +2493,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
version = "2.0.17"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
@ -2276,10 +2546,9 @@ dependencies = [
"bytes",
"libc",
"mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"socket2 0.6.0",
"tokio-macros",
"windows-sys 0.61.2",
]
@ -2295,6 +2564,16 @@ dependencies = [
"syn 2.0.90",
]
[[package]]
name = "tokio-rustls"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.17"
@ -2426,6 +2705,12 @@ dependencies = [
"tracing-log",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "ts-rs"
version = "10.1.0"
@ -2434,7 +2719,7 @@ checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
dependencies = [
"chrono",
"lazy_static",
"thiserror 2.0.17",
"thiserror 2.0.18",
"ts-rs-macros",
"uuid",
]
@ -2481,6 +2766,12 @@ version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f720def6ce1ee2fc44d40ac9ed6d3a59c361c80a75a7aa8e75bb9baed31cf2ea"
[[package]]
name = "unicase"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
[[package]]
name = "unicode-bidi"
version = "0.3.17"
@ -2514,6 +2805,12 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.4"
@ -2577,6 +2874,25 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
@ -2623,6 +2939,19 @@ dependencies = [
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
dependencies = [
"cfg-if",
"js-sys",
"once_cell",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.99"
@ -2652,6 +2981,44 @@ version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
[[package]]
name = "web-sys"
version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.26.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
dependencies = [
"webpki-roots 1.0.6",
]
[[package]]
name = "webpki-roots"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "whoami"
version = "1.5.2"
@ -2692,6 +3059,36 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.48.0"

View file

@ -1,6 +1,6 @@
[package]
name = "sync_server"
rust-version = "1.92.0"
rust-version = "1.94.0"
authors = ["Andras Schmelczer <andras@schmelczer.dev>"]
edition = "2024"
license = "MIT"
@ -10,7 +10,7 @@ version = "0.14.0"
[dependencies]
serde = { version = "1.0.219", default-features = false, features = ["derive"] }
thiserror = { version = "2.0.12", default-features = false }
tokio = { version = "1.48.0", features = ["full"]}
tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "sync", "time", "net", "fs", "signal"]}
uuid = { version = "1.16.0", features = ["v4", "serde"] }
log = { version = "0.4.28" }
anyhow = { version = "1.0.100", features = ["backtrace"] }
@ -33,7 +33,10 @@ serde_json = "1.0.140"
bimap = "0.6.3"
ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] }
base64 = "0.22.1"
reconcile-text = { version = "0.8.0", features = ["serde"] }
reconcile-text = { version = "0.11.0", features = ["serde"] }
rust-embed = "8.5"
mime_guess = "2.0"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
[profile.release]
codegen-units = 1

View file

@ -1,5 +1,16 @@
// generated by `sqlx migrate build-script`
fn main() {
// trigger recompilation when a new migration is added
println!("cargo:rerun-if-changed=migrations");
// Ensure the history-ui dist directory exists so rust-embed can compile
// even when the frontend hasn't been built yet.
let dist_path = std::path::Path::new("../frontend/history-ui/dist");
if !dist_path.exists() {
std::fs::create_dir_all(dist_path).expect("Failed to create history-ui dist directory");
std::fs::write(
dist_path.join("index.html"),
"<!DOCTYPE html><html><body><p>Run <code>npm run build -w history-ui</code> first.</p></body></html>",
)
.expect("Failed to write placeholder index.html");
}
}

View file

@ -1,12 +1,14 @@
database:
databases_directory_path: databases
max_connections_per_vault: 12
max_connections_per_vault: 64
cursor_timeout: 1m
server:
host: 0.0.0.0
port: 3000
port: 3010
max_body_size_mb: 512
max_clients_per_vault: 256
broadcast_channel_capacity: 1024
dev_proxy_url: "http://localhost:5173"
response_timeout: 30m
mergeable_file_extensions:
- md

View file

@ -1,5 +1,5 @@
[toolchain]
channel = "1.92.0"
channel = "1.94.0"
targets = [
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",

View file

@ -2,6 +2,11 @@ pub mod cursors;
pub mod database;
pub mod websocket;
use std::sync::{
Arc,
atomic::AtomicUsize,
};
use anyhow::Result;
use cursors::Cursors;
use database::Database;
@ -15,21 +20,34 @@ pub struct AppState {
pub database: Database,
pub cursors: Cursors,
pub broadcasts: Broadcasts,
/// Tracks WebSocket connections that have upgraded but not yet completed
/// the authentication handshake.
pub pending_ws_connections: Arc<AtomicUsize>,
/// Send on this channel to stop background tasks (cursor cleanup,
/// idle-pool cleanup). Held by `AppState` so dropping it also
/// triggers shutdown.
#[allow(dead_code)]
shutdown_tx: Arc<tokio::sync::watch::Sender<()>>,
}
impl AppState {
pub async fn try_new(config: Config) -> Result<Self> {
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(());
let broadcasts = Broadcasts::new(&config.server);
let database = Database::try_new(&config.database, &broadcasts).await?;
let database =
Database::try_new(&config.database, &broadcasts, shutdown_rx.clone()).await?;
let cursors: Cursors = Cursors::new(&config.database, &broadcasts);
Cursors::start_background_task(cursors.clone());
Cursors::start_background_task(cursors.clone(), shutdown_rx);
Ok(Self {
config,
database,
cursors,
broadcasts,
pending_ws_connections: Arc::new(AtomicUsize::new(0)),
shutdown_tx: Arc::new(shutdown_tx),
})
}
}

View file

@ -42,7 +42,9 @@ impl Cursors {
) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
let all_device_cursors = vault_to_cursors.entry(vault_id).or_insert_with(Vec::new);
let all_device_cursors = vault_to_cursors
.entry(vault_id.clone())
.or_insert_with(Vec::new);
all_device_cursors.retain(|c| &c.client_cursors.device_id != device_id);
all_device_cursors.push(ClientCursorsWithTimeToLive::new(ClientCursors {
@ -51,8 +53,11 @@ impl Cursors {
documents_with_cursors: document_to_cursors,
}));
drop(vault_to_cursors); // Explicitly drop the lock before broadcasting to avoid deadlock
self.broadcast_cursors().await;
// IMPORTANT: Drop the lock BEFORE calling broadcast_cursors_for_vault,
// which re-acquires the same lock internally. Holding the lock here
// while calling broadcast would cause a deadlock.
drop(vault_to_cursors);
self.broadcast_cursors_for_vault(&vault_id).await;
}
pub async fn get_cursors(&self, vault_id: &VaultId) -> Vec<ClientCursors> {
@ -69,45 +74,83 @@ impl Cursors {
.unwrap_or_default()
}
pub fn start_background_task(self) {
pub fn start_background_task(self, mut shutdown: tokio::sync::watch::Receiver<()>) {
tokio::spawn(async move {
loop {
self.remove_expired_cursors().await;
tokio::time::sleep(Duration::from_secs(1)).await;
tokio::select! {
() = tokio::time::sleep(Duration::from_secs(1)) => {
self.remove_expired_cursors().await;
}
Ok(()) = shutdown.changed() => break,
}
}
});
}
async fn remove_expired_cursors(&self) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
let changed_vaults: Vec<VaultId> = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
for (_vault_id, cursors) in vault_to_cursors.iter_mut() {
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
let mut changed = Vec::new();
for (vault_id, cursors) in vault_to_cursors.iter_mut() {
let before = cursors.len();
cursors.retain(|cursor| !cursor.is_expired(self.config.cursor_timeout));
if cursors.len() != before {
changed.push(vault_id.clone());
}
}
// Remove empty vault entries to prevent unbounded growth
vault_to_cursors.retain(|_, cursors| !cursors.is_empty());
changed
};
for vault_id in &changed_vaults {
self.broadcast_cursors_for_vault(vault_id).await;
}
}
async fn broadcast_cursors(&self) {
let vault_to_cursors = self.vault_to_cursors.lock().await;
async fn broadcast_cursors_for_vault(&self, vault_id: &VaultId) {
let client_cursors: Vec<ClientCursors> = {
let vault_to_cursors = self.vault_to_cursors.lock().await;
vault_to_cursors
.get(vault_id)
.map(|cursors| cursors.iter().map(|c| c.client_cursors.clone()).collect())
.unwrap_or_default()
};
for (vault_id, cursors) in vault_to_cursors.iter() {
self.broadcasts
.send_document_update(
vault_id.clone(),
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
CursorPositionFromServer {
clients: cursors.iter().map(|c| c.client_cursors.clone()).collect(),
},
)),
)
.await;
}
self.broadcasts
.send_document_update(
vault_id.clone(),
WebSocketServerMessageWithOrigin::new(WebSocketServerMessage::CursorPositions(
CursorPositionFromServer {
clients: client_cursors,
},
)),
)
.await;
}
pub async fn remove_cursors_of_device(&self, vault_id: &str, device_id: &str) {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
pub async fn remove_cursors_of_device(&self, vault_id: &VaultId, device_id: &DeviceId) {
let changed = {
let mut vault_to_cursors = self.vault_to_cursors.lock().await;
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
cursors.retain(|c| c.client_cursors.device_id != device_id);
if let Some(cursors) = vault_to_cursors.get_mut(vault_id) {
let before = cursors.len();
cursors.retain(|c| c.client_cursors.device_id != *device_id);
let changed = cursors.len() != before;
if cursors.is_empty() {
vault_to_cursors.remove(vault_id);
}
changed
} else {
false
}
};
if changed {
self.broadcast_cursors_for_vault(vault_id).await;
}
}
}

View file

@ -6,14 +6,29 @@ use log::info;
use models::{
DocumentId, DocumentVersionWithoutContent, StoredDocumentVersion, VaultId, VaultUpdateId,
};
use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions, types::chrono::Utc};
use sqlx::{ConnectOptions, Connection, sqlite::SqliteConnectOptions, types::chrono::Utc};
pub mod models;
use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions};
use tokio::sync::Mutex;
use sqlx::{
Pool, Sqlite, pool::PoolConnection, sqlite::SqliteConnection, sqlite::SqlitePoolOptions,
};
use tokio::sync::{Mutex, OnceCell};
use tokio::time::Instant;
use uuid::fmt::Hyphenated;
/// Row struct for vault history queries (used by `sqlx::query_as!`)
#[derive(Debug)]
struct VaultHistoryRow {
vault_update_id: models::VaultUpdateId,
document_id: models::DocumentId,
relative_path: String,
updated_date: chrono::DateTime<chrono::Utc>,
is_deleted: bool,
user_id: String,
device_id: String,
content_size: Option<u64>,
}
use super::websocket::{
broadcasts::Broadcasts,
models::{WebSocketServerMessage, WebSocketServerMessageWithOrigin, WebSocketVaultUpdate},
@ -21,32 +36,98 @@ use super::websocket::{
use crate::config::database_config::DatabaseConfig;
use crate::consts::IDLE_POOL_TIMEOUT;
#[derive(Clone)]
struct PoolWithTimestamp {
pool: Pool<Sqlite>,
last_accessed: Instant,
}
impl std::fmt::Debug for PoolWithTimestamp {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PoolWithTimestamp")
.field("pool", &"Pool<Sqlite>")
.field("last_accessed", &self.last_accessed)
.finish()
}
#[derive(Debug)]
struct VaultPool {
cell: Arc<OnceCell<Pool<Sqlite>>>,
last_accessed: Mutex<Instant>,
}
#[derive(Clone, Debug)]
pub struct Database {
config: DatabaseConfig,
broadcasts: Broadcasts,
connection_pools: Arc<Mutex<HashMap<VaultId, PoolWithTimestamp>>>,
connection_pools: Arc<Mutex<HashMap<VaultId, Arc<VaultPool>>>>,
}
pub type Transaction<'a> = sqlx::Transaction<'a, Sqlite>;
/// A write transaction backed by a raw `BEGIN IMMEDIATE` instead of sqlx's
/// savepoint-based `Transaction`. This avoids the savepoint mismatch caused
/// by the old `END; BEGIN IMMEDIATE;` workaround.
pub struct WriteTransaction {
conn: Option<PoolConnection<Sqlite>>,
}
impl WriteTransaction {
async fn new(pool: &Pool<Sqlite>) -> Result<Self> {
let mut conn = pool
.acquire()
.await
.context("Cannot acquire connection for write transaction")?;
sqlx::query("BEGIN IMMEDIATE")
.execute(&mut *conn)
.await
.context("Cannot begin immediate transaction")?;
Ok(Self { conn: Some(conn) })
}
pub async fn commit(mut self) -> Result<()> {
if let Some(mut conn) = self.conn.take() {
sqlx::query("COMMIT")
.execute(&mut *conn)
.await
.context("Failed to commit transaction")?;
}
Ok(())
}
pub async fn rollback(mut self) -> Result<()> {
if let Some(mut conn) = self.conn.take() {
sqlx::query("ROLLBACK")
.execute(&mut *conn)
.await
.context("Failed to rollback transaction")?;
}
Ok(())
}
}
impl Drop for WriteTransaction {
fn drop(&mut self) {
if self.conn.is_some() {
// The connection is returned to the pool with an open transaction.
// The pool's `before_acquire` hook issues a ROLLBACK before
// handing it to the next consumer, so no async work is needed
// here. If the pool is being shut down, SQLite itself rolls back
// uncommitted transactions when the connection closes.
log::warn!("WriteTransaction dropped without commit or rollback");
}
}
}
impl std::ops::Deref for WriteTransaction {
type Target = SqliteConnection;
fn deref(&self) -> &Self::Target {
self.conn
.as_ref()
.expect("BUG: WriteTransaction dereferenced after being consumed")
.deref()
}
}
impl std::ops::DerefMut for WriteTransaction {
fn deref_mut(&mut self) -> &mut Self::Target {
self.conn
.as_mut()
.expect("BUG: WriteTransaction dereferenced after being consumed")
.deref_mut()
}
}
impl Database {
pub async fn try_new(config: &DatabaseConfig, broadcasts: &Broadcasts) -> Result<Self> {
pub async fn try_new(
config: &DatabaseConfig,
broadcasts: &Broadcasts,
shutdown: tokio::sync::watch::Receiver<()>,
) -> Result<Self> {
tokio::fs::create_dir_all(&config.databases_directory_path)
.await
.with_context(|| {
@ -71,13 +152,17 @@ impl Database {
.trim_end_matches(".sqlite")
.to_owned();
Self::validate_vault_id(&vault)?;
let pool = Self::create_vault_database(config, &vault).await?;
let cell = Arc::new(OnceCell::new());
cell.set(pool).expect("cell is new");
connection_pools.insert(
vault.clone(),
PoolWithTimestamp {
pool,
last_accessed: Instant::now(),
},
Arc::new(VaultPool {
cell,
last_accessed: Mutex::new(Instant::now()),
}),
);
}
info!("Database migrations applied");
@ -88,8 +173,7 @@ impl Database {
broadcasts: broadcasts.clone(),
};
// Start background task to cleanup idle connection pools
database.start_idle_pool_cleanup();
database.start_idle_pool_cleanup(shutdown);
Ok(database)
}
@ -102,91 +186,128 @@ impl Database {
.databases_directory_path
.join(format!("{vault}.sqlite"));
let connection_options = SqliteConnectOptions::new()
// Database-level PRAGMAs (auto_vacuum, journal_mode) require a write
// lock and persist across connections. Set them once with a dedicated
// init connection so pool connections never need the write lock just to
// open.
let init_options = SqliteConnectOptions::new()
.filename(file_name.clone())
.create_if_missing(true)
.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
.busy_timeout(Duration::from_secs(30))
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal);
// Run migrations on a dedicated connection, NOT through the pool.
// The pool's `before_acquire` hook issues ROLLBACK on every checkout,
// which can roll back the migration's bookkeeping transaction (the
// _sqlx_migrations INSERT) while the DDL (ALTER TABLE) has already
// auto-committed — leaving the migration in a dirty state.
//
// Uses `run_direct` instead of `run` because `run` takes
// `impl Acquire<'_>`, whose lifetime bound prevents the enclosing
// future from satisfying the `Send` requirement of axum handlers.
let mut init_conn = sqlx::SqliteConnection::connect_with(&init_options).await?;
sqlx::migrate!("src/app_state/database/migrations")
.run_direct(&mut init_conn)
.await
.context("Cannot run pending migrations")?;
drop(init_conn);
// Pool connections only set per-connection PRAGMAs that don't require a
// write lock. journal_mode = WAL is a no-op on an already-WAL database.
let pool_options = SqliteConnectOptions::new()
.filename(file_name.clone())
.journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
.busy_timeout(Duration::from_secs(30))
.log_slow_statements(log::LevelFilter::Warn, Duration::from_secs(30));
let pool = SqlitePoolOptions::new()
.max_connections(config.max_connections_per_vault)
.acquire_slow_threshold(Duration::from_secs(30))
.test_before_acquire(true)
.connect_with(connection_options)
.before_acquire(|conn, _meta| {
Box::pin(async move {
// Ensure the connection has no leftover open transaction
// (e.g. from a WriteTransaction that was dropped without
// commit/rollback). ROLLBACK is a harmless no-op if no
// transaction is active.
if let Err(e) = sqlx::query("ROLLBACK").execute(&mut *conn).await {
// "cannot rollback - no transaction is active" is the
// common case (connection returned cleanly). Only
// unexpected errors deserve attention.
log::debug!("before_acquire ROLLBACK failed: {e}");
}
Ok(true)
})
})
.connect_with(pool_options)
.await
.with_context(|| format!("Cannot open database at `{}`", file_name.display()))?;
Self::run_migrations(&pool).await?;
Ok(pool)
}
async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
sqlx::migrate!("src/app_state/database/migrations")
.run(pool)
.await
.context("Cannot check for pending migrations")
fn validate_vault_id(vault: &VaultId) -> Result<()> {
if vault.is_empty() {
anyhow::bail!("Vault ID must not be empty");
}
if vault.contains('/')
|| vault.contains('\\')
|| vault.contains("..")
|| vault.contains('\0')
{
anyhow::bail!(
"Invalid vault ID: must not contain path separators, '..', or null bytes"
);
}
Ok(())
}
async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> {
// First, check if the pool exists without holding the lock during creation
{
Self::validate_vault_id(vault)?;
// Get or create the VaultPool entry. The global lock is held only
// long enough for a HashMap lookup/insert — never across
// create_vault_database.
let vault_pool = {
let mut pools = self.connection_pools.lock().await;
if let Some(pool_with_timestamp) = pools.get_mut(vault) {
pool_with_timestamp.last_accessed = Instant::now();
return Ok(pool_with_timestamp.pool.clone());
}
}
pools
.entry(vault.clone())
.or_insert_with(|| {
Arc::new(VaultPool {
cell: Arc::new(OnceCell::new()),
last_accessed: Mutex::new(Instant::now()),
})
})
.clone()
};
// Create the pool outside of the lock to avoid blocking other vaults
// Note: This may result in multiple pools being created for the same vault
// under high concurrency, but only one will be kept
let new_pool = Self::create_vault_database(&self.config, vault).await?;
// Re-acquire lock and insert (or use existing if another task created it)
let mut pools = self.connection_pools.lock().await;
let pool_with_timestamp = pools
.entry(vault.clone())
.or_insert_with(|| PoolWithTimestamp {
pool: new_pool.clone(),
last_accessed: Instant::now(),
});
pool_with_timestamp.last_accessed = Instant::now();
Ok(pool_with_timestamp.pool.clone())
}
/// Attempting to write from this transaction might result in a
/// database locked error. Use this transaction for read-only operations.
pub async fn create_readonly_transaction(
&self,
vault: &VaultId,
) -> Result<Transaction<'static>> {
self.get_connection_pool(vault)
.await?
.begin()
.await
.context("Cannot create transaction")
}
pub async fn create_write_transaction(&self, vault: &VaultId) -> Result<Transaction<'static>> {
let mut transaction = self.create_readonly_transaction(vault).await?;
// sqlx doesn't support immediate transactions for sqlite: https://github.com/launchbadge/sqlx/issues/481
sqlx::query!("END; BEGIN IMMEDIATE;")
.execute(&mut *transaction)
// OnceCell::get_or_try_init guarantees exactly-once
// initialization: concurrent callers for the same vault wait
// here; callers for other vaults are not blocked.
let config = self.config.clone();
let vault_clone = vault.clone();
let pool = vault_pool
.cell
.get_or_try_init(|| async {
Self::create_vault_database(&config, &vault_clone).await
})
.await?;
Ok(transaction)
*vault_pool.last_accessed.lock().await = Instant::now();
Ok(pool.clone())
}
pub async fn create_write_transaction(&self, vault: &VaultId) -> Result<WriteTransaction> {
let pool = self.get_connection_pool(vault).await?;
WriteTransaction::new(&pool).await
}
/// Return the latest state of all documents in the vault
pub async fn get_latest_documents(
&self,
vault: &VaultId,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<Vec<DocumentVersionWithoutContent>> {
let query = sqlx::query!(
r#"
@ -204,8 +325,8 @@ impl Database {
"#,
);
if let Some(transaction) = transaction {
query.fetch_all(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_all(&mut *conn).await
} else {
query
.fetch_all(&self.get_connection_pool(vault).await?)
@ -222,9 +343,7 @@ impl Database {
is_deleted: row.is_deleted,
user_id: row.user_id,
device_id: row.device_id,
content_size: row
.content_size
.expect("Content size can't be null but sqlx can't infer it"),
content_size: row.content_size.unwrap_or(0),
})
.collect()
})
@ -236,7 +355,7 @@ impl Database {
&self,
vault: &VaultId,
vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<Vec<DocumentVersionWithoutContent>> {
let query = sqlx::query!(
r#"
@ -256,8 +375,8 @@ impl Database {
vault_update_id
);
if let Some(transaction) = transaction {
query.fetch_all(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_all(&mut *conn).await
} else {
query
.fetch_all(&self.get_connection_pool(vault).await?)
@ -276,9 +395,7 @@ impl Database {
is_deleted: row.is_deleted,
user_id: row.user_id,
device_id: row.device_id,
content_size: row
.content_size
.expect("Content size can't be null but sqlx can't infer it"),
content_size: row.content_size.unwrap_or(0),
})
.collect()
})
@ -287,7 +404,7 @@ impl Database {
pub async fn get_max_update_id_in_vault(
&self,
vault: &VaultId,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<i64> {
let query = sqlx::query!(
r#"
@ -296,8 +413,8 @@ impl Database {
"#,
);
if let Some(transaction) = transaction {
query.fetch_one(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_one(&mut *conn).await
} else {
query
.fetch_one(&self.get_connection_pool(vault).await?)
@ -311,7 +428,7 @@ impl Database {
&self,
vault: &VaultId,
relative_path: &str,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> {
let query = sqlx::query_as!(
StoredDocumentVersion,
@ -337,8 +454,8 @@ impl Database {
relative_path
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_optional(&mut *conn).await
} else {
query
.fetch_optional(&self.get_connection_pool(vault).await?)
@ -351,7 +468,7 @@ impl Database {
&self,
vault: &VaultId,
document_id: &DocumentId,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> {
let document_id = document_id.as_hyphenated();
let query = sqlx::query_as!(
@ -374,8 +491,8 @@ impl Database {
document_id
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_optional(&mut *conn).await
} else {
query
.fetch_optional(&self.get_connection_pool(vault).await?)
@ -388,7 +505,7 @@ impl Database {
&self,
vault: &VaultId,
vault_update_id: VaultUpdateId,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> {
let query = sqlx::query_as!(
StoredDocumentVersion,
@ -409,8 +526,8 @@ impl Database {
vault_update_id
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_optional(&mut *conn).await
} else {
query
.fetch_optional(&self.get_connection_pool(vault).await?)
@ -424,7 +541,7 @@ impl Database {
&self,
vault_id: &VaultId,
version: &StoredDocumentVersion,
transaction: Option<Transaction<'_>>,
transaction: Option<WriteTransaction>,
) -> Result<()> {
let document_id = version.document_id.as_hyphenated();
let query = sqlx::query!(
@ -438,9 +555,10 @@ impl Database {
is_deleted,
user_id,
device_id,
idempotency_key
idempotency_key,
has_been_merged
)
values (?, ?, ?, ?, ?, ?, ?, ?, ?)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"#,
version.vault_update_id,
document_id,
@ -450,7 +568,8 @@ impl Database {
version.is_deleted,
version.user_id,
version.device_id,
version.idempotency_key
version.idempotency_key,
version.has_been_merged
);
if let Some(mut transaction) = transaction {
@ -490,32 +609,36 @@ impl Database {
&self,
vault: &VaultId,
idempotency_key: &str,
transaction: Option<&mut Transaction<'_>>,
connection: Option<&mut SqliteConnection>,
) -> Result<Option<StoredDocumentVersion>> {
// Start from the `documents` table (which has an index on
// `idempotency_key`) to find the document_id, then join to
// `latest_document_versions` for the latest state.
let query = sqlx::query_as!(
StoredDocumentVersion,
r#"
select
d.vault_update_id,
d.document_id as "document_id: Hyphenated",
d.relative_path,
d.updated_date as "updated_date: chrono::DateTime<Utc>",
d.content,
d.is_deleted,
d.user_id,
d.device_id,
d.has_been_merged,
d.idempotency_key
from latest_document_versions d
inner join documents d2 on d.document_id = d2.document_id
where d2.idempotency_key = ?
ldv.vault_update_id,
ldv.document_id as "document_id: Hyphenated",
ldv.relative_path,
ldv.updated_date as "updated_date: chrono::DateTime<Utc>",
ldv.content,
ldv.is_deleted,
ldv.user_id,
ldv.device_id,
ldv.has_been_merged,
ldv.idempotency_key
from documents d
inner join latest_document_versions ldv on d.document_id = ldv.document_id
where d.idempotency_key = ?
order by ldv.vault_update_id desc
limit 1
"#,
idempotency_key
);
if let Some(transaction) = transaction {
query.fetch_optional(&mut **transaction).await
if let Some(conn) = connection {
query.fetch_optional(&mut *conn).await
} else {
query
.fetch_optional(&self.get_connection_pool(vault).await?)
@ -524,39 +647,192 @@ impl Database {
.context("Cannot fetch document by idempotency key")
}
/// Return all versions (without content) of a specific document, ordered by `vault_update_id`
pub async fn get_document_versions(
&self,
vault: &VaultId,
document_id: &DocumentId,
connection: Option<&mut SqliteConnection>,
) -> Result<Vec<DocumentVersionWithoutContent>> {
let document_id = document_id.as_hyphenated();
let query = sqlx::query!(
r#"
select
vault_update_id,
document_id as "document_id: Hyphenated",
relative_path,
updated_date as "updated_date: chrono::DateTime<Utc>",
is_deleted,
user_id,
device_id,
length(content) as "content_size: u64"
from documents
where document_id = ?
order by vault_update_id
"#,
document_id,
);
if let Some(conn) = connection {
query.fetch_all(&mut *conn).await
} else {
query
.fetch_all(&self.get_connection_pool(vault).await?)
.await
}
.with_context(|| format!("Cannot fetch document versions for document `{document_id}`"))
.map(|rows| {
rows.into_iter()
.map(|row| DocumentVersionWithoutContent {
vault_update_id: row.vault_update_id,
document_id: row.document_id.into(),
relative_path: row.relative_path,
updated_date: row.updated_date,
is_deleted: row.is_deleted,
user_id: row.user_id,
device_id: row.device_id,
content_size: row.content_size.unwrap_or(0),
})
.collect()
})
}
/// Return all versions across all documents, paginated, ordered by `vault_update_id` DESC
pub async fn get_vault_history(
&self,
vault: &VaultId,
limit: i64,
before_update_id: Option<VaultUpdateId>,
connection: Option<&mut SqliteConnection>,
) -> Result<Vec<DocumentVersionWithoutContent>> {
let map_row = |row: VaultHistoryRow| DocumentVersionWithoutContent {
vault_update_id: row.vault_update_id,
document_id: row.document_id,
relative_path: row.relative_path,
updated_date: row.updated_date,
is_deleted: row.is_deleted,
user_id: row.user_id,
device_id: row.device_id,
content_size: row.content_size.unwrap_or(0),
};
if let Some(before) = before_update_id {
let query = sqlx::query_as!(
VaultHistoryRow,
r#"
select
vault_update_id,
document_id as "document_id: Hyphenated",
relative_path,
updated_date as "updated_date: chrono::DateTime<Utc>",
is_deleted,
user_id,
device_id,
length(content) as "content_size: u64"
from documents
where vault_update_id < ?
order by vault_update_id desc
limit ?
"#,
before,
limit,
);
let rows = if let Some(conn) = connection {
query.fetch_all(&mut *conn).await
} else {
query
.fetch_all(&self.get_connection_pool(vault).await?)
.await
}
.context("Cannot fetch vault history")?;
Ok(rows.into_iter().map(map_row).collect())
} else {
let query = sqlx::query_as!(
VaultHistoryRow,
r#"
select
vault_update_id,
document_id as "document_id: Hyphenated",
relative_path,
updated_date as "updated_date: chrono::DateTime<Utc>",
is_deleted,
user_id,
device_id,
length(content) as "content_size: u64"
from documents
order by vault_update_id desc
limit ?
"#,
limit,
);
let rows = if let Some(conn) = connection {
query.fetch_all(&mut *conn).await
} else {
query
.fetch_all(&self.get_connection_pool(vault).await?)
.await
}
.context("Cannot fetch vault history")?;
Ok(rows.into_iter().map(map_row).collect())
}
}
/// Cleanup idle connection pools that haven't been accessed in more than 5 minutes
async fn cleanup_idle_pools(&self) {
let mut pools = self.connection_pools.lock().await;
let now = Instant::now();
// Collect idle vaults and remove them from the map while holding
// the lock briefly. Close pools OUTSIDE the lock so that
// pool.close().await doesn't block other get_connection_pool calls.
let idle_pools: Vec<(VaultId, Arc<VaultPool>)> = {
let mut pools = self.connection_pools.lock().await;
let now = Instant::now();
// Collect vaults to remove
let vaults_to_remove: Vec<VaultId> = pools
.iter()
.filter(|(_, pool_with_timestamp)| {
now.duration_since(pool_with_timestamp.last_accessed) > IDLE_POOL_TIMEOUT
})
.map(|(vault_id, _)| vault_id.clone())
.collect();
let vaults_to_remove: Vec<VaultId> = pools
.iter()
.filter(|(_, vp)| {
// If the lock is contested, the pool is actively used — not idle.
let Ok(last) = vp.last_accessed.try_lock() else {
return false;
};
now.duration_since(*last) > IDLE_POOL_TIMEOUT
})
.map(|(vault_id, _)| vault_id.clone())
.collect();
// Close and remove idle pools
for vault_id in &vaults_to_remove {
if let Some(pool_with_timestamp) = pools.remove(vault_id) {
vaults_to_remove
.into_iter()
.filter_map(|id| pools.remove(&id).map(|vp| (id, vp)))
.collect()
};
for (vault_id, vault_pool) in idle_pools {
if let Some(pool) = vault_pool.cell.get() {
info!("Closing idle database connection pool for vault `{vault_id}`");
pool_with_timestamp.pool.close().await;
pool.close().await;
}
}
}
/// Start a background task that periodically cleans up idle connection pools
fn start_idle_pool_cleanup(&self) {
fn start_idle_pool_cleanup(&self, mut shutdown: tokio::sync::watch::Receiver<()>) {
let database = self.clone();
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60)); // Check every minute
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
interval.tick().await;
database.cleanup_idle_pools().await;
tokio::select! {
_ = interval.tick() => {
database.cleanup_idle_pools().await;
}
_ = shutdown.changed() => {
info!("Idle pool cleanup task shutting down");
break;
}
}
}
});
}

View file

@ -0,0 +1,2 @@
CREATE UNIQUE INDEX IF NOT EXISTS idx_documents_idempotency_key
ON documents (idempotency_key) WHERE idempotency_key IS NOT NULL AND is_deleted = 0;

View file

@ -0,0 +1,2 @@
CREATE INDEX IF NOT EXISTS idx_documents_document_id
ON documents (document_id, vault_update_id);

View file

@ -25,6 +25,8 @@ pub struct StoredDocumentVersion {
pub idempotency_key: Option<String>,
}
/// Equality is based solely on `vault_update_id` (the primary key).
/// Two rows with the same PK are the same database record.
impl PartialEq<Self> for StoredDocumentVersion {
fn eq(&self, other: &Self) -> bool {
self.vault_update_id == other.vault_update_id
@ -34,7 +36,7 @@ impl PartialEq<Self> for StoredDocumentVersion {
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersionWithoutContent {
#[ts(as = "i32")]
#[ts(type = "number")]
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,
@ -44,7 +46,7 @@ pub struct DocumentVersionWithoutContent {
pub user_id: UserId,
pub device_id: DeviceId,
#[ts(as = "i32")]
#[ts(type = "number")]
pub content_size: u64,
}
@ -66,7 +68,7 @@ impl From<StoredDocumentVersion> for DocumentVersionWithoutContent {
#[derive(TS, Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct DocumentVersion {
#[ts(as = "i32")]
#[ts(type = "number")]
pub vault_update_id: VaultUpdateId,
pub document_id: DocumentId,

View file

@ -1,24 +1,21 @@
use std::{collections::HashMap, sync::Arc};
use anyhow::Context;
use log::{debug, warn};
use tokio::sync::{Mutex, broadcast};
use super::models::WebSocketServerMessageWithOrigin;
use crate::{
app_state::database::models::VaultId, config::server_config::ServerConfig, errors::server_error,
};
use crate::{app_state::database::models::VaultId, config::server_config::ServerConfig};
#[derive(Debug, Clone)]
pub struct Broadcasts {
max_clients_per_vault: usize,
broadcast_channel_capacity: usize,
tx: Arc<Mutex<HashMap<VaultId, broadcast::Sender<WebSocketServerMessageWithOrigin>>>>,
}
impl Broadcasts {
pub fn new(server_config: &ServerConfig) -> Self {
Self {
max_clients_per_vault: server_config.max_clients_per_vault,
broadcast_channel_capacity: server_config.broadcast_channel_capacity,
tx: Arc::new(Mutex::new(HashMap::new())),
}
}
@ -26,10 +23,25 @@ impl Broadcasts {
pub async fn get_receiver(
&self,
vault: VaultId,
) -> broadcast::Receiver<WebSocketServerMessageWithOrigin> {
let tx = self.get_or_create(vault).await;
max_clients: usize,
) -> Result<broadcast::Receiver<WebSocketServerMessageWithOrigin>, crate::errors::SyncServerError>
{
let mut tx_map = self.tx.lock().await;
tx.subscribe()
// Prune senders for vaults with no active receivers
tx_map.retain(|_, sender| sender.receiver_count() > 0);
let sender = tx_map
.entry(vault)
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
if sender.receiver_count() >= max_clients {
return Err(crate::errors::client_error(anyhow::anyhow!(
"Vault has reached the maximum number of clients ({max_clients})"
)));
}
Ok(sender.subscribe())
}
/// Notify all clients (who are subscribed to the vault) about an update.
@ -39,31 +51,22 @@ impl Broadcasts {
vault: VaultId,
document: WebSocketServerMessageWithOrigin,
) {
let tx = self.get_or_create(vault.clone()).await;
let mut tx_map = self.tx.lock().await;
if tx.receiver_count() == 0 {
// Prune senders for vaults with no active receivers
tx_map.retain(|_, sender| sender.receiver_count() > 0);
let sender = tx_map
.entry(vault.clone())
.or_insert_with(|| broadcast::channel(self.broadcast_channel_capacity).0);
if sender.receiver_count() == 0 {
debug!("Skipping broadcast, no clients connected for vault `{vault}`");
return;
}
let result = tx
.send(document)
.context("Cannot broadcast server message to websocket listeners")
.map_err(server_error);
if result.is_err() {
warn!("Failed to send message: {result:?}");
if let Err(e) = sender.send(document) {
warn!("Failed to broadcast to vault `{vault}`: {e}");
}
}
async fn get_or_create(
&self,
vault: VaultId,
) -> broadcast::Sender<WebSocketServerMessageWithOrigin> {
let mut tx = self.tx.lock().await;
tx.entry(vault)
.or_insert_with(|| broadcast::channel(self.max_clients_per_vault).0.clone())
.clone()
}
}

View file

@ -11,7 +11,7 @@ pub struct WebSocketHandshake {
pub token: String,
pub device_id: DeviceId,
#[ts(as = "Option<i32>")]
#[ts(type = "number | null")]
pub last_seen_vault_update_id: Option<VaultUpdateId>,
}
@ -28,7 +28,7 @@ pub struct DocumentWithCursors {
// that it exists and can be client-side
// interpolated. However, the actual
// position is meaningless.
#[ts(as = "Option<u32>")]
#[ts(type = "number | null")]
pub vault_update_id: Option<VaultUpdateId>,
pub document_id: DocumentId,
@ -70,6 +70,7 @@ pub struct WebSocketVaultUpdate {
pub enum WebSocketClientMessage {
Handshake(WebSocketHandshake),
CursorPositions(CursorPositionFromClient),
Ping {},
}
#[derive(TS, Serialize, Clone, Debug)]

View file

@ -9,7 +9,7 @@ use crate::{
database::models::{DocumentVersionWithoutContent, VaultId, VaultUpdateId},
},
config::user_config::User,
errors::{SyncServerError, server_error, unauthenticated_error},
errors::{SyncServerError, client_error, server_error, unauthenticated_error},
server::auth::auth,
};
@ -26,16 +26,16 @@ pub fn get_authenticated_handshake(
if let Some(Message::Text(message)) = message {
let message: WebSocketClientMessage = serde_json::from_str(&message)
.context("Failed to parse message")
.map_err(server_error)?;
.map_err(client_error)?;
match message {
WebSocketClientMessage::Handshake(handshake) => {
let user = auth(state, handshake.token.trim(), vault_id)?;
Ok(AuthenticatedWebSocketHandshake { handshake, user })
}
WebSocketClientMessage::CursorPositions(_) => Err(unauthenticated_error(
anyhow::anyhow!("Expected a handshake message"),
)),
WebSocketClientMessage::CursorPositions(_) | WebSocketClientMessage::Ping {} => Err(
unauthenticated_error(anyhow::anyhow!("Expected a handshake message")),
),
}
} else {
Err(unauthenticated_error(anyhow::anyhow!(

View file

@ -28,23 +28,20 @@ pub struct Config {
impl Config {
pub async fn read_or_create(path: &Path) -> Result<Self> {
let config = if path.exists() {
info!(
"Loading configuration from `{}`",
path.canonicalize().unwrap().display()
);
Self::load_from_file(path).await?
let display_path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf());
if path.exists() {
info!("Loading configuration from `{}`", display_path.display());
Self::load_from_file(path).await
} else {
Self::default()
};
config.write(path).await?;
info!(
"Updated configuration at `{}`",
path.canonicalize().unwrap().display()
);
Ok(config)
let config = Self::default();
config.write(path).await?;
info!(
"Created default configuration at `{}`",
display_path.display()
);
Ok(config)
}
}
pub async fn load_from_file(path: &Path) -> Result<Self> {

View file

@ -1,10 +1,13 @@
use anyhow::{Result, ensure};
use log::debug;
use serde::{Deserialize, Serialize};
use std::time::Duration;
use crate::consts::{
DEFAULT_HOST, DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT,
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RESPONSE_TIMEOUT_SECONDS,
DEFAULT_ALLOWED_ORIGINS, DEFAULT_BROADCAST_CHANNEL_CAPACITY, DEFAULT_HOST,
DEFAULT_MAX_BODY_SIZE_MB, DEFAULT_MAX_CLIENTS_PER_VAULT, DEFAULT_MAX_PENDING_WS_CONNECTIONS,
DEFAULT_MERGEABLE_FILE_EXTENSIONS, DEFAULT_PORT, DEFAULT_RATE_LIMIT_PER_SECOND,
DEFAULT_RESPONSE_TIMEOUT_SECONDS,
};
#[derive(Debug, Deserialize, Serialize, Clone, Default)]
@ -21,11 +24,68 @@ pub struct ServerConfig {
#[serde(default = "default_max_clients_per_vault")]
pub max_clients_per_vault: usize,
#[serde(default = "default_broadcast_channel_capacity")]
pub broadcast_channel_capacity: usize,
#[serde(default = "default_response_timeout", with = "humantime_serde")]
pub response_timeout: Duration,
#[serde(default = "default_mergeable_file_extensions")]
pub mergeable_file_extensions: Vec<String>,
/// Maximum requests per second (0 = disabled).
#[serde(default = "default_rate_limit_per_second")]
pub rate_limit_per_second: u64,
/// Allowed CORS origins. Default: `["*"]` (allow all).
#[serde(default = "default_allowed_origins")]
pub allowed_origins: Vec<String>,
/// Maximum concurrent unauthenticated WebSocket connections waiting for
/// handshake. Limits resource consumption from clients that connect but
/// never authenticate.
#[serde(default = "default_max_pending_websocket_connections")]
pub max_pending_websocket_connections: usize,
/// When set, proxies all UI requests (index, assets, Vite HMR) to this
/// URL instead of serving embedded assets. Typically
/// `http://localhost:5173` for the Vite dev server.
#[serde(default)]
pub dev_proxy_url: Option<String>,
}
impl ServerConfig {
pub fn validate(&self) -> Result<()> {
ensure!(
!self.response_timeout.is_zero(),
"response_timeout must be greater than 0"
);
ensure!(
self.max_body_size_mb > 0,
"max_body_size_mb must be greater than 0"
);
ensure!(
self.max_clients_per_vault > 0,
"max_clients_per_vault must be greater than 0"
);
ensure!(
self.broadcast_channel_capacity > 0,
"broadcast_channel_capacity must be greater than 0"
);
ensure!(
self.max_pending_websocket_connections > 0,
"max_pending_websocket_connections must be greater than 0"
);
ensure!(
self.max_clients_per_vault <= 10_000,
"max_clients_per_vault must be at most 10000"
);
ensure!(
self.broadcast_channel_capacity <= 1_000_000,
"broadcast_channel_capacity must be at most 1000000"
);
Ok(())
}
}
fn default_host() -> String {
@ -48,6 +108,11 @@ fn default_max_clients_per_vault() -> usize {
DEFAULT_MAX_CLIENTS_PER_VAULT
}
fn default_broadcast_channel_capacity() -> usize {
debug!("Using default broadcast channel capacity: {DEFAULT_BROADCAST_CHANNEL_CAPACITY}");
DEFAULT_BROADCAST_CHANNEL_CAPACITY
}
fn default_response_timeout() -> Duration {
debug!("Using default response timeout: {DEFAULT_RESPONSE_TIMEOUT_SECONDS:?}");
DEFAULT_RESPONSE_TIMEOUT_SECONDS
@ -60,3 +125,23 @@ fn default_mergeable_file_extensions() -> Vec<String> {
.map(|s| (*s).to_owned())
.collect()
}
fn default_rate_limit_per_second() -> u64 {
debug!("Using default rate limit per second: {DEFAULT_RATE_LIMIT_PER_SECOND}");
DEFAULT_RATE_LIMIT_PER_SECOND
}
fn default_allowed_origins() -> Vec<String> {
debug!("Using default allowed origins: {DEFAULT_ALLOWED_ORIGINS:?}");
DEFAULT_ALLOWED_ORIGINS
.iter()
.map(|s| (*s).to_owned())
.collect()
}
fn default_max_pending_websocket_connections() -> usize {
debug!(
"Using default max pending WebSocket connections: {DEFAULT_MAX_PENDING_WS_CONNECTIONS}"
);
DEFAULT_MAX_PENDING_WS_CONNECTIONS
}

View file

@ -19,10 +19,19 @@ where
let mut user_token_map = BiHashMap::new();
for user in &users {
if let Some(existing_name) = user_token_map.get_by_right(&user.token) {
let redacted = if user.token.len() > 6 {
format!(
"{}...{}",
&user.token[..3],
&user.token[user.token.len() - 3..]
)
} else {
"***".to_owned()
};
return Err(D::Error::custom(format!(
"Duplicate user token found: `{}` for users `{}` and `{}`. User tokens must be \
unique.",
user.token, existing_name, user.name
"Duplicate user token found: `{redacted}` for users `{}` and `{}`. User tokens \
must be unique.",
existing_name, user.name
)));
}
@ -41,10 +50,23 @@ where
impl UserConfig {
pub fn get_user(&self, token: &str) -> Option<&User> {
self.user_configs.iter().find(|u| u.token == token)
self.user_configs
.iter()
.find(|u| constant_time_eq(u.token.as_bytes(), token.as_bytes()))
}
}
/// Constant-time byte comparison to prevent timing attacks on token lookups.
fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
if a.len() != b.len() {
return false;
}
a.iter()
.zip(b.iter())
.fold(0u8, |acc, (x, y)| acc | (x ^ y))
== 0
}
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct User {
pub name: String,

View file

@ -13,12 +13,21 @@ pub const DEFAULT_PORT: u16 = 3000;
pub const DEFAULT_MAX_BODY_SIZE_MB: usize = 4096;
pub const DEFAULT_RESPONSE_TIMEOUT_SECONDS: Duration = Duration::from_mins(30);
pub const DEFAULT_MAX_CLIENTS_PER_VAULT: usize = 256;
pub const DEFAULT_BROADCAST_CHANNEL_CAPACITY: usize = 4096;
pub const DEFAULT_MAX_PENDING_WS_CONNECTIONS: usize = 128;
pub const DEFAULT_LOG_DIRECTORY: &str = "logs";
pub const DEFAULT_LOG_ROTATION_INTERVAL: Duration = Duration::from_hours(24);
pub const IDLE_POOL_TIMEOUT: Duration = Duration::from_mins(5);
pub const GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10);
pub const DEFAULT_LOG_LEVEL: LogLevel = LogLevel::Info;
pub const DEFAULT_MERGEABLE_FILE_EXTENSIONS: &[&str] = &["md", "txt"];
/// 0 means rate limiting is disabled.
pub const DEFAULT_RATE_LIMIT_PER_SECOND: u64 = 0;
/// Default: allow all origins.
pub const DEFAULT_ALLOWED_ORIGINS: &[&str] = &["*"];
pub const SUPPORTED_API_VERSION: u32 = 3;

View file

@ -5,7 +5,7 @@ use axum::{
http::StatusCode,
response::{IntoResponse, Response},
};
use log::debug;
use log::{debug, error, warn};
use serde::Serialize;
use thiserror::Error;
use ts_rs::TS;
@ -69,7 +69,19 @@ impl Display for SerializedError {
impl IntoResponse for SyncServerError {
fn into_response(self) -> Response {
let body = Json(self.serialize());
let serialized = self.serialize();
match &self {
Self::InitError(_) | Self::ServerError(_) => {
error!("{serialized}");
}
Self::ClientError(_) | Self::NotFound(_) => {
warn!("{serialized}");
}
Self::Unauthenticated(_) | Self::PermissionDeniedError(_) => {}
}
let body = Json(serialized);
match self {
Self::InitError(_) | Self::ServerError(_) => {

View file

@ -41,7 +41,15 @@ async fn main() -> ExitCode {
}
};
let mut result = set_up_logging(&args, &config.logging);
let mut result = config
.server
.validate()
.context("Invalid server configuration")
.map_err(init_error);
if result.is_ok() {
result = set_up_logging(&args, &config.logging);
}
if result.is_ok() {
result = start_server(config).await;

Some files were not shown because too many files have changed in this diff Show more