This commit is contained in:
Andras Schmelczer 2026-09-27 08:29:17 +01:00
commit 3dab48f58b
11 changed files with 1141 additions and 1376 deletions

View file

@ -1 +0,0 @@
DATABASE_URL=sqlite://db.sqlite3

View file

@ -21,7 +21,7 @@ tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout
tracing = "0.1.41"
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
humantime-serde = "1.1.1"
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
sqlx = { version = "0.8.6", default-features = false, features = ["sqlite", "runtime-tokio", "uuid", "chrono", "macros"] }
chrono = { version = "0.4.41", features = ["serde"] }
rand = "0.9.0"
regex = "1.12.2"

View file

@ -5,14 +5,11 @@ WORKDIR /usr/src/backend
RUN apt update && \
apt install -y \
libssl-dev \
pkg-config && \
cargo install sqlx-cli
pkg-config
COPY . .
RUN sqlx database create --database-url sqlite://db.sqlite3 && \
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 && \
cargo build --release
RUN cargo build --release
FROM debian:trixie-slim

View file

@ -127,11 +127,12 @@ flushes database directory entries; the filesystem must support those durability
operations. Content, file manifests, and events (including request fingerprints)
are retained indefinitely.
Deploy API v5 clients and server together. The v4 database schema migrates by
adding event-incarnation tokens without deleting content or receipts. Schemas
from before v4 still require a separate database directory and explicit recovery.
Vault databases open and migrate independently on first access; one corrupt or
incompatible vault does not prevent healthy vaults from serving requests.
Deploy API v5 clients and server together. Only the current database schema is
supported; there are no schema migrations. Empty vault databases are initialized
from `src/app_state/database/schema.sql` on first access. Existing databases are
opened without conversion. Use fresh database and client state directories when
changing formats. Each vault opens independently, so one corrupt vault does not
prevent healthy vaults from serving requests.
Diffs are limited to 10,000 items and reconstructed in one pass over Unicode
scalars on a blocking worker, before taking the SQLite writer transaction. The

View file

@ -1,5 +0,0 @@
// generated by `sqlx migrate build-script`
fn main() {
// trigger recompilation when a new migration is added
println!("cargo:rerun-if-changed=src/app_state/database/migrations");
}

View file

@ -9,7 +9,7 @@ use anyhow::{Context as _, Result};
use log::info;
use models::VaultId;
use sha2::{Digest, Sha256};
use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions};
use sqlx::{ConnectOptions, Executor, sqlite::SqliteConnectOptions};
use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions};
use tokio::sync::Mutex;
use tokio::time::Instant;
@ -115,7 +115,7 @@ impl Database {
.await
.with_context(|| format!("Cannot open database at `{}`", file_name.display()))?;
Self::run_migrations(&pool).await?;
Self::initialize_schema(&pool).await?;
if let Some(directory) = file_name.parent() {
#[cfg(unix)]
@ -138,11 +138,21 @@ impl Database {
Ok(())
}
async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
sqlx::migrate!("src/app_state/database/migrations")
.run(pool)
async fn initialize_schema(pool: &Pool<Sqlite>) -> Result<()> {
let mut tx = pool.begin_with("BEGIN IMMEDIATE").await?;
let initialized: bool = sqlx::query_scalar(
"SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%')",
)
.fetch_one(&mut *tx)
.await?;
if !initialized {
tx.execute(include_str!("database/schema.sql"))
.await
.context("Cannot initialize vault database")?;
}
tx.commit()
.await
.context("Cannot check for pending migrations")
.context("Cannot commit vault database initialization")
}
async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> {

View file

@ -125,9 +125,8 @@ pub struct EventBatch {
pub head_event_id: VaultUpdateId,
/// Last event in this page; `head_event_id` is the vault's current head.
#[serde(skip_serializing_if = "Option::is_none")]
#[ts(optional, as = "Option<f64>")]
pub end_event_id: Option<VaultUpdateId>,
#[ts(as = "f64")]
pub end_event_id: VaultUpdateId,
pub events: Vec<EventRecord>,
}

View file

@ -233,7 +233,7 @@ impl Database {
Ok(EventBatch {
head_event_id,
end_event_id: Some(end_event_id),
end_event_id,
events,
})
}

View file

@ -0,0 +1,44 @@
CREATE TABLE events (
-- Numeric position in the vault's history: orders events, serves as
-- vault_update_id/file_manifest_id, parent IDs, and page cursors
event_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(event_id <= 9007199254740991), -- JavaScript’s Number.MAX_SAFE_INTEGER
-- Server-minted random identity of this event's incarnation, minted once
-- per INSERT. Unlike request_id it is never reproduced by a resubmitted
-- request, so the history checkpoint `event_id:event_token` fails whenever
-- a restored backup reuses a numeric ID or a client resubmits the same
-- request UUID against a different history
event_token TEXT NOT NULL,
-- Client-chosen idempotency key, stable across retries so the server can
-- deduplicate a replayed request
request_id TEXT NOT NULL UNIQUE,
-- Hash of the request payload that carried this event
request_fingerprint BLOB NOT NULL
);
CREATE TABLE documents (
vault_update_id INTEGER PRIMARY KEY REFERENCES events(event_id),
document_id TEXT NOT NULL,
updated_date TEXT NOT NULL,
content BLOB NOT NULL,
user_id TEXT NOT NULL,
device_id TEXT NOT NULL
);
CREATE INDEX documents_by_id ON documents(document_id);
CREATE VIEW latest_document_versions AS
SELECT d.* FROM documents d JOIN (
SELECT document_id, MAX(vault_update_id) AS version FROM documents GROUP BY document_id
) heads ON d.vault_update_id = heads.version;
CREATE TABLE file_manifests (
file_manifest_id INTEGER PRIMARY KEY REFERENCES events(event_id)
);
CREATE TABLE file_manifest_entries (
file_manifest_id INTEGER NOT NULL REFERENCES file_manifests(file_manifest_id),
document_id TEXT NOT NULL,
path TEXT NOT NULL,
PRIMARY KEY (file_manifest_id, document_id),
UNIQUE (file_manifest_id, path)
);

View file

@ -96,15 +96,15 @@ async fn rejects_invalid_vault_names_at_the_database_boundary() {
}
#[tokio::test]
async fn migrations_can_be_reapplied_without_losing_requests_or_content() {
async fn schema_initialization_preserves_existing_requests_and_content() {
let directory = tempfile::tempdir().unwrap();
let config = config(&directory);
let database = Database::try_new(&config).await.unwrap();
let vault = "migrations".to_owned();
let vault = "initialization".to_owned();
let version = insert_content(&database, &vault, uuid::Uuid::new_v4(), b"only copy").await;
let pool = database.get_connection_pool(&vault).await.unwrap();
Database::run_migrations(&pool).await.unwrap();
Database::run_migrations(&pool).await.unwrap();
Database::initialize_schema(&pool).await.unwrap();
Database::initialize_schema(&pool).await.unwrap();
let reopened = Database::try_new(&config).await.unwrap();
assert_eq!(
reopened
@ -307,7 +307,7 @@ async fn replay_byte_budget_prevents_accumulating_large_historical_manifests() {
assert!(serde_json::to_vec(&batch).unwrap().len() < MAX_EVENTS_PAGE_BYTES + 1024);
assert_eq!(
batch.end_event_id,
batch.events.last().map(|event| event.event_id)
batch.events.last().unwrap().event_id
);
}
@ -318,8 +318,8 @@ async fn opening_a_locked_vault_does_not_block_unrelated_vaults() {
let database = Database::try_new(&config).await.unwrap();
database.vault_snapshot(&"other".to_owned()).await.unwrap();
let vault = "locked-opening".to_owned();
// A second process holds the writer lock while this process lazily checks
// migrations. Even already-open, unrelated vaults must remain available.
// A second process holds the writer lock while this process initializes
// the schema. Even already-open, unrelated vaults must remain available.
let external = Database::open_database(&config, &Database::database_path(&config, &vault))
.await
.unwrap();
@ -346,7 +346,7 @@ async fn opening_a_locked_vault_does_not_block_unrelated_vaults() {
);
assert_eq!(
other
.expect("a blocked migration must not lock every vault")
.expect("a blocked database open must not lock every vault")
.unwrap()
.head_event_id,
0

View file

@ -20,7 +20,7 @@ pub struct PingResponse {
/// List of file extensions that are allowed to be merged.
pub mergeable_file_extensions: Vec<String>,
/// API version ensuring backwards & forwards compatibility between the client
/// API version that must match exactly between the client
/// and server.
pub supported_api_version: u32,
}