,,
This commit is contained in:
parent
0c75d3b4c2
commit
3dab48f58b
11 changed files with 1141 additions and 1376 deletions
|
|
@ -1 +0,0 @@
|
|||
DATABASE_URL=sqlite://db.sqlite3
|
||||
|
|
@ -21,7 +21,7 @@ tower-http = { version = "0.6.1", features = ["cors", "trace", "limit", "timeout
|
|||
tracing = "0.1.41"
|
||||
tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"]}
|
||||
humantime-serde = "1.1.1"
|
||||
sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "uuid", "chrono"] }
|
||||
sqlx = { version = "0.8.6", default-features = false, features = ["sqlite", "runtime-tokio", "uuid", "chrono", "macros"] }
|
||||
chrono = { version = "0.4.41", features = ["serde"] }
|
||||
rand = "0.9.0"
|
||||
regex = "1.12.2"
|
||||
|
|
|
|||
|
|
@ -5,14 +5,11 @@ WORKDIR /usr/src/backend
|
|||
RUN apt update && \
|
||||
apt install -y \
|
||||
libssl-dev \
|
||||
pkg-config && \
|
||||
cargo install sqlx-cli
|
||||
pkg-config
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN sqlx database create --database-url sqlite://db.sqlite3 && \
|
||||
sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3 && \
|
||||
cargo build --release
|
||||
RUN cargo build --release
|
||||
|
||||
FROM debian:trixie-slim
|
||||
|
||||
|
|
|
|||
|
|
@ -127,11 +127,12 @@ flushes database directory entries; the filesystem must support those durability
|
|||
operations. Content, file manifests, and events (including request fingerprints)
|
||||
are retained indefinitely.
|
||||
|
||||
Deploy API v5 clients and server together. The v4 database schema migrates by
|
||||
adding event-incarnation tokens without deleting content or receipts. Schemas
|
||||
from before v4 still require a separate database directory and explicit recovery.
|
||||
Vault databases open and migrate independently on first access; one corrupt or
|
||||
incompatible vault does not prevent healthy vaults from serving requests.
|
||||
Deploy API v5 clients and server together. Only the current database schema is
|
||||
supported; there are no schema migrations. Empty vault databases are initialized
|
||||
from `src/app_state/database/schema.sql` on first access. Existing databases are
|
||||
opened without conversion. Use fresh database and client state directories when
|
||||
changing formats. Each vault opens independently, so one corrupt vault does not
|
||||
prevent healthy vaults from serving requests.
|
||||
|
||||
Diffs are limited to 10,000 items and reconstructed in one pass over Unicode
|
||||
scalars on a blocking worker, before taking the SQLite writer transaction. The
|
||||
|
|
|
|||
|
|
@ -1,5 +0,0 @@
|
|||
// generated by `sqlx migrate build-script`
|
||||
fn main() {
|
||||
// trigger recompilation when a new migration is added
|
||||
println!("cargo:rerun-if-changed=src/app_state/database/migrations");
|
||||
}
|
||||
|
|
@ -9,7 +9,7 @@ use anyhow::{Context as _, Result};
|
|||
use log::info;
|
||||
use models::VaultId;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sqlx::{ConnectOptions, sqlite::SqliteConnectOptions};
|
||||
use sqlx::{ConnectOptions, Executor, sqlite::SqliteConnectOptions};
|
||||
use sqlx::{Pool, Sqlite, sqlite::SqlitePoolOptions};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::Instant;
|
||||
|
|
@ -115,7 +115,7 @@ impl Database {
|
|||
.await
|
||||
.with_context(|| format!("Cannot open database at `{}`", file_name.display()))?;
|
||||
|
||||
Self::run_migrations(&pool).await?;
|
||||
Self::initialize_schema(&pool).await?;
|
||||
|
||||
if let Some(directory) = file_name.parent() {
|
||||
#[cfg(unix)]
|
||||
|
|
@ -138,11 +138,21 @@ impl Database {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_migrations(pool: &Pool<Sqlite>) -> Result<()> {
|
||||
sqlx::migrate!("src/app_state/database/migrations")
|
||||
.run(pool)
|
||||
async fn initialize_schema(pool: &Pool<Sqlite>) -> Result<()> {
|
||||
let mut tx = pool.begin_with("BEGIN IMMEDIATE").await?;
|
||||
let initialized: bool = sqlx::query_scalar(
|
||||
"SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%')",
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
if !initialized {
|
||||
tx.execute(include_str!("database/schema.sql"))
|
||||
.await
|
||||
.context("Cannot initialize vault database")?;
|
||||
}
|
||||
tx.commit()
|
||||
.await
|
||||
.context("Cannot check for pending migrations")
|
||||
.context("Cannot commit vault database initialization")
|
||||
}
|
||||
|
||||
async fn get_connection_pool(&self, vault: &VaultId) -> Result<Pool<Sqlite>> {
|
||||
|
|
|
|||
|
|
@ -125,9 +125,8 @@ pub struct EventBatch {
|
|||
pub head_event_id: VaultUpdateId,
|
||||
|
||||
/// Last event in this page; `head_event_id` is the vault's current head.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[ts(optional, as = "Option<f64>")]
|
||||
pub end_event_id: Option<VaultUpdateId>,
|
||||
#[ts(as = "f64")]
|
||||
pub end_event_id: VaultUpdateId,
|
||||
|
||||
pub events: Vec<EventRecord>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -233,7 +233,7 @@ impl Database {
|
|||
|
||||
Ok(EventBatch {
|
||||
head_event_id,
|
||||
end_event_id: Some(end_event_id),
|
||||
end_event_id,
|
||||
events,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
44
sync-server/src/app_state/database/schema.sql
Normal file
44
sync-server/src/app_state/database/schema.sql
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
CREATE TABLE events (
|
||||
-- Numeric position in the vault's history: orders events, serves as
|
||||
-- vault_update_id/file_manifest_id, parent IDs, and page cursors
|
||||
event_id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(event_id <= 9007199254740991), -- JavaScript’s Number.MAX_SAFE_INTEGER
|
||||
-- Server-minted random identity of this event's incarnation, minted once
|
||||
-- per INSERT. Unlike request_id it is never reproduced by a resubmitted
|
||||
-- request, so the history checkpoint `event_id:event_token` fails whenever
|
||||
-- a restored backup reuses a numeric ID or a client resubmits the same
|
||||
-- request UUID against a different history
|
||||
event_token TEXT NOT NULL,
|
||||
-- Client-chosen idempotency key, stable across retries so the server can
|
||||
-- deduplicate a replayed request
|
||||
request_id TEXT NOT NULL UNIQUE,
|
||||
-- Hash of the request payload that carried this event
|
||||
request_fingerprint BLOB NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE documents (
|
||||
vault_update_id INTEGER PRIMARY KEY REFERENCES events(event_id),
|
||||
document_id TEXT NOT NULL,
|
||||
updated_date TEXT NOT NULL,
|
||||
content BLOB NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX documents_by_id ON documents(document_id);
|
||||
|
||||
CREATE VIEW latest_document_versions AS
|
||||
SELECT d.* FROM documents d JOIN (
|
||||
SELECT document_id, MAX(vault_update_id) AS version FROM documents GROUP BY document_id
|
||||
) heads ON d.vault_update_id = heads.version;
|
||||
|
||||
CREATE TABLE file_manifests (
|
||||
file_manifest_id INTEGER PRIMARY KEY REFERENCES events(event_id)
|
||||
);
|
||||
|
||||
CREATE TABLE file_manifest_entries (
|
||||
file_manifest_id INTEGER NOT NULL REFERENCES file_manifests(file_manifest_id),
|
||||
document_id TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
PRIMARY KEY (file_manifest_id, document_id),
|
||||
UNIQUE (file_manifest_id, path)
|
||||
);
|
||||
|
|
@ -96,15 +96,15 @@ async fn rejects_invalid_vault_names_at_the_database_boundary() {
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migrations_can_be_reapplied_without_losing_requests_or_content() {
|
||||
async fn schema_initialization_preserves_existing_requests_and_content() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let config = config(&directory);
|
||||
let database = Database::try_new(&config).await.unwrap();
|
||||
let vault = "migrations".to_owned();
|
||||
let vault = "initialization".to_owned();
|
||||
let version = insert_content(&database, &vault, uuid::Uuid::new_v4(), b"only copy").await;
|
||||
let pool = database.get_connection_pool(&vault).await.unwrap();
|
||||
Database::run_migrations(&pool).await.unwrap();
|
||||
Database::run_migrations(&pool).await.unwrap();
|
||||
Database::initialize_schema(&pool).await.unwrap();
|
||||
Database::initialize_schema(&pool).await.unwrap();
|
||||
let reopened = Database::try_new(&config).await.unwrap();
|
||||
assert_eq!(
|
||||
reopened
|
||||
|
|
@ -307,7 +307,7 @@ async fn replay_byte_budget_prevents_accumulating_large_historical_manifests() {
|
|||
assert!(serde_json::to_vec(&batch).unwrap().len() < MAX_EVENTS_PAGE_BYTES + 1024);
|
||||
assert_eq!(
|
||||
batch.end_event_id,
|
||||
batch.events.last().map(|event| event.event_id)
|
||||
batch.events.last().unwrap().event_id
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -318,8 +318,8 @@ async fn opening_a_locked_vault_does_not_block_unrelated_vaults() {
|
|||
let database = Database::try_new(&config).await.unwrap();
|
||||
database.vault_snapshot(&"other".to_owned()).await.unwrap();
|
||||
let vault = "locked-opening".to_owned();
|
||||
// A second process holds the writer lock while this process lazily checks
|
||||
// migrations. Even already-open, unrelated vaults must remain available.
|
||||
// A second process holds the writer lock while this process initializes
|
||||
// the schema. Even already-open, unrelated vaults must remain available.
|
||||
let external = Database::open_database(&config, &Database::database_path(&config, &vault))
|
||||
.await
|
||||
.unwrap();
|
||||
|
|
@ -346,7 +346,7 @@ async fn opening_a_locked_vault_does_not_block_unrelated_vaults() {
|
|||
);
|
||||
assert_eq!(
|
||||
other
|
||||
.expect("a blocked migration must not lock every vault")
|
||||
.expect("a blocked database open must not lock every vault")
|
||||
.unwrap()
|
||||
.head_event_id,
|
||||
0
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pub struct PingResponse {
|
|||
/// List of file extensions that are allowed to be merged.
|
||||
pub mergeable_file_extensions: Vec<String>,
|
||||
|
||||
/// API version ensuring backwards & forwards compatibility between the client
|
||||
/// API version that must match exactly between the client
|
||||
/// and server.
|
||||
pub supported_api_version: u32,
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue