diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 4d40e80e..153bd4b5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "chrono" @@ -575,12 +575,6 @@ dependencies = [ "serde", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "digest" version = "0.10.7" @@ -1596,16 +1590,6 @@ dependencies = [ "zerocopy 0.7.35", ] -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1715,14 +1699,12 @@ dependencies = [ ] [[package]] -name = "reconcile" -version = "0.4.0" +name = "reconcile-text" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54fa4679b1042b1110aeac9c00fe292339af66426833da724a3fcaae0052b4da" dependencies = [ - "insta", - "pretty_assertions", - "serde", - "serde_yaml", - "test-case", + "cfg-if", ] [[package]] @@ -2318,7 +2300,6 @@ dependencies = [ "base64 0.22.1", "console_error_panic_hook", "insta", - "reconcile", "thiserror 2.0.12", "wasm-bindgen", "wasm-bindgen-test", @@ -2339,6 +2320,7 @@ dependencies = [ "futures", "log", "rand 0.9.0", + "reconcile-text", "regex", "sanitize-filename", "serde", @@ -2395,39 +2377,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "test-case" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" -dependencies = [ - "test-case-macros", -] - -[[package]] -name = "test-case-core" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 2.0.90", -] - -[[package]] -name = "test-case-macros" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.90", - "test-case-core", -] - [[package]] name = "thiserror" version = "1.0.69" @@ -3203,12 +3152,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.7.5" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5fb4a5a5..738dbdd3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,7 +1,6 @@ [workspace] resolver = "2" members = [ - "reconcile", "sync_server", "sync_lib" ] @@ -59,11 +58,6 @@ verbose_file_reads = "warn" large_stack_arrays = { level = "allow", priority = 1 } # https://github.com/rust-lang/rust-clippy/issues/13774 -# TODO: fix these -cast_possible_truncation = { level = "allow", priority = 1 } -cast_sign_loss = { level = "allow", priority = 1 } -cast_possible_wrap = { level = "allow", priority = 1 } - # Silly lints implicit_return = { level = "allow", priority = 1 } question_mark_used = { level = "allow", priority = 1 } diff --git a/backend/sync_lib/Cargo.toml b/backend/sync_lib/Cargo.toml index 72f3de87..6147a357 100644 --- a/backend/sync_lib/Cargo.toml +++ b/backend/sync_lib/Cargo.toml @@ -11,7 +11,6 @@ crate-type = ["cdylib", "rlib"] [dependencies] base64 = "0.22.1" -reconcile = { path = "../reconcile" } wasm-bindgen = "0.2.99" thiserror = { workspace = true } diff --git a/backend/sync_lib/src/cursor.rs b/backend/sync_lib/src/cursor.rs deleted file mode 100644 index 2f7135eb..00000000 --- a/backend/sync_lib/src/cursor.rs +++ /dev/null @@ -1,88 +0,0 @@ -use wasm_bindgen::prelude::*; - -/// Wrapper type to expose `TextWithCursors` to JS. -#[wasm_bindgen] -#[derive(Debug, Clone, PartialEq)] -pub struct TextWithCursors { - text: String, - cursors: Vec, -} - -#[wasm_bindgen] -impl TextWithCursors { - #[wasm_bindgen(constructor)] - #[must_use] - pub fn new(text: String, cursors: Vec) -> Self { Self { text, cursors } } - - #[must_use] - pub fn text(&self) -> String { self.text.clone() } - - #[must_use] - pub fn cursors(&self) -> Vec { self.cursors.clone() } -} - -impl From for reconcile::TextWithCursors<'_> { - fn from(owned: TextWithCursors) -> Self { - reconcile::TextWithCursors::new_owned( - owned.text.to_string(), - owned - .cursors - .into_iter() - .map(std::convert::Into::into) - .collect(), - ) - } -} - -impl From> for TextWithCursors { - fn from(text_with_cursors: reconcile::TextWithCursors<'_>) -> Self { - TextWithCursors { - text: text_with_cursors.text.into_owned(), - cursors: text_with_cursors - .cursors - .into_iter() - .map(std::convert::Into::into) - .collect(), - } - } -} - -/// Wrapper type to expose `CursorPosition` to JS. -#[wasm_bindgen] -#[derive(Debug, Clone, PartialEq)] -pub struct CursorPosition { - id: usize, - char_index: usize, -} - -#[wasm_bindgen] -impl CursorPosition { - #[wasm_bindgen(constructor)] - #[must_use] - pub fn new(id: usize, char_index: usize) -> Self { Self { id, char_index } } - - #[must_use] - pub fn id(&self) -> usize { self.id } - - #[wasm_bindgen(js_name = characterPosition)] - #[must_use] - pub fn char_index(&self) -> usize { self.char_index } -} - -impl From for reconcile::CursorPosition { - fn from(owned: CursorPosition) -> Self { - reconcile::CursorPosition { - id: owned.id, - char_index: owned.char_index, - } - } -} - -impl From for CursorPosition { - fn from(cursor: reconcile::CursorPosition) -> Self { - CursorPosition { - id: cursor.id, - char_index: cursor.char_index, - } - } -} diff --git a/backend/sync_lib/src/lib.rs b/backend/sync_lib/src/lib.rs index d2a54cf9..437ce9bd 100644 --- a/backend/sync_lib/src/lib.rs +++ b/backend/sync_lib/src/lib.rs @@ -1,6 +1,6 @@ //! This crate provides utilities for easily communicating between backend & //! frontend and ensuring the same logic for encoding and decoding binary data, -//! and 3-way-merging documents in Rust and JavaScript. +//! and filtering files. //! //! The crate is designed to be used as a Rust library and as a //! TypeScript/JavaScript package through WebAssembly (WASM). @@ -12,11 +12,9 @@ use core::str; use base64::{Engine as _, engine::general_purpose::STANDARD}; -use cursor::TextWithCursors; use errors::SyncLibError; use wasm_bindgen::prelude::*; -pub mod cursor; pub mod errors; /// Encode binary data for easy transport over HTTP. Inverse of @@ -62,78 +60,6 @@ pub fn base64_to_bytes(input: &str) -> Result, SyncLibError> { STANDARD.decode(input).map_err(SyncLibError::from) } -/// Merge two documents with a common parent. Relies on `reconcile::reconcile` -/// for texts and returns the right document as-is if either of the updated -/// documents is binary. -/// -/// # Arguments -/// -/// - `parent`: The common parent document. -/// - `left`: The left document updated by one user. -/// - `right`: The right document updated by another user. -/// -/// # Returns -/// -/// The merged document. -/// -/// # Panics -/// -/// If any of the input documents are not valid UTF-8 strings. -#[wasm_bindgen] -#[must_use] -pub fn merge(parent: &[u8], left: &[u8], right: &[u8]) -> Vec { - set_panic_hook(); - - if is_binary(parent) || is_binary(left) || is_binary(right) { - right.to_vec() - } else { - reconcile::reconcile( - str::from_utf8(parent).expect("parent must be valid UTF-8 because it's not binary"), - str::from_utf8(left).expect("left must be valid UTF-8 because it's not binary"), - str::from_utf8(right).expect("right must be valid UTF-8 because it's not binary"), - ) - .into_bytes() - } -} - -/// WASM wrapper around `reconcile::reconcile` for merging text. -#[wasm_bindgen(js_name = mergeText)] -#[must_use] -pub fn merge_text(parent: &str, left: &str, right: &str) -> String { - set_panic_hook(); - - reconcile::reconcile(parent, left, right) -} - -/// WASM wrapper around `reconcile::reconcile_with_cursors` for merging text. -#[wasm_bindgen(js_name = mergeTextWithCursors)] -#[must_use] -pub fn merge_text_with_cursors( - parent: &str, - left: TextWithCursors, - right: TextWithCursors, -) -> TextWithCursors { - set_panic_hook(); - - reconcile::reconcile_with_cursors(parent, left.into(), right.into()).into() -} - -/// Heuristically determine if the given data is a binary or a text file's -/// content. -#[wasm_bindgen(js_name = isBinary)] -#[must_use] -pub fn is_binary(data: &[u8]) -> bool { - set_panic_hook(); - - if data.contains(&0) { - // Even though the NUL character is valid in UTF-8, it's highly suspicious in - // human-readable text. - return true; - } - - std::str::from_utf8(data).is_err() -} - /// We don't want to support merging structured data like JSON, YAML, etc. #[wasm_bindgen(js_name = isFileTypeMergable)] #[must_use] diff --git a/backend/sync_lib/tests/web.rs b/backend/sync_lib/tests/web.rs index cf82aa7e..42aa5309 100644 --- a/backend/sync_lib/tests/web.rs +++ b/backend/sync_lib/tests/web.rs @@ -1,8 +1,5 @@ use insta::assert_debug_snapshot; -use sync_lib::{ - cursor::{CursorPosition, TextWithCursors}, - *, -}; +use sync_lib::*; use wasm_bindgen_test::*; #[wasm_bindgen_test(unsupported = test)] @@ -25,66 +22,6 @@ fn test_base64_to_bytes_error() { assert_debug_snapshot!(base64_to_bytes(input)); } -#[wasm_bindgen_test(unsupported = test)] -fn test_merge() { - let left = b"hello "; - let right = b"world"; - let result = merge(b"", left, right); - assert_eq!(result, b"hello world"); - - let left = b"\0binary"; - let right = b"other"; - let result = merge(b"", left, right); - assert_eq!(result, right); -} - -#[wasm_bindgen_test(unsupported = test)] -fn test_merge_text() { - let left = "hello "; - let right = "world"; - let result = merge_text("", left, right); - assert_eq!(result, "hello world"); -} - -#[wasm_bindgen_test(unsupported = test)] -fn test_merge_text_with_cursors() { - let result = merge_text_with_cursors( - "hi", - TextWithCursors::new("hi world".to_owned(), vec![]), - TextWithCursors::new( - "hi".to_owned(), - vec![CursorPosition::new(0, 1), CursorPosition::new(1, 2)], - ), - ); - - assert_eq!( - result, - TextWithCursors::new( - "hi world".to_owned(), - vec![CursorPosition::new(0, 1), CursorPosition::new(1, 2)] - ), - ); -} - -#[wasm_bindgen_test(unsupported = test)] -fn merge_binary() { - let left = [0, 1, 2]; - let right = [3, 4, 5]; - assert_eq!(merge(b"", &left, &right), right); -} - -#[wasm_bindgen_test(unsupported = test)] -fn test_is_binary() { - assert!(is_binary(&[0, 159, 146, 150])); - assert!(is_binary(&[0, 12])); - assert!(!is_binary(b"hello")); -} - -#[wasm_bindgen_test(unsupported = test)] -fn test_is_binary_empty() { - assert!(!is_binary(b"")); -} - #[wasm_bindgen_test(unsupported = test)] fn test_is_file_type_mergable() { assert!(is_file_type_mergable(".md")); diff --git a/backend/sync_server/Cargo.toml b/backend/sync_server/Cargo.toml index 3ca2c75a..25d4f1df 100644 --- a/backend/sync_server/Cargo.toml +++ b/backend/sync_server/Cargo.toml @@ -35,6 +35,7 @@ clap-verbosity-flag = "3.0.3" bimap = "0.6.3" ts-rs = { version = "10.1", features = ["uuid-impl", "chrono-impl"] } serde_with = "3.12.0" +reconcile-text = "0.4.8" [lints] workspace = true diff --git a/backend/sync_server/src/server/update_document.rs b/backend/sync_server/src/server/update_document.rs index a3ab25e1..a0c443e2 100644 --- a/backend/sync_server/src/server/update_document.rs +++ b/backend/sync_server/src/server/update_document.rs @@ -6,8 +6,9 @@ use axum::{ use axum_extra::TypedHeader; use axum_typed_multipart::TypedMultipart; use log::info; +use reconcile_text::{BuiltinTokenizer, is_binary, reconcile}; use serde::Deserialize; -use sync_lib::{is_file_type_mergable, merge}; +use sync_lib::is_file_type_mergable; use super::{ device_id_header::DeviceIdHeader, requests::UpdateDocumentVersion, @@ -117,8 +118,25 @@ pub async fn update_document( ))); } - let merged_content = if is_file_type_mergable(&sanitized_relative_path) { - merge(&parent_document.content, &latest_version.content, &content) + let merged_content = if is_file_type_mergable(&sanitized_relative_path) + && is_binary(&parent_document.content) + && is_binary(&latest_version.content) + && is_binary(&content) + { + reconcile( + &str::from_utf8(&parent_document.content) + .expect("parent must be valid UTF-8 because it's not binary"), + &str::from_utf8(&latest_version.content) + .expect("latest_version must be valid UTF-8 because it's not binary") + .into(), + &str::from_utf8(&content) + .expect("content must be valid UTF-8 because it's not binary") + .into(), + &*BuiltinTokenizer::Word, + ) + .apply() + .text() + .into_bytes() } else { content.clone() };