Fix syncing when network latency is present (#4)

* WIP

* Add debug

* Dedupe inserts

* Add deterministic ordering

* Fix whitespaces

* Update insta

* Add integration test script

* Rename

* Add test

* Working for non-deletes

* omg it mostly works for deletes

* Isdeleted fix

* remove created dates

* update api

* Take document id

* No max attempt

* works

* Use string uuids

* .

* working!!!! (hopefully)

* Improve bundling

* Add module

* lint

* .

* lint

* Fix CI

* use toolchain

* clean up

* Add useSlowFileEvents

* Delete fuzz

* Fix CI

* use docker

* fix script

* clean up

* Clean up

* change node version

* Build docker image on every commit

* fix ci

* 1 db per vault

* Add scritps folder

* Bump versions

* Lint

* .

* Fix tests for real

* Style

* .

* try

* Consistent ordering

* Fix tests

* hmm

* .

* Clean up diff

* Fixes

* .

* Fix version bump

* .

* .

* .

.github/workflows/check.yml

@@ -17,11 +17,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Setup
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
+        with:
+          node-version: "22.x"
+          check-latest: true
+
+      - name: Setup rust
         run: |
-          rustup install nightly
-          rustup default nightly
-          rustup component add clippy rustfmt
           cargo install sqlx-cli
           cd backend
           sqlx database create --database-url sqlite://db.sqlite3
@@ -44,7 +47,7 @@ jobs:
           cd backend
           cargo test --verbose
           cd sync_lib
-          # wasm-pack test --node # todo: fix this in CI
+          wasm-pack test --node
 
       - name: Lint frontend
         run: |

.github/workflows/e2e.yml

@@ -0,0 +1,45 @@
+name: E2E tests
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-Dwarnings"
+
+jobs:
+  build:
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v4.2.0
+        with:
+          node-version: "22.x"
+          check-latest: true
+
+      - name: Setup rust
+        run: |
+          cargo install sqlx-cli wasm-pack
+          cd backend
+          sqlx database create --database-url sqlite://db.sqlite3
+          sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
+
+      - name: Build wasm
+        run: |
+          cd backend
+          wasm-pack build --target web sync_lib
+
+      - name: E2E tests
+        run: |
+          cd backend
+          RUST_BACKTRACE=1 cargo run -p sync_server &
+          cd ../frontend
+          npm ci
+          cd ..
+          scripts/e2e.sh 32

.github/workflows/publish-docker.yml

@@ -7,8 +7,9 @@ name: Publish server Docker image
 
 on:
   push:
-    tags:
+    branches: ["master"]
-      - "*"
+  pull_request:
+    branches: ["master"]
 
 env:
   # Use docker.io for Docker Hub if empty
@@ -17,8 +18,9 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build-docker:
+  publish-docker:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+
     permissions:
       contents: read
       packages: write
@@ -33,7 +35,7 @@ jobs:
       # Install the cosign tool except on PR
       # https://github.com/sigstore/cosign-installer
       - name: Install cosign
-        if: github.event_name != 'pull_request'
+        if: ${{ github.ref_type == 'tag' }}
         uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
         with:
           cosign-release: "v2.2.4"
@@ -47,7 +49,7 @@ jobs:
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
-        if: github.event_name != 'pull_request'
+        if: ${{ github.ref_type == 'tag' }}
         uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
         with:
           registry: ${{ env.REGISTRY }}
@@ -69,7 +71,7 @@ jobs:
         uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
         with:
           context: backend
-          push: ${{ github.event_name != 'pull_request' }}
+          push: ${{ github.ref_type == 'tag' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
@@ -81,7 +83,7 @@ jobs:
       # transparency data even for private images, pass --force to cosign below.
       # https://github.com/sigstore/cosign
       - name: Sign the published Docker image
-        if: ${{ github.event_name != 'pull_request' }}
+        if: ${{ github.ref_type == 'tag' }}
         env:
           # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
           TAGS: ${{ steps.meta.outputs.tags }}

.github/workflows/publish-plugin.yml

@@ -2,29 +2,27 @@ name: Publish Obsidian plugin
 
 on:
   push:
-    tags:
+    tags: ["*"]
-      - "*"
 
 env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build-plugin:
+  publish-plugin:
     runs-on: self-hosted
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - name: Use Node.js
+      - name: Setup Node.js environment
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4.2.0
         with:
-          node-version: "18.x"
+          node-version: "22.x"
+          check-latest: true
 
       - name: Build wasm
         run: |
           cd backend
-          rustup install nightly
-          rustup default nightly
           cargo install wasm-pack
           wasm-pack build --target web sync_lib
 

.gitignore

@@ -10,6 +10,7 @@ backend/target
 frontend/*/dist
 
 backend/db.sqlite3*
+backend/databases
 backend/config.yml
 
 *.log

README.md

@@ -1,65 +1,56 @@
-## VaultLink self-hosted Obsidian sync plugin
+# VaultLink self-hosted Obsidian plugin for file syncing
 
 [![Check](https://github.com/schmelczer/vault-link/actions/workflows/check.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/check.yml)
+[![E2E tests](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml)
 [![Publish server Docker image](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml)
 [![Publish Obsidian plugin](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml/badge.svg)](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml)
 
 
-## Install [nvm](https://github.com/nvm-sh/nvm)
+## Develop
+
+### Install [nvm](https://github.com/nvm-sh/nvm)
 
 - `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
-- `nvm install 20`
+- `nvm install 22`
-- `nvm use 20`
+- `nvm use 22`
-- Optionally set the system-wide default: `nvm alias default 20`
+- Optionally set the system-wide default: `nvm alias default 22`
 
 
-## Set up Rust
+### Set up Rust
 
 - Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
-- `sudo apt install llvm -y`
-- `rustup self update`
-- `rustup update`
-- `rustup install nightly`
-- `rustup default nightly`
-- `rustup component add llvm-tools-preview`
-- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
 - Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
 - `cargo install cargo-insta sqlx-cli cargo-edit`
 
-
+### Install Obsidian on Linux
-## Publish new version 
 
 ```sh
-./bump-version.sh patch
-```
-
-
-## Update HTTP API TS bindings
-
-```sh 
-npm install -g openapi-typescript
-openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts
-```
-
-```
-
-todo: enable
-[workspace.lints.clippy]
-single_call_fn = { level = "allow", priority = 1 }
-absolute_paths = { level = "allow", priority = 1 }
-arithmetic_side_effects = { level = "allow", priority = 1 }
-similar_names = { level = "allow", priority = 1 }
-self_named_module_files = { level = "allow", priority = 1 }
-single_char_lifetime_names = { level = "allow", priority = 1 }
-missing_docs_in_private_items = { level = "allow", priority = 1 }
-question_mark_used =  { level = "allow", priority = 1 }
-implicit_return = { level = "allow", priority = 1 }
-pedantic = { level = "warn", priority = 0 }
-cargo = { level = "warn", priority = 0 }
-
-```
-
 apt install flatpak
 flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo
 flatpak install flathub md.obsidian.Obsidian
 flatpak run md.obsidian.Obsidian
+```
+
+### Scripts
+
+#### Update HTTP API TS bindings
+
+```sh 
+scripts/update-api-types.sh
+```
+
+#### Publish new version 
+
+```sh
+scripts/bump-version.sh patch
+```
+
+
+#### Run E2E tests
+
+```sh 
+scripts/e2e.sh
+```
+
+And to clean up the logs & database files, run `scripts/clean-up.sh`
+```

backend/Cargo.lock

@@ -105,12 +105,6 @@ dependencies = [
  "backtrace",
 ]
 
-[[package]]
-name = "arbitrary"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
-
 [[package]]
 name = "async-trait"
 version = "0.1.85"
@@ -381,8 +375,6 @@ version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
 dependencies = [
- "jobserver",
- "libc",
  "shlex",
 ]
 
@@ -1228,15 +1220,6 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
-[[package]]
-name = "jobserver"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "js-sys"
 version = "0.3.76"
@@ -1290,16 +1273,6 @@ version = "0.2.167"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
 
-[[package]]
-name = "libfuzzer-sys"
-version = "0.4.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa"
-dependencies = [
- "arbitrary",
- "cc",
-]
-
 [[package]]
 name = "libm"
 version = "0.2.11"
@@ -1781,14 +1754,6 @@ dependencies = [
  "test-case",
 ]
 
-[[package]]
-name = "reconcile-fuzz"
-version = "0.0.30"
-dependencies = [
- "libfuzzer-sys",
- "reconcile",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.5.7"

backend/Cargo.toml

@@ -2,7 +2,6 @@
 resolver = "2"
 members = [
     "reconcile",
-    "fuzz",
     "sync_server", 
     "sync_lib"
 ]
@@ -57,3 +56,19 @@ uninlined_format_args = "warn"
 unnested_or_patterns = "warn"
 unused_self = "warn"
 verbose_file_reads = "warn"
+
+cast_possible_truncation = { level = "allow", priority = 1 }
+doc_link_with_quotes = { level = "allow", priority = 1 }
+cast_sign_loss = { level = "allow", priority = 1 }
+cast_possible_wrap = { level = "allow", priority = 1 }
+struct_field_names = { level = "allow", priority = 1 }
+single_call_fn = { level = "allow", priority = 1 }
+absolute_paths = { level = "allow", priority = 1 }
+arithmetic_side_effects = { level = "allow", priority = 1 }
+similar_names = { level = "allow", priority = 1 }
+self_named_module_files = { level = "allow", priority = 1 }
+single_char_lifetime_names = { level = "allow", priority = 1 }
+missing_docs_in_private_items = { level = "allow", priority = 1 }
+question_mark_used =  { level = "allow", priority = 1 }
+implicit_return = { level = "allow", priority = 1 }
+pedantic = { level = "warn", priority = 0 }

backend/Dockerfile

@@ -3,8 +3,6 @@ FROM rust:1.83 AS builder
 WORKDIR /usr/src/backend
 
 RUN apt update && apt install -y musl-tools
-RUN rustup install nightly && rustup default nightly
-RUN rustup target add x86_64-unknown-linux-musl
 RUN cargo install sqlx-cli
 
 COPY . .
@@ -23,7 +21,7 @@ RUN apk add --no-cache curl
 
 COPY --from=builder /usr/src/backend/target/x86_64-unknown-linux-musl/release/sync_server /app/sync_server
 
-VOLUME /data
+VOLUME /data/databases
 EXPOSE 3000/tcp
 
 WORKDIR /data

backend/fuzz/.gitignore

@@ -1,4 +0,0 @@
-target
-corpus
-artifacts
-coverage

backend/fuzz/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "reconcile-fuzz"
-version.workspace = true
-edition.workspace = true
-authors.workspace = true
-license.workspace = true
-repository.workspace = true
-publish = false
-
-[package.metadata]
-cargo-fuzz = true
-
-[dependencies]
-libfuzzer-sys = "0.4"
-reconcile = { path = "../reconcile" }
-
-[[bin]]
-name = "reconcile"
-path = "fuzz_targets/reconcile.rs"
-test = false
-doc = false
-bench = false
-
-[lints]
-workspace = true

backend/fuzz/fuzz_targets/reconcile.rs

@@ -1,8 +0,0 @@
-#![no_main]
-
-use libfuzzer_sys::fuzz_target;
-
-fuzz_target!(|texts: (String, String, String)| {
-    let (original, left, right) = texts;
-    let _ = reconcile::reconcile(&original, &left, &right);
-});

backend/reconcile/src/diffs/myers.rs

@@ -38,7 +38,7 @@ use crate::{
 /// execution time permitted before it bails and falls back to an approximation.
 pub fn diff<T>(old: &[Token<T>], new: &[Token<T>]) -> Vec<RawOperation<T>>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     let max_d = (old.len() + new.len()).div_ceil(2) + 1;
     let mut vb = V::new(max_d);
@@ -99,7 +99,6 @@ impl IndexMut<isize> for V {
     }
 }
 
-#[inline(always)]
 fn split_at(range: Range<usize>, at: usize) -> (Range<usize>, Range<usize>) {
     (range.start..at, at..range.end)
 }
@@ -124,7 +123,7 @@ fn find_middle_snake<T>(
     vb: &mut V,
 ) -> Option<(usize, usize)>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     let n = old_range.len();
     let m = new_range.len();
@@ -230,7 +229,7 @@ fn conquer<T>(
     vb: &mut V,
     result: &mut Vec<RawOperation<T>>,
 ) where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     // Check for common prefix
     let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone());

backend/reconcile/src/diffs/raw_operation.rs

@@ -3,7 +3,7 @@ use crate::tokenizer::token::Token;
 #[derive(Debug, Clone, PartialEq)]
 pub enum RawOperation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     Insert(Vec<Token<T>>),
     Delete(Vec<Token<T>>),
@@ -12,13 +12,13 @@ where
 
 impl<T> RawOperation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     pub fn tokens(&self) -> &Vec<Token<T>> {
         match self {
-            RawOperation::Insert(tokens) => tokens,
+            RawOperation::Insert(tokens)
-            RawOperation::Delete(tokens) => tokens,
+            | RawOperation::Delete(tokens)
-            RawOperation::Equal(tokens) => tokens,
+            | RawOperation::Equal(tokens) => tokens,
         }
     }
 

backend/reconcile/src/operation_transformation.rs

@@ -37,7 +37,7 @@ pub fn reconcile_with_tokenizer<F, T>(
     tokenizer: &Tokenizer<T>,
 ) -> String
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer);
     let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
@@ -73,7 +73,8 @@ mod test {
             "original_1 edit_1 original_3",
         );
 
-        // One deleted a large range, the other deleted subranges and inserted as well
+        // One deleted a large range, the other deleted subranges and inserted as
+        // well
         test_merge_both_ways(
             "original_1 original_2 original_3 original_4 original_5",
             "original_1 original_5",
@@ -120,9 +121,6 @@ mod test {
             "hi, my friend!",
         );
 
-        // test_merge_both_ways("hello world", "world !", "hi hello world", "hi world
-        // !");
-
         test_merge_both_ways(
             "both delete the same word",
             "both the same word",
@@ -147,7 +145,33 @@ mod test {
         );
     }
 
-    #[ignore = "it's too slow"]
+    #[test]
+    fn test_reconcile_idempotent_inserts() {
+        // Both inserted the same prefix; this should get deduped
+        test_merge_both_ways(
+            "hi ",
+            "hi there ",
+            "hi there my friend ",
+            "hi there my friend ",
+        );
+
+        // The prefix of the 2nd appears on the 1st so it shouldn't get duplicated
+        test_merge_both_ways(
+            "hi ",
+            "hi there you ",
+            "hi there my friend ",
+            "hi there my friend you ",
+        );
+
+        test_merge_both_ways("a", "a b c", "a b c d", "a b c d");
+
+        test_merge_both_ways(
+            "      |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|   ",
+              "      |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|      |cd9195cc-103a-4f13-90c8-4fba0ba421ee|      |d39156cc-cfd6-42a8-b70a-75020896069d|      |fbad794c-9c47-41f2-a343-490284ecb5a0|      |dup|   ",
+             "       |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|      |cd9195cc-103a-4f13-90c8-4fba0ba421ee|      |dup|   ",
+            "      |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001|      |cd9195cc-103a-4f13-90c8-4fba0ba421ee|      |d39156cc-cfd6-42a8-b70a-75020896069d|      |fbad794c-9c47-41f2-a343-490284ecb5a0|      |dup|      |dup|   ");
+    }
+
     #[test_matrix( [
         "pride_and_prejudice.txt",
         "romeo_and_juliet.txt",

backend/reconcile/src/operation_transformation/edited_text.rs

@@ -25,7 +25,7 @@ use crate::{
 #[derive(Debug, Clone, PartialEq, Default)]
 pub struct EditedText<'a, T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     text: &'a str,
     operations: Vec<OrderedOperation<T>>,
@@ -46,7 +46,7 @@ impl<'a> EditedText<'a, String> {
 
 impl<'a, T> EditedText<'a, T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     /// Create an `EditedText` from the given original (old) and updated (new)
     /// strings. The returned `EditedText` represents the changes from the
@@ -65,7 +65,6 @@ where
 
         Self::new(
             original,
-            // Self::cook_operations(diff),
             Self::cook_operations(Self::elongate_operations(diff)).collect(),
         )
     }
@@ -191,7 +190,7 @@ where
     pub fn merge(self, other: Self) -> Self {
         debug_assert_eq!(
             self.text, other.text,
-            "EditedText-s must be derived from the same text to be mergable"
+            "`EditedText`-s must be derived from the same text to be mergable"
         );
 
         let mut left_merge_context = MergeContext::default();
@@ -207,9 +206,21 @@ where
                     |(operation, _)| {
                         (
                             operation.order,
-                            // Operations on left and right must come in the same order so that
+                            // Operations on the left and right must come in the same order so that
                             // inserts can be merged with other inserts and deletes with deletes.
                             usize::from(matches!(operation.operation, Operation::Delete { .. })),
+                            // Make sure that the ordering is deterministic regardless which text
+                            // is left or right.
+                            match &operation.operation {
+                                Operation::Insert { text, .. } => text
+                                    .iter()
+                                    .map(super::super::tokenizer::token::Token::original)
+                                    .collect::<String>(),
+                                Operation::Delete {
+                                    deleted_character_count,
+                                    ..
+                                } => deleted_character_count.to_string(),
+                            },
                         )
                     },
                 )
@@ -232,6 +243,7 @@ where
     }
 
     /// Apply the operations to the text and return the resulting text.
+    #[must_use]
     pub fn apply(&self) -> String {
         let mut builder: StringBuilder<'_> = StringBuilder::new(self.text);
 
@@ -282,7 +294,7 @@ mod tests {
         let original = "hello world! ...";
         let left = "Hello world! I'm Andras.";
         let right = "Hello world! How are you?";
-        let expected = "Hello world! I'm Andras.How are you?";
+        let expected = "Hello world! How are you? I'm Andras.";
 
         let operations_1 = EditedText::from_strings(original, left);
         let operations_2 = EditedText::from_strings(original, right);

backend/reconcile/src/operation_transformation/merge_context.rs

@@ -5,7 +5,7 @@ use crate::operation_transformation::Operation;
 #[derive(Clone)]
 pub struct MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     last_operation: Option<Operation<T>>,
     pub shift: i64,
@@ -13,7 +13,7 @@ where
 
 impl<T> Default for MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn default() -> Self {
         MergeContext {
@@ -25,7 +25,7 @@ where
 
 impl<T> Debug for MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         f.debug_struct("MergeContext")
@@ -37,7 +37,7 @@ where
 
 impl<T> MergeContext<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     pub fn last_operation(&self) -> Option<&Operation<T>> { self.last_operation.as_ref() }
 

backend/reconcile/src/operation_transformation/operation.rs

@@ -1,7 +1,5 @@
-use core::{
+use core::fmt::{Debug, Display};
-    fmt::{Debug, Display},
+use std::ops::Range;
-    ops::Range,
-};
 
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
@@ -9,7 +7,10 @@ use serde::{Deserialize, Serialize};
 use super::merge_context::MergeContext;
 use crate::{
     Token,
-    utils::{find_common_overlap::find_common_overlap, string_builder::StringBuilder},
+    utils::{
+        find_longest_prefix_contained_within::find_longest_prefix_contained_within,
+        string_builder::StringBuilder,
+    },
 };
 
 /// Represents a change that can be applied to a text document.
@@ -19,7 +20,7 @@ use crate::{
 #[derive(Clone, PartialEq)]
 pub enum Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     Insert {
         index: usize,
@@ -37,7 +38,7 @@ where
 
 impl<T> Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     /// Creates an insert operation with the given index and text.
     /// If the text is empty (meaning that the operation would be a no-op),
@@ -81,15 +82,8 @@ where
         })
     }
 
-    /// Tries to apply the operation to the given `ropey::Rope` text, returning
+    /// Applies the operation to the given `StringBuilder`, returning the
-    /// the modified text.
+    /// modified `StringBuilder`.
-    ///
-    /// # Errors
-    ///
-    /// Returns a `SyncLibError::OperationApplicationError` if the operation
-    /// cannot be applied.
-    ///
-    /// # Panics
     ///
     /// When compiled in debug mode, panics if a delete operation is attempted
     /// on a range of text that does not match the text to be deleted.
@@ -114,7 +108,7 @@ where
 
                 builder.delete(self.range());
             }
-        };
+        }
 
         builder
     }
@@ -122,8 +116,7 @@ where
     /// Returns the index of the first character that the operation affects.
     pub fn start_index(&self) -> usize {
         match self {
-            Operation::Insert { index, .. } => *index,
+            Operation::Insert { index, .. } | Operation::Delete { index, .. } => *index,
-            Operation::Delete { index, .. } => *index,
         }
     }
 
@@ -137,6 +130,7 @@ where
     }
 
     /// Returns the range of indices of characters that the operation affects.
+    #[allow(clippy::range_plus_one)]
     pub fn range(&self) -> Range<usize> { self.start_index()..self.end_index() + 1 }
 
     /// Returns the number of affected characters. It is always greater than 0
@@ -212,17 +206,20 @@ where
                     ..
                 }),
             ) => {
-                let offset_in_tokens = find_common_overlap(previous_inserted_text, &text);
+                // In case the current insert's prefix appears in the previously inserted text,
-                let trimmed_length_in_tokens = previous_inserted_text.len() - offset_in_tokens;
+                // we can trim the current insert to only include the non-overlapping part.
-                let trimmed_length = previous_inserted_text
+                // This way, we don't end up duplicating text.
+                let offset_in_tokens =
+                    find_longest_prefix_contained_within(previous_inserted_text, &text);
+                let offset_in_length = text
                     .iter()
-                    .skip(offset_in_tokens)
+                    .take(offset_in_tokens)
                     .map(Token::get_original_length)
                     .sum::<usize>();
                 let trimmed_operation =
-                    Operation::create_insert(index, text[trimmed_length_in_tokens..].to_vec());
+                    Operation::create_insert(index, text[offset_in_tokens..].to_vec());
 
-                affecting_context.shift -= trimmed_length as i64;
+                affecting_context.shift -= offset_in_length as i64;
                 produced_context.shift += trimmed_operation
                     .as_ref()
                     .map(Operation::len)
@@ -297,7 +294,7 @@ where
 
 impl<T> Display for Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         match self {
@@ -341,7 +338,7 @@ where
 
 impl<T> Debug for Operation<T>
 where
-    T: PartialEq + Clone,
+    T: PartialEq + Clone + std::fmt::Debug,
 {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{self}") }
 }
@@ -353,7 +350,7 @@ mod tests {
     use super::*;
 
     #[test]
-    #[should_panic]
+    #[should_panic(expected = "Shifted index must be non-negative")]
     fn test_shifting_error() {
         insta::assert_debug_snapshot!(
             Operation::create_insert(1, vec!["hi".into()])

backend/reconcile/src/operation_transformation/snapshots/reconcile__operation_transformation__edited_text__tests__calculate_operations.snap

@@ -8,19 +8,19 @@ EditedText {
     operations: [
         OrderedOperation {
             order: 0,
-            operation: <insert 'Hello, my friend! ' from index 0>,
+            operation: <insert 'Hello, my friend!' from index 0>,
         },
         OrderedOperation {
             order: 0,
-            operation: <delete 'hello world! ' from index 18>,
+            operation: <delete 'hello world!' from index 17>,
         },
         OrderedOperation {
-            order: 21,
+            order: 20,
-            operation: <insert 'you doing? Albert' from index 26>,
+            operation: <insert ' you doing? Albert' from index 25>,
         },
         OrderedOperation {
-            order: 21,
+            order: 20,
-            operation: <delete 'you?  Adam' from index 43>,
+            operation: <delete ' you?  Adam' from index 43>,
         },
     ],
 }

backend/reconcile/src/tokenizer/snapshots/reconcile__tokenizer__word_tokenizer__tests__with_snapshots-2.snap

@@ -0,0 +1,6 @@
+---
+source: reconcile/src/tokenizer/word_tokenizer.rs
+expression: "word_tokenizer(\"\")"
+snapshot_kind: text
+---
+[]