andras/vault-link
andras/vault-link
Watch
1
0
Fork
You've already forked vault-link
 0
Code Issues Pull requests 18 Projects Releases 55 Packages Wiki Activity Actions 38

Extract reconcile (#85)

Browse source
This commit is contained in:
Andras Schmelczer 2025-07-13 11:06:42 +01:00 • committed by GitHub
commit bb0e44f06f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
141 changed files with 294 additions and 36720 deletions
Download patch file Download diff file Expand all files Collapse all files

24
.github/workflows/check.yml vendored
View file

@ -25,29 +25,22 @@ jobs:
- name: Setup rust - name: Setup rust
run: | run: |
cargo install sqlx-cli wasm-pack cargo-machete cargo install sqlx-cli cargo-machete
cd backend cd sync-server
sqlx database create --database-url sqlite://db.sqlite3 sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3 sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm - name: Lint sync-server
run: | run: |
cd backend cd sync-server
wasm-pack build --target web sync_lib
- name: Lint backend
run: |
cd backend
cargo clippy --all-targets --all-features cargo clippy --all-targets --all-features
cargo fmt --all -- --check cargo fmt --all -- --check
cargo machete cargo machete
- name: Test backend - name: Test sync-server
run: | run: |
cd backend cd sync-server
cargo test --verbose -- --include-ignored cargo test --verbose
cd sync_lib
wasm-pack test --node
- name: Lint frontend - name: Lint frontend
run: | run: |
@ -64,4 +57,5 @@ jobs:
- name: Test frontend - name: Test frontend
run: | run: |
cd frontend cd frontend
npm ci
npm run test npm run test

15
.github/workflows/e2e.yml vendored
View file

@ -25,20 +25,15 @@ jobs:
- name: Setup rust - name: Setup rust
run: | run: |
cargo install sqlx-cli wasm-pack cargo install sqlx-cli
cd backend cd sync-server
sqlx database create --database-url sqlite://db.sqlite3 sqlx database create --database-url sqlite://db.sqlite3
sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3 sqlx migrate run --source src/app_state/database/migrations --database-url sqlite://db.sqlite3
- name: Build wasm
run: |
cd backend
wasm-pack build --target web sync_lib
- name: E2E tests - name: E2E tests
run: | run: |
cd backend cd sync-server
cargo run -p sync_server config-e2e.yml --color never & cargo run config-e2e.yml --color never &
cd .. cd ..
scripts/update-api-types.sh scripts/update-api-types.sh

2
.github/workflows/publish-docker.yml vendored
View file

@ -66,7 +66,7 @@ jobs:
id: build-and-push id: build-and-push
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with: with:
context: backend context: sync-server
push: ${{ github.ref_type == 'tag' }} push: ${{ github.ref_type == 'tag' }}
tags: ${{ steps.meta.outputs.tags }} tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }} labels: ${{ steps.meta.outputs.labels }}

6
.github/workflows/publish-plugin.yml vendored
View file

@ -20,12 +20,6 @@ jobs:
node-version: "22.x" node-version: "22.x"
check-latest: true check-latest: true
- name: Build wasm
run: |
cd backend
cargo install wasm-pack
wasm-pack build --target web sync_lib
- name: Build plugin - name: Build plugin
run: | run: |
cd frontend cd frontend

12
.gitignore vendored
View file

@ -4,15 +4,17 @@ node_modules
# Exclude macOS Finder (System Explorer) View States # Exclude macOS Finder (System Explorer) View States
.DS_Store .DS_Store
# Rust build folder
backend/target
# Frontend build folders # Frontend build folders
frontend/*/dist frontend/*/dist
backend/db.sqlite3* sync-server/db.sqlite3*
backend/databases sync-server/databases
backend/sync_server/bindings/*.ts
# Rust build folders
sync-server/target
sync-server/bindings/*.ts
*.log *.log
*.sqlx *.sqlx

2
README.md
View file

@ -56,4 +56,4 @@ And to clean up the logs & database files, run `scripts/clean-up.sh`
## Projects ## Projects
- [Sync server](./backend/sync_server/README.md) - [Sync server](./sync-server/README.md)

4
backend/Dockerfile
View file

@ -8,9 +8,9 @@ RUN cargo install sqlx-cli
COPY . . COPY . .
RUN sqlx database create --database-url sqlite://db.sqlite3 RUN sqlx database create --database-url sqlite://db.sqlite3
RUN sqlx migrate run --source sync_server/src/app_state/database/migrations --database-url sqlite://db.sqlite3 RUN sqlx migrate run --source sync-server/src/app_state/database/migrations --database-url sqlite://db.sqlite3
RUN cargo build --package sync_server --release --target x86_64-unknown-linux-musl RUN cargo build --release --target x86_64-unknown-linux-musl
# Runtime image # Runtime image
FROM alpine:3.22.0 FROM alpine:3.22.0

23
backend/reconcile/Cargo.toml
View file

@ -1,23 +0,0 @@
[package]
name = "reconcile"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
serde = { version = "1.0.219", optional = true, features = ["derive"] }
[features]
serde = [ "dep:serde" ]
[dev-dependencies]
insta = "1.42.2"
pretty_assertions = "1.4.1"
serde = { version = "1.0.219", features = ["derive"] }
serde_yaml ="0.9.34"
test-case = "3.3.1"
[lints]
workspace = true

2
backend/reconcile/src/diffs.rs
View file

@ -1,2 +0,0 @@
pub mod myers;
pub mod raw_operation;

357
backend/reconcile/src/diffs/myers.rs
View file

@ -1,357 +0,0 @@
//! Taken from <https://github.com/mitsuhiko/similar/blob/7e15c44de11a1cd61e1149189929e189ef977fd8/src/algorithms/myers.rs>
//!
//! Myers' diff algorithm.
//!
//! * time: `O((N+M)D)`
//! * space `O(N+M)`
//!
//! See [the original article by Eugene W. Myers](http://www.xmailserver.org/diff2.pdf)
//! describing it.
//!
//! The implementation of this algorithm is based on the implementation by
//! Brandon Williams.
//!
//! # Heuristics
//!
//! At present this implementation of Myers' does not implement any more
//! advanced heuristics that would solve some pathological cases. For instance
//! passing two large and completely distinct sequences to the algorithm will
//! make it spin without making reasonable progress.
//! For potential improvements here see [similar#15](https://github.com/mitsuhiko/similar/issues/15).
use std::{
ops::{Index, IndexMut, Range},
vec,
};
use super::raw_operation::RawOperation;
use crate::{
tokenizer::token::Token,
utils::{common_prefix_len::common_prefix_len, common_suffix_len::common_suffix_len},
};
/// Myers' diff algorithm with deadline.
///
/// Diff `old`, between indices `old_range` and `new` between indices
/// `new_range`.
///
/// The returned `RawOperations` all have a token count of 1.
pub fn diff<T>(old: &[Token<T>], new: &[Token<T>]) -> Vec<RawOperation<T>>
where
T: PartialEq + Clone + std::fmt::Debug,
{
let max_d = (old.len() + new.len()).div_ceil(2) + 1;
let mut vb = V::new(max_d);
let mut vf = V::new(max_d);
let mut result: Vec<RawOperation<T>> = vec![];
conquer(
old,
0..old.len(),
new,
0..new.len(),
&mut vf,
&mut vb,
&mut result,
);
debug_assert!(
result.iter().all(|op| op.tokens().len() == 1),
"All operations should be of length 1"
);
result
}
// A D-path is a path which starts at (0,0) that has exactly D non-diagonal
// edges. All D-paths consist of a (D - 1)-path followed by a non-diagonal edge
// and then a possibly empty sequence of diagonal edges called a snake.
/// `V` contains the endpoints of the furthest reaching `D-paths`. For each
/// recorded endpoint `(x,y)` in diagonal `k`, we only need to retain `x`
/// because `y` can be computed from `x - k`. In other words, `V` is an array of
/// integers where `V[k]` contains the row index of the endpoint of the furthest
/// reaching path in diagonal `k`.
///
/// We can't use a traditional Vec to represent `V` since we use `k` as an index
/// and it can take on negative values. So instead `V` is represented as a
/// light-weight wrapper around a Vec plus an `offset` which is the maximum
/// value `k` can take on in order to map negative `k`'s back to a value >= 0.
#[derive(Debug)]
struct V {
offset: isize,
v: Vec<usize>, // Look into initializing this to -1 and storing isize
}
impl V {
fn new(max_d: usize) -> Self {
Self {
offset: max_d as isize,
v: vec![0; 2 * max_d],
}
}
fn len(&self) -> usize { self.v.len() }
}
impl Index<isize> for V {
type Output = usize;
fn index(&self, index: isize) -> &Self::Output { &self.v[(index + self.offset) as usize] }
}
impl IndexMut<isize> for V {
fn index_mut(&mut self, index: isize) -> &mut Self::Output {
&mut self.v[(index + self.offset) as usize]
}
}
fn split_at(range: Range<usize>, at: usize) -> (Range<usize>, Range<usize>) {
(range.start..at, at..range.end)
}
/// A `Snake` is a sequence of diagonal edges in the edit graph. Normally
/// a snake has a start end end point (and it is possible for a snake to have
/// a length of zero, meaning the start and end points are the same) however
/// we do not need the end point which is why it's not implemented here.
///
/// The divide part of a divide-and-conquer strategy. A D-path has D+1 snakes
/// some of which may be empty. The divide step requires finding the ceil(D/2) +
/// 1 or middle snake of an optimal D-path. The idea for doing so is to
/// simultaneously run the basic algorithm in both the forward and reverse
/// directions until furthest reaching forward and reverse paths starting at
/// opposing corners 'overlap'.
fn find_middle_snake<T>(
old: &[Token<T>],
old_range: Range<usize>,
new: &[Token<T>],
new_range: Range<usize>,
vf: &mut V,
vb: &mut V,
) -> Option<(usize, usize)>
where
T: PartialEq + Clone + std::fmt::Debug,
{
let n = old_range.len();
let m = new_range.len();
// By Lemma 1 in the paper, the optimal edit script length is odd or even as
// `delta` is odd or even.
let delta = n as isize - m as isize;
let odd = delta & 1 == 1;
// The initial point at (0, -1)
vf[1] = 0;
// The initial point at (N, M+1)
vb[1] = 0;
let d_max = (n + m).div_ceil(2) + 1;
assert!(vf.len() >= d_max);
assert!(vb.len() >= d_max);
for d in 0..d_max as isize {
// Forward path
for k in (-d..=d).rev().step_by(2) {
let mut x = if k == -d || (k != d && vf[k - 1] < vf[k + 1]) {
vf[k + 1]
} else {
vf[k - 1] + 1
};
let y = (x as isize - k) as usize;
// The coordinate of the start of a snake
let (x0, y0) = (x, y);
// While these sequences are identical, keep moving through the
// graph with no cost
if x < old_range.len() && y < new_range.len() {
let advance = common_prefix_len(
old,
old_range.start + x..old_range.end,
new,
new_range.start + y..new_range.end,
);
x += advance;
}
// This is the new best x value
vf[k] = x;
// Only check for connections from the forward search when N - M is
// odd and when there is a reciprocal k line coming from the other
// direction.
if odd && (k - delta).abs() <= (d - 1) {
// TODO optimize this so we don't have to compare against n
if vf[k] + vb[-(k - delta)] >= n {
// Return the snake
return Some((x0 + old_range.start, y0 + new_range.start));
}
}
}
// Backward path
for k in (-d..=d).rev().step_by(2) {
let mut x = if k == -d || (k != d && vb[k - 1] < vb[k + 1]) {
vb[k + 1]
} else {
vb[k - 1] + 1
};
let mut y = (x as isize - k) as usize;
// The coordinate of the start of a snake
if x < n && y < m {
let advance = common_suffix_len(
old,
old_range.start..old_range.start + n - x,
new,
new_range.start..new_range.start + m - y,
);
x += advance;
y += advance;
}
// This is the new best x value
vb[k] = x;
if !odd && (k - delta).abs() <= d {
// TODO optimize this so we don't have to compare against n
if vb[k] + vf[-(k - delta)] >= n {
// Return the snake
return Some((n - x + old_range.start, m - y + new_range.start));
}
}
}
// TODO: Maybe there's an opportunity to optimize and bail early?
}
None
}
fn conquer<T>(
old: &[Token<T>],
mut old_range: Range<usize>,
new: &[Token<T>],
mut new_range: Range<usize>,
vf: &mut V,
vb: &mut V,
result: &mut Vec<RawOperation<T>>,
) where
T: PartialEq + Clone + std::fmt::Debug,
{
// Check for common prefix
let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone());
if common_prefix_len > 0 {
result.extend(
old[old_range.start..old_range.start + common_prefix_len]
.iter()
.map(|token| RawOperation::Equal(vec![token.clone()])),
);
}
old_range.start += common_prefix_len;
new_range.start += common_prefix_len;
// Check for common suffix
let common_suffix_len = common_suffix_len(old, old_range.clone(), new, new_range.clone());
let common_suffix = (
old_range.end - common_suffix_len,
new_range.end - common_suffix_len,
);
old_range.end -= common_suffix_len;
new_range.end -= common_suffix_len;
if old_range.is_empty() && new_range.is_empty() {
// do nothing
} else if new_range.is_empty() {
result.extend(
old[old_range.start..old_range.start + old_range.len()]
.iter()
.map(|token| RawOperation::Delete(vec![token.clone()])),
);
} else if old_range.is_empty() {
result.extend(
new[new_range.start..new_range.start + new_range.len()]
.iter()
.map(|token| RawOperation::Insert(vec![token.clone()])),
);
} else if let Some((x_start, y_start)) =
find_middle_snake(old, old_range.clone(), new, new_range.clone(), vf, vb)
{
let (old_a, old_b) = split_at(old_range, x_start);
let (new_a, new_b) = split_at(new_range, y_start);
conquer(old, old_a, new, new_a, vf, vb, result);
conquer(old, old_b, new, new_b, vf, vb, result);
} else {
result.extend(
old[old_range.start..old_range.end]
.iter()
.map(|token| RawOperation::Delete(vec![token.clone()])),
);
result.extend(
new[new_range.start..new_range.end]
.iter()
.map(|token| RawOperation::Insert(vec![token.clone()])),
);
}
if common_suffix_len > 0 {
result.extend(
old[common_suffix.0..common_suffix.0 + common_suffix_len]
.iter()
.map(|token| RawOperation::Equal(vec![token.clone()])),
);
}
}
#[cfg(test)]
mod tests {
use insta::assert_debug_snapshot;
use super::*;
#[test]
fn test_empty_diff() {
let old: Vec<Token<String>> = vec![];
let new: Vec<Token<String>> = vec![];
let result = diff(&old, &new);
assert_eq!(result.len(), 0);
}
#[test]
fn test_identical_content() {
let content = vec!["a".into(), "b".into(), "c".into()];
let result = diff(&content, &content);
assert_debug_snapshot!(result);
}
#[test]
fn test_insert_only() {
let old: Vec<Token<String>> = vec![];
let new: Vec<Token<String>> = vec!["a".into(), "b".into()];
let result = diff(&old, &new);
assert_debug_snapshot!(result);
}
#[test]
fn test_delete_only() {
let old = vec!["a".into(), "b".into()];
let new: Vec<Token<String>> = vec![];
let result = diff(&old, &new);
assert_debug_snapshot!(result);
}
#[test]
fn test_prefix_and_suffix() {
let old = vec!["a".into(), "b".into(), "c".into(), "d".into()];
let new = vec!["a".into(), "x".into(), "d".into()];
let result = diff(&old, &new);
assert_debug_snapshot!(result);
}
#[test]
fn test_complex_diff() {
let old = vec!["a".into(), "b".into(), "c".into(), "d".into()];
let new = vec!["a".into(), "x".into(), "c".into(), "y".into()];
let result = diff(&old, &new);
assert_debug_snapshot!(result);
}
}

64
backend/reconcile/src/diffs/raw_operation.rs
View file

@ -1,64 +0,0 @@
use crate::tokenizer::token::Token;
#[derive(Debug, Clone, PartialEq)]
pub enum RawOperation<T>
where
T: PartialEq + Clone + std::fmt::Debug,
{
Insert(Vec<Token<T>>),
Delete(Vec<Token<T>>),
Equal(Vec<Token<T>>),
}
impl<T> RawOperation<T>
where
T: PartialEq + Clone + std::fmt::Debug,
{
pub fn tokens(&self) -> &Vec<Token<T>> {
match self {
RawOperation::Insert(tokens)
| RawOperation::Delete(tokens)
| RawOperation::Equal(tokens) => tokens,
}
}
pub fn original_text_length(&self) -> usize {
self.tokens().iter().map(Token::get_original_length).sum()
}
pub fn get_original_text(self) -> String { self.tokens().iter().map(Token::original).collect() }
pub fn is_left_joinable(&self) -> bool {
let first_token = self.tokens().first();
first_token.is_none_or(super::super::tokenizer::token::Token::get_is_left_joinable)
}
pub fn is_right_joinable(&self) -> bool {
let last_token = self.tokens().last();
last_token.is_none_or(super::super::tokenizer::token::Token::get_is_right_joinable)
}
/// Extends the operation with another operation. Only operations of the
/// same type as self can be used to extend self, otherwise the function
/// will panic.
pub fn extend(self, other: RawOperation<T>) -> RawOperation<T> {
debug_assert!(
std::mem::discriminant(&self) == std::mem::discriminant(&other),
"Cannot extend operations of different types. This should have been handled before \
calling this function."
);
match (self, other) {
(RawOperation::Insert(tokens1), RawOperation::Insert(tokens2)) => {
RawOperation::Insert(tokens1.into_iter().chain(tokens2).collect())
}
(RawOperation::Delete(tokens1), RawOperation::Delete(tokens2)) => {
RawOperation::Delete(tokens1.into_iter().chain(tokens2).collect())
}
(RawOperation::Equal(tokens1), RawOperation::Equal(tokens2)) => {
RawOperation::Equal(tokens1.into_iter().chain(tokens2).collect())
}
_ => unreachable!("Only operations of the same type can be extended"),
}
}
}

67
backend/reconcile/src/diffs/snapshots/reconcile__diffs__myers__tests__complex_diff.snap
View file

@ -1,67 +0,0 @@
---
source: reconcile/src/diffs/myers.rs
expression: result
snapshot_kind: text
---
[
Equal(
[
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Insert(
[
Token {
normalised: "x",
original: "x",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Delete(
[
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Equal(
[
Token {
normalised: "c",
original: "c",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Insert(
[
Token {
normalised: "y",
original: "y",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Delete(
[
Token {
normalised: "d",
original: "d",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
]

27
backend/reconcile/src/diffs/snapshots/reconcile__diffs__myers__tests__delete_only.snap
View file

@ -1,27 +0,0 @@
---
source: reconcile/src/diffs/myers.rs
expression: result
snapshot_kind: text
---
[
Delete(
[
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Delete(
[
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
]

37
backend/reconcile/src/diffs/snapshots/reconcile__diffs__myers__tests__identical_content.snap
View file

@ -1,37 +0,0 @@
---
source: reconcile/src/diffs/myers.rs
expression: result
snapshot_kind: text
---
[
Equal(
[
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Equal(
[
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Equal(
[
Token {
normalised: "c",
original: "c",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
]

27
backend/reconcile/src/diffs/snapshots/reconcile__diffs__myers__tests__insert_only.snap
View file

@ -1,27 +0,0 @@
---
source: reconcile/src/diffs/myers.rs
expression: result
snapshot_kind: text
---
[
Insert(
[
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Insert(
[
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
]

57
backend/reconcile/src/diffs/snapshots/reconcile__diffs__myers__tests__prefix_and_suffix.snap
View file

@ -1,57 +0,0 @@
---
source: reconcile/src/diffs/myers.rs
expression: result
snapshot_kind: text
---
[
Equal(
[
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Delete(
[
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Delete(
[
Token {
normalised: "c",
original: "c",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Insert(
[
Token {
normalised: "x",
original: "x",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
Equal(
[
Token {
normalised: "d",
original: "d",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
]

10
backend/reconcile/src/lib.rs
View file

@ -1,10 +0,0 @@
mod diffs;
mod operation_transformation;
mod tokenizer;
mod utils;
pub use operation_transformation::{
CursorPosition, EditedText, TextWithCursors, reconcile, reconcile_with_cursors,
reconcile_with_tokenizer,
};
pub use tokenizer::{Tokenizer, token::Token};

166
backend/reconcile/src/operation_transformation.rs
View file

@ -1,166 +0,0 @@
mod cursor;
mod edited_text;
mod merge_context;
mod operation;
mod ordered_operation;
pub use cursor::{CursorPosition, TextWithCursors};
pub use edited_text::EditedText;
pub use operation::Operation;
use crate::Tokenizer;
#[must_use]
pub fn reconcile(original: &str, left: &str, right: &str) -> String {
reconcile_with_cursors(original, left.into(), right.into())
.text
.to_string()
}
#[must_use]
pub fn reconcile_with_cursors<'a>(
original: &'a str,
left: TextWithCursors<'a>,
right: TextWithCursors<'a>,
) -> TextWithCursors<'static> {
let left_operations = EditedText::from_strings(original, left);
let right_operations = EditedText::from_strings(original, right);
let merged_operations = left_operations.merge(right_operations);
TextWithCursors::new_owned(merged_operations.apply(), merged_operations.cursors)
}
#[must_use]
pub fn reconcile_with_tokenizer<'a, F, T>(
original: &str,
left: TextWithCursors<'a>,
right: TextWithCursors<'a>,
tokenizer: &Tokenizer<T>,
) -> TextWithCursors<'static>
where
T: PartialEq + Clone + std::fmt::Debug,
{
let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer);
let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
let merged_operations = left_operations.merge(right_operations);
TextWithCursors::new_owned(merged_operations.apply(), merged_operations.cursors)
}
#[cfg(test)]
mod test {
use std::{fs, ops::Range, path::Path};
use pretty_assertions::assert_eq;
use test_case::test_matrix;
use super::*;
use crate::CursorPosition;
#[test]
fn test_cursor_complex() {
let original = "this is some complex text to test cursor positions";
let left = TextWithCursors::new(
"this is really complex text for testing cursor positions",
vec![
CursorPosition {
id: 0,
char_index: 8,
}, // after "this is "
CursorPosition {
id: 1,
char_index: 22,
}, // after "this is really complex text"
],
);
let right = TextWithCursors::new(
"that was some complex sample to test cursor movements",
vec![
CursorPosition {
id: 2,
char_index: 5,
}, // after "that "
CursorPosition {
id: 3,
char_index: 29,
}, // after "some complex sample "
],
);
let merged = reconcile_with_cursors(original, left, right);
assert_eq!(
merged,
TextWithCursors::new(
"that was really complex sample for testing cursor movements",
vec![
CursorPosition {
id: 2,
char_index: 5
}, // unchanged
CursorPosition {
id: 0,
char_index: 9
}, // before "really"
CursorPosition {
id: 1,
char_index: 23
}, // inside of "s|ample" because "text" got replaced by "sample"
CursorPosition {
id: 3,
char_index: 43
}, // before "cursor movements"
]
)
);
}
#[ignore = "expensive to run, only run in CI"]
#[test_matrix( [
"pride_and_prejudice.txt",
"room_with_a_view.txt",
"kun_lu.txt",
"blns.txt"
], [
"pride_and_prejudice.txt",
"room_with_a_view.txt",
"kun_lu.txt",
"blns.txt"
], [
"pride_and_prejudice.txt",
"room_with_a_view.txt",
"kun_lu.txt",
"blns.txt"
], [0..10000, 10000..20000], [0..10000, 10000..20000], [0..10000, 10000..20000])]
fn test_merge_files_without_panic(
file_name_1: &str,
file_name_2: &str,
file_name_3: &str,
range_1: Range<usize>,
range_2: Range<usize>,
range_3: Range<usize>,
) {
let files = [file_name_1, file_name_2, file_name_3];
let permutations = [range_1, range_2, range_3];
let root = Path::new("tests/resources/");
let contents = files
.iter()
.zip(permutations.iter())
.map(|(file, range)| {
let path = root.join(file);
fs::read_to_string(&path)
.unwrap()
.chars()
.skip(range.start)
.take(range.end)
.collect::<String>()
})
.collect::<Vec<_>>();
let _ = reconcile(&contents[0], &contents[1], &contents[2]);
}
}

57
backend/reconcile/src/operation_transformation/cursor.rs
View file

@ -1,57 +0,0 @@
use std::borrow::Cow;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
// CursorPosition represents the position of an identifiable cursor in a text
// document based on its (UTF-8) character index.
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone, PartialEq, Default)]
pub struct CursorPosition {
pub id: usize,
pub char_index: usize,
}
impl CursorPosition {
#[must_use]
pub fn with_index(&self, index: usize) -> Self {
CursorPosition {
id: self.id,
char_index: index,
}
}
}
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone, PartialEq, Default)]
pub struct TextWithCursors<'a> {
pub text: Cow<'a, str>,
pub cursors: Vec<CursorPosition>,
}
impl<'a> TextWithCursors<'a> {
#[must_use]
pub fn new(text: &'a str, cursors: Vec<CursorPosition>) -> Self {
Self {
text: text.into(),
cursors,
}
}
#[must_use]
pub fn new_owned(text: String, cursors: Vec<CursorPosition>) -> Self {
Self {
text: text.into(),
cursors,
}
}
}
impl<'a> From<&'a str> for TextWithCursors<'a> {
fn from(text: &'a str) -> Self {
Self {
text: text.into(),
cursors: Vec::new(),
}
}
}

381
backend/reconcile/src/operation_transformation/edited_text.rs
View file

@ -1,381 +0,0 @@
use core::iter;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use super::{CursorPosition, Operation, TextWithCursors, ordered_operation::OrderedOperation};
use crate::{
diffs::{myers::diff, raw_operation::RawOperation},
operation_transformation::merge_context::MergeContext,
tokenizer::{Tokenizer, word_tokenizer::word_tokenizer},
utils::{merge_iters::MergeSorted as _, side::Side, string_builder::StringBuilder},
};
/// A sequence of operations that can be applied to a text document.
/// `EditedText` supports merging two sequences of operations using the
/// principle of Operational Transformation.
///
/// It's mainly created through the `from_strings` method, then merged with
/// another `EditedText` derived from the same original text and then applied to
/// the original text to get the reconciled text of concurrent edits.
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone, PartialEq, Default)]
pub struct EditedText<'a, T>
where
T: PartialEq + Clone + std::fmt::Debug,
{
text: &'a str,
operations: Vec<OrderedOperation<T>>,
pub(crate) cursors: Vec<CursorPosition>,
}
impl<'a> EditedText<'a, String> {
/// Create an `EditedText` from the given original (old) and updated (new)
/// strings. The returned `EditedText` represents the changes from the
/// original to the updated text. When the return value is applied to
/// the original text, it will result in the updated text. The default
/// word tokenizer is used to tokenize the text which splits the text on
/// whitespaces.
#[must_use]
pub fn from_strings(original: &'a str, updated: TextWithCursors<'a>) -> Self {
Self::from_strings_with_tokenizer(original, updated, &word_tokenizer)
}
}
impl<'a, T> EditedText<'a, T>
where
T: PartialEq + Clone + std::fmt::Debug,
{
/// Create an `EditedText` from the given original (old) and updated (new)
/// strings. The returned `EditedText` represents the changes from the
/// original to the updated text. When the return value is applied to
/// the original text, it will result in the updated text. The tokenizer
/// function is used to tokenize the text.
pub fn from_strings_with_tokenizer(
original: &'a str,
updated: TextWithCursors<'a>,
tokenizer: &Tokenizer<T>,
) -> Self {
let original_tokens = (tokenizer)(original);
let updated_tokens = (tokenizer)(&updated.text);
let diff: Vec<RawOperation<T>> = diff(&original_tokens, &updated_tokens);
Self::new(
original,
Self::cook_operations(Self::elongate_operations(diff)).collect(),
updated.cursors,
)
}
fn elongate_operations<I>(raw_operations: I) -> Vec<RawOperation<T>>
where
I: IntoIterator<Item = RawOperation<T>>,
{
// This might look bad, but this makes sense. The inserts and deltes can be
// interleaved, such as: IDIDID and we need to turn this into IIIDDD.
// So we need to keep track of both the last insert and delete operations, not
// just the last one.
let mut maybe_previous_insert: Option<RawOperation<T>> = None;
let mut maybe_previous_delete: Option<RawOperation<T>> = None;
let mut result: Vec<RawOperation<T>> = raw_operations
.into_iter()
.flat_map(|next| match next {
RawOperation::Insert(..) => match maybe_previous_insert.take() {
Some(prev) if prev.is_right_joinable() && next.is_left_joinable() => {
maybe_previous_insert = Some(prev.extend(next));
Box::new(iter::empty()) as Box<dyn Iterator<Item = RawOperation<T>>>
}
prev => {
maybe_previous_insert = Some(next);
Box::new(prev.into_iter())
}
},
RawOperation::Delete(..) => match maybe_previous_delete.take() {
Some(prev) if prev.is_right_joinable() && next.is_left_joinable() => {
maybe_previous_delete = Some(prev.extend(next));
Box::new(iter::empty()) as Box<dyn Iterator<Item = RawOperation<T>>>
}
prev => {
maybe_previous_delete = Some(next);
Box::new(prev.into_iter())
}
},
RawOperation::Equal(..) => Box::new(
maybe_previous_insert
.take()
.into_iter()
.chain(maybe_previous_delete.take())
.chain(iter::once(next)),
)
as Box<dyn Iterator<Item = RawOperation<T>>>,
})
.collect();
if let Some(prev) = maybe_previous_insert {
result.push(prev);
}
if let Some(prev) = maybe_previous_delete {
result.push(prev);
}
result
}
// Turn raw operations into ordered operations while keeping track of old & new
// indexes.
fn cook_operations<I>(raw_operations: I) -> impl Iterator<Item = OrderedOperation<T>>
where
I: IntoIterator<Item = RawOperation<T>>,
{
let mut new_index = 0; // this is the start index of the operation on the new text
let mut order = 0; // this is the start index of the operation on the original text
raw_operations.into_iter().filter_map(move |raw_operation| {
let length = raw_operation.original_text_length();
match raw_operation {
RawOperation::Equal(..) => {
let op = if cfg!(debug_assertions) {
Operation::create_equal_with_text(
new_index,
raw_operation.get_original_text(),
)
} else {
Operation::create_equal(new_index, length)
}
.map(|operation| OrderedOperation { order, operation });
new_index += length;
order += length;
op
}
RawOperation::Insert(tokens) => {
let op = Operation::create_insert(new_index, tokens)
.map(|operation| OrderedOperation { order, operation });
new_index += length;
op
}
RawOperation::Delete(..) => {
let op = if cfg!(debug_assertions) {
Operation::create_delete_with_text(
new_index,
raw_operation.get_original_text(),
)
} else {
Operation::create_delete(new_index, length)
}
.map(|operation| OrderedOperation { order, operation });
order += length;
op
}
}
})
}
/// Create a new `EditedText` with the given operations.
/// The operations must be in the order in which they are meant to be
/// applied. The operations must not overlap.
fn new(
text: &'a str,
operations: Vec<OrderedOperation<T>>,
mut cursors: Vec<CursorPosition>,
) -> Self {
operations
.iter()
.zip(operations.iter().skip(1))
.for_each(|(previous, next)| {
debug_assert!(
previous.operation.start_index() <= next.operation.start_index(),
"{} must not come before {} yet it does",
previous.operation,
next.operation
);
});
cursors.sort_by_key(|cursor| cursor.char_index);
Self {
text,
operations,
cursors,
}
}
#[must_use]
pub fn merge(self, other: Self) -> Self {
debug_assert_eq!(
self.text, other.text,
"`EditedText`-s must be derived from the same text to be mergable"
);
let mut left_merge_context = MergeContext::default();
let mut right_merge_context = MergeContext::default();
let mut merged_cursors = Vec::with_capacity(self.cursors.len() + other.cursors.len());
let mut left_cursors = self.cursors.into_iter().peekable();
let mut right_cursors = other.cursors.into_iter().peekable();
let merged_operations: Vec<OrderedOperation<T>> = self
.operations
.into_iter()
// The current text is always the left; the other operation is the right side.
.map(|op| (op, Side::Left))
.merge_sorted_by_key(
other.operations.into_iter().map(|op| (op, Side::Right)),
|(operation, _)| {
(
operation.order,
operation.operation.start_index(),
// Make sure that the ordering is deterministic regardless which text
// is left or right.
match &operation.operation {
Operation::Equal { index, .. } => index.to_string(),
Operation::Insert { text, .. } => text
.iter()
.map(crate::tokenizer::token::Token::original)
.collect::<String>(),
Operation::Delete {
deleted_character_count,
..
} => deleted_character_count.to_string(),
},
)
},
)
.flat_map(|(OrderedOperation { order, operation }, side)| {
let original_start = operation.start_index() as i64;
let original_end = operation.end_index();
let original_length = operation.len() as i64;
let result = match side {
Side::Left => operation.merge_operations_with_context(
&mut right_merge_context,
&mut left_merge_context,
),
Side::Right => operation.merge_operations_with_context(
&mut left_merge_context,
&mut right_merge_context,
),
};
if let Some(ref op @ (Operation::Insert { .. } | Operation::Equal { .. })) = result
{
let shift = op.start_index() as i64 - original_start + op.len() as i64
- original_length;
match side {
Side::Left => {
while let Some(cursor) =
left_cursors.next_if(|cursor| cursor.char_index <= original_end + 1)
{
merged_cursors.push(cursor.with_index(
(op.start_index() as i64).max(cursor.char_index as i64 + shift)
as usize,
));
}
}
Side::Right => {
while let Some(cursor) = right_cursors
.next_if(|cursor| cursor.char_index <= original_end + 1)
{
merged_cursors.push(cursor.with_index(
(op.start_index() as i64).max(cursor.char_index as i64 + shift)
as usize,
));
}
}
}
}
result
.map(|operation| OrderedOperation { order, operation })
.into_iter()
})
.collect();
let last_index = merged_operations
.iter()
.filter(|operation| {
matches!(
operation.operation,
Operation::Insert { .. } | Operation::Equal { .. }
)
})
.next_back()
.map_or(0, |op| op.operation.end_index());
for cursor in left_cursors.chain(right_cursors) {
merged_cursors.push(cursor.with_index(last_index));
}
Self::new(self.text, merged_operations, merged_cursors)
}
/// Apply the operations to the text and return the resulting text.
#[must_use]
pub fn apply(&self) -> String {
let mut builder: StringBuilder<'_> = StringBuilder::new(self.text);
for OrderedOperation { operation, .. } in &self.operations {
builder = operation.apply(builder);
}
builder.build()
}
}
#[cfg(test)]
mod tests {
use std::env;
use insta::assert_debug_snapshot;
use pretty_assertions::assert_eq;
use super::*;
#[test]
fn test_calculate_operations() {
let left = "hello world! How are you? Adam";
let right = "Hello, my friend! How are you doing? Albert";
let operations = EditedText::from_strings(left, right.into());
insta::assert_debug_snapshot!(operations);
let new_right = operations.apply();
assert_eq!(new_right.to_string(), right);
}
#[test]
fn test_calculate_operations_with_no_diff() {
let text = "hello world!";
let operations = EditedText::from_strings(text, text.into());
assert_debug_snapshot!(operations);
let new_right = operations.apply();
assert_eq!(new_right.to_string(), text);
}
#[test]
fn test_calculate_operations_with_insert() {
let original = "hello world! ...";
let left = "Hello world! I'm Andras.";
let right = "Hello world! How are you?";
let expected = "Hello world! How are you? I'm Andras.";
let operations_1 = EditedText::from_strings(original, left.into());
let operations_2 = EditedText::from_strings(original, right.into());
let operations = operations_1.merge(operations_2);
assert_eq!(operations.apply(), expected);
}
}

Some files were not shown because too many files have changed in this diff Show more