Fix syncing when network latency is present (#4)
* WIP * Add debug * Dedupe inserts * Add deterministic ordering * Fix whitespaces * Update insta * Add integration test script * Rename * Add test * Working for non-deletes * omg it mostly works for deletes * Isdeleted fix * remove created dates * update api * Take document id * No max attempt * works * Use string uuids * . * working!!!! (hopefully) * Improve bundling * Add module * lint * . * lint * Fix CI * use toolchain * clean up * Add useSlowFileEvents * Delete fuzz * Fix CI * use docker * fix script * clean up * Clean up * change node version * Build docker image on every commit * fix ci * 1 db per vault * Add scritps folder * Bump versions * Lint * . * Fix tests for real * Style * . * try * Consistent ordering * Fix tests * hmm * . * Clean up diff * Fixes * . * Fix version bump * . * . * .
This commit is contained in:
GitHub
parent
bcf48c428d
commit
8b8f1d91d9
91 changed files with 2216 additions and 1550 deletions
13
.github/workflows/check.yml
vendored
13
.github/workflows/check.yml
vendored
|
|
@ -17,11 +17,14 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: "22.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Setup rust
|
||||
run: |
|
||||
rustup install nightly
|
||||
rustup default nightly
|
||||
rustup component add clippy rustfmt
|
||||
cargo install sqlx-cli
|
||||
cd backend
|
||||
sqlx database create --database-url sqlite://db.sqlite3
|
||||
|
|
@ -44,7 +47,7 @@ jobs:
|
|||
cd backend
|
||||
cargo test --verbose
|
||||
cd sync_lib
|
||||
# wasm-pack test --node # todo: fix this in CI
|
||||
wasm-pack test --node
|
||||
|
||||
- name: Lint frontend
|
||||
run: |
|
||||
|
|
|
|||
45
.github/workflows/e2e.yml
vendored
Normal file
45
.github/workflows/e2e.yml
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
name: E2E tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["master"]
|
||||
pull_request:
|
||||
branches: ["master"]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTFLAGS: "-Dwarnings"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: self-hosted
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: "22.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Setup rust
|
||||
run: |
|
||||
cargo install sqlx-cli wasm-pack
|
||||
cd backend
|
||||
sqlx database create --database-url sqlite://db.sqlite3
|
||||
sqlx migrate run --source sync_server/src/database/migrations --database-url sqlite://db.sqlite3
|
||||
|
||||
- name: Build wasm
|
||||
run: |
|
||||
cd backend
|
||||
wasm-pack build --target web sync_lib
|
||||
|
||||
- name: E2E tests
|
||||
run: |
|
||||
cd backend
|
||||
RUST_BACKTRACE=1 cargo run -p sync_server &
|
||||
cd ../frontend
|
||||
npm ci
|
||||
cd ..
|
||||
scripts/e2e.sh 32
|
||||
18
.github/workflows/publish-docker.yml
vendored
18
.github/workflows/publish-docker.yml
vendored
|
|
@ -7,8 +7,9 @@ name: Publish server Docker image
|
|||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "*"
|
||||
branches: ["master"]
|
||||
pull_request:
|
||||
branches: ["master"]
|
||||
|
||||
env:
|
||||
# Use docker.io for Docker Hub if empty
|
||||
|
|
@ -17,8 +18,9 @@ env:
|
|||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
runs-on: ubuntu-latest
|
||||
publish-docker:
|
||||
runs-on: self-hosted
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
|
@ -33,7 +35,7 @@ jobs:
|
|||
# Install the cosign tool except on PR
|
||||
# https://github.com/sigstore/cosign-installer
|
||||
- name: Install cosign
|
||||
if: github.event_name != 'pull_request'
|
||||
if: ${{ github.ref_type == 'tag' }}
|
||||
uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
|
||||
with:
|
||||
cosign-release: "v2.2.4"
|
||||
|
|
@ -47,7 +49,7 @@ jobs:
|
|||
# Login against a Docker registry except on PR
|
||||
# https://github.com/docker/login-action
|
||||
- name: Log into registry ${{ env.REGISTRY }}
|
||||
if: github.event_name != 'pull_request'
|
||||
if: ${{ github.ref_type == 'tag' }}
|
||||
uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
|
|
@ -69,7 +71,7 @@ jobs:
|
|||
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
|
||||
with:
|
||||
context: backend
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
push: ${{ github.ref_type == 'tag' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
|
|
@ -81,7 +83,7 @@ jobs:
|
|||
# transparency data even for private images, pass --force to cosign below.
|
||||
# https://github.com/sigstore/cosign
|
||||
- name: Sign the published Docker image
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
if: ${{ github.ref_type == 'tag' }}
|
||||
env:
|
||||
# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
|
||||
TAGS: ${{ steps.meta.outputs.tags }}
|
||||
|
|
|
|||
16
.github/workflows/publish-plugin.yml
vendored
16
.github/workflows/publish-plugin.yml
vendored
|
|
@ -2,29 +2,27 @@ name: Publish Obsidian plugin
|
|||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "*"
|
||||
tags: ["*"]
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
build-plugin:
|
||||
publish-plugin:
|
||||
runs-on: self-hosted
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Use Node.js
|
||||
uses: actions/setup-node@v3
|
||||
- name: Setup Node.js environment
|
||||
uses: actions/setup-node@v4.2.0
|
||||
with:
|
||||
node-version: "18.x"
|
||||
node-version: "22.x"
|
||||
check-latest: true
|
||||
|
||||
- name: Build wasm
|
||||
run: |
|
||||
cd backend
|
||||
rustup install nightly
|
||||
rustup default nightly
|
||||
cargo install wasm-pack
|
||||
wasm-pack build --target web sync_lib
|
||||
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -10,6 +10,7 @@ backend/target
|
|||
frontend/*/dist
|
||||
|
||||
backend/db.sqlite3*
|
||||
backend/databases
|
||||
backend/config.yml
|
||||
|
||||
*.log
|
||||
|
|
|
|||
79
README.md
79
README.md
|
|
@ -1,65 +1,56 @@
|
|||
## VaultLink self-hosted Obsidian sync plugin
|
||||
# VaultLink self-hosted Obsidian plugin for file syncing
|
||||
|
||||
[](https://github.com/schmelczer/vault-link/actions/workflows/check.yml)
|
||||
[](https://github.com/schmelczer/vault-link/actions/workflows/e2e.yml)
|
||||
[](https://github.com/schmelczer/vault-link/actions/workflows/publish-docker.yml)
|
||||
[](https://github.com/schmelczer/vault-link/actions/workflows/publish-plugin.yml)
|
||||
|
||||
|
||||
## Install [nvm](https://github.com/nvm-sh/nvm)
|
||||
## Develop
|
||||
|
||||
### Install [nvm](https://github.com/nvm-sh/nvm)
|
||||
|
||||
- `curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash`
|
||||
- `nvm install 20`
|
||||
- `nvm use 20`
|
||||
- Optionally set the system-wide default: `nvm alias default 20`
|
||||
- `nvm install 22`
|
||||
- `nvm use 22`
|
||||
- Optionally set the system-wide default: `nvm alias default 22`
|
||||
|
||||
|
||||
## Set up Rust
|
||||
### Set up Rust
|
||||
|
||||
- Install [`rustup`](https://rustup.rs): `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh`
|
||||
- `sudo apt install llvm -y`
|
||||
- `rustup self update`
|
||||
- `rustup update`
|
||||
- `rustup install nightly`
|
||||
- `rustup default nightly`
|
||||
- `rustup component add llvm-tools-preview`
|
||||
- `cargo install cargo-generate cargo-fuzz cargo-insta rustfilt cargo-binutils`
|
||||
- Install [`wasm-pack`](https://rustwasm.github.io/wasm-pack/installer): `curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh`
|
||||
- `cargo install cargo-insta sqlx-cli cargo-edit`
|
||||
|
||||
|
||||
## Publish new version
|
||||
### Install Obsidian on Linux
|
||||
|
||||
```sh
|
||||
./bump-version.sh patch
|
||||
```
|
||||
|
||||
|
||||
## Update HTTP API TS bindings
|
||||
|
||||
```sh
|
||||
npm install -g openapi-typescript
|
||||
openapi-typescript http://localhost:3030/api.json --output frontend/sync-client/src/services/types.ts
|
||||
```
|
||||
|
||||
```
|
||||
|
||||
todo: enable
|
||||
[workspace.lints.clippy]
|
||||
single_call_fn = { level = "allow", priority = 1 }
|
||||
absolute_paths = { level = "allow", priority = 1 }
|
||||
arithmetic_side_effects = { level = "allow", priority = 1 }
|
||||
similar_names = { level = "allow", priority = 1 }
|
||||
self_named_module_files = { level = "allow", priority = 1 }
|
||||
single_char_lifetime_names = { level = "allow", priority = 1 }
|
||||
missing_docs_in_private_items = { level = "allow", priority = 1 }
|
||||
question_mark_used = { level = "allow", priority = 1 }
|
||||
implicit_return = { level = "allow", priority = 1 }
|
||||
pedantic = { level = "warn", priority = 0 }
|
||||
cargo = { level = "warn", priority = 0 }
|
||||
|
||||
```
|
||||
|
||||
apt install flatpak
|
||||
flatpak remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo
|
||||
flatpak install flathub md.obsidian.Obsidian
|
||||
flatpak run md.obsidian.Obsidian
|
||||
```
|
||||
|
||||
### Scripts
|
||||
|
||||
#### Update HTTP API TS bindings
|
||||
|
||||
```sh
|
||||
scripts/update-api-types.sh
|
||||
```
|
||||
|
||||
#### Publish new version
|
||||
|
||||
```sh
|
||||
scripts/bump-version.sh patch
|
||||
```
|
||||
|
||||
|
||||
#### Run E2E tests
|
||||
|
||||
```sh
|
||||
scripts/e2e.sh
|
||||
```
|
||||
|
||||
And to clean up the logs & database files, run `scripts/clean-up.sh`
|
||||
```
|
||||
|
|
|
|||
35
backend/Cargo.lock
generated
35
backend/Cargo.lock
generated
|
|
@ -105,12 +105,6 @@ dependencies = [
|
|||
"backtrace",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.85"
|
||||
|
|
@ -381,8 +375,6 @@ version = "1.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
|
|
@ -1228,15 +1220,6 @@ version = "1.0.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.76"
|
||||
|
|
@ -1290,16 +1273,6 @@ version = "0.2.167"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
|
||||
|
||||
[[package]]
|
||||
name = "libfuzzer-sys"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b9569d2f74e257076d8c6bfa73fb505b46b851e51ddaecc825944aa3bed17fa"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.11"
|
||||
|
|
@ -1781,14 +1754,6 @@ dependencies = [
|
|||
"test-case",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reconcile-fuzz"
|
||||
version = "0.0.30"
|
||||
dependencies = [
|
||||
"libfuzzer-sys",
|
||||
"reconcile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.7"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
resolver = "2"
|
||||
members = [
|
||||
"reconcile",
|
||||
"fuzz",
|
||||
"sync_server",
|
||||
"sync_lib"
|
||||
]
|
||||
|
|
@ -57,3 +56,19 @@ uninlined_format_args = "warn"
|
|||
unnested_or_patterns = "warn"
|
||||
unused_self = "warn"
|
||||
verbose_file_reads = "warn"
|
||||
|
||||
cast_possible_truncation = { level = "allow", priority = 1 }
|
||||
doc_link_with_quotes = { level = "allow", priority = 1 }
|
||||
cast_sign_loss = { level = "allow", priority = 1 }
|
||||
cast_possible_wrap = { level = "allow", priority = 1 }
|
||||
struct_field_names = { level = "allow", priority = 1 }
|
||||
single_call_fn = { level = "allow", priority = 1 }
|
||||
absolute_paths = { level = "allow", priority = 1 }
|
||||
arithmetic_side_effects = { level = "allow", priority = 1 }
|
||||
similar_names = { level = "allow", priority = 1 }
|
||||
self_named_module_files = { level = "allow", priority = 1 }
|
||||
single_char_lifetime_names = { level = "allow", priority = 1 }
|
||||
missing_docs_in_private_items = { level = "allow", priority = 1 }
|
||||
question_mark_used = { level = "allow", priority = 1 }
|
||||
implicit_return = { level = "allow", priority = 1 }
|
||||
pedantic = { level = "warn", priority = 0 }
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ FROM rust:1.83 AS builder
|
|||
WORKDIR /usr/src/backend
|
||||
|
||||
RUN apt update && apt install -y musl-tools
|
||||
RUN rustup install nightly && rustup default nightly
|
||||
RUN rustup target add x86_64-unknown-linux-musl
|
||||
RUN cargo install sqlx-cli
|
||||
|
||||
COPY . .
|
||||
|
|
@ -23,7 +21,7 @@ RUN apk add --no-cache curl
|
|||
|
||||
COPY --from=builder /usr/src/backend/target/x86_64-unknown-linux-musl/release/sync_server /app/sync_server
|
||||
|
||||
VOLUME /data
|
||||
VOLUME /data/databases
|
||||
EXPOSE 3000/tcp
|
||||
|
||||
WORKDIR /data
|
||||
|
|
|
|||
4
backend/fuzz/.gitignore
vendored
4
backend/fuzz/.gitignore
vendored
|
|
@ -1,4 +0,0 @@
|
|||
target
|
||||
corpus
|
||||
artifacts
|
||||
coverage
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
[package]
|
||||
name = "reconcile-fuzz"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
publish = false
|
||||
|
||||
[package.metadata]
|
||||
cargo-fuzz = true
|
||||
|
||||
[dependencies]
|
||||
libfuzzer-sys = "0.4"
|
||||
reconcile = { path = "../reconcile" }
|
||||
|
||||
[[bin]]
|
||||
name = "reconcile"
|
||||
path = "fuzz_targets/reconcile.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
#![no_main]
|
||||
|
||||
use libfuzzer_sys::fuzz_target;
|
||||
|
||||
fuzz_target!(|texts: (String, String, String)| {
|
||||
let (original, left, right) = texts;
|
||||
let _ = reconcile::reconcile(&original, &left, &right);
|
||||
});
|
||||
|
|
@ -38,7 +38,7 @@ use crate::{
|
|||
/// execution time permitted before it bails and falls back to an approximation.
|
||||
pub fn diff<T>(old: &[Token<T>], new: &[Token<T>]) -> Vec<RawOperation<T>>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
let max_d = (old.len() + new.len()).div_ceil(2) + 1;
|
||||
let mut vb = V::new(max_d);
|
||||
|
|
@ -99,7 +99,6 @@ impl IndexMut<isize> for V {
|
|||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn split_at(range: Range<usize>, at: usize) -> (Range<usize>, Range<usize>) {
|
||||
(range.start..at, at..range.end)
|
||||
}
|
||||
|
|
@ -124,7 +123,7 @@ fn find_middle_snake<T>(
|
|||
vb: &mut V,
|
||||
) -> Option<(usize, usize)>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
let n = old_range.len();
|
||||
let m = new_range.len();
|
||||
|
|
@ -230,7 +229,7 @@ fn conquer<T>(
|
|||
vb: &mut V,
|
||||
result: &mut Vec<RawOperation<T>>,
|
||||
) where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
// Check for common prefix
|
||||
let common_prefix_len = common_prefix_len(old, old_range.clone(), new, new_range.clone());
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use crate::tokenizer::token::Token;
|
|||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum RawOperation<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
Insert(Vec<Token<T>>),
|
||||
Delete(Vec<Token<T>>),
|
||||
|
|
@ -12,13 +12,13 @@ where
|
|||
|
||||
impl<T> RawOperation<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
pub fn tokens(&self) -> &Vec<Token<T>> {
|
||||
match self {
|
||||
RawOperation::Insert(tokens) => tokens,
|
||||
RawOperation::Delete(tokens) => tokens,
|
||||
RawOperation::Equal(tokens) => tokens,
|
||||
RawOperation::Insert(tokens)
|
||||
| RawOperation::Delete(tokens)
|
||||
| RawOperation::Equal(tokens) => tokens,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ pub fn reconcile_with_tokenizer<F, T>(
|
|||
tokenizer: &Tokenizer<T>,
|
||||
) -> String
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
let left_operations = EditedText::from_strings_with_tokenizer(original, left, tokenizer);
|
||||
let right_operations = EditedText::from_strings_with_tokenizer(original, right, tokenizer);
|
||||
|
|
@ -73,7 +73,8 @@ mod test {
|
|||
"original_1 edit_1 original_3",
|
||||
);
|
||||
|
||||
// One deleted a large range, the other deleted subranges and inserted as well
|
||||
// One deleted a large range, the other deleted subranges and inserted as
|
||||
// well
|
||||
test_merge_both_ways(
|
||||
"original_1 original_2 original_3 original_4 original_5",
|
||||
"original_1 original_5",
|
||||
|
|
@ -120,9 +121,6 @@ mod test {
|
|||
"hi, my friend!",
|
||||
);
|
||||
|
||||
// test_merge_both_ways("hello world", "world !", "hi hello world", "hi world
|
||||
// !");
|
||||
|
||||
test_merge_both_ways(
|
||||
"both delete the same word",
|
||||
"both the same word",
|
||||
|
|
@ -147,7 +145,33 @@ mod test {
|
|||
);
|
||||
}
|
||||
|
||||
#[ignore = "it's too slow"]
|
||||
#[test]
|
||||
fn test_reconcile_idempotent_inserts() {
|
||||
// Both inserted the same prefix; this should get deduped
|
||||
test_merge_both_ways(
|
||||
"hi ",
|
||||
"hi there ",
|
||||
"hi there my friend ",
|
||||
"hi there my friend ",
|
||||
);
|
||||
|
||||
// The prefix of the 2nd appears on the 1st so it shouldn't get duplicated
|
||||
test_merge_both_ways(
|
||||
"hi ",
|
||||
"hi there you ",
|
||||
"hi there my friend ",
|
||||
"hi there my friend you ",
|
||||
);
|
||||
|
||||
test_merge_both_ways("a", "a b c", "a b c d", "a b c d");
|
||||
|
||||
test_merge_both_ways(
|
||||
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| ",
|
||||
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| ",
|
||||
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |dup| ",
|
||||
" |7ca2b36d-6ee7-49eb-8eb1-d77e4cc1a001| |cd9195cc-103a-4f13-90c8-4fba0ba421ee| |d39156cc-cfd6-42a8-b70a-75020896069d| |fbad794c-9c47-41f2-a343-490284ecb5a0| |dup| |dup| ");
|
||||
}
|
||||
|
||||
#[test_matrix( [
|
||||
"pride_and_prejudice.txt",
|
||||
"romeo_and_juliet.txt",
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ use crate::{
|
|||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct EditedText<'a, T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
text: &'a str,
|
||||
operations: Vec<OrderedOperation<T>>,
|
||||
|
|
@ -46,7 +46,7 @@ impl<'a> EditedText<'a, String> {
|
|||
|
||||
impl<'a, T> EditedText<'a, T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
/// Create an `EditedText` from the given original (old) and updated (new)
|
||||
/// strings. The returned `EditedText` represents the changes from the
|
||||
|
|
@ -65,7 +65,6 @@ where
|
|||
|
||||
Self::new(
|
||||
original,
|
||||
// Self::cook_operations(diff),
|
||||
Self::cook_operations(Self::elongate_operations(diff)).collect(),
|
||||
)
|
||||
}
|
||||
|
|
@ -191,7 +190,7 @@ where
|
|||
pub fn merge(self, other: Self) -> Self {
|
||||
debug_assert_eq!(
|
||||
self.text, other.text,
|
||||
"EditedText-s must be derived from the same text to be mergable"
|
||||
"`EditedText`-s must be derived from the same text to be mergable"
|
||||
);
|
||||
|
||||
let mut left_merge_context = MergeContext::default();
|
||||
|
|
@ -207,9 +206,21 @@ where
|
|||
|(operation, _)| {
|
||||
(
|
||||
operation.order,
|
||||
// Operations on left and right must come in the same order so that
|
||||
// Operations on the left and right must come in the same order so that
|
||||
// inserts can be merged with other inserts and deletes with deletes.
|
||||
usize::from(matches!(operation.operation, Operation::Delete { .. })),
|
||||
// Make sure that the ordering is deterministic regardless which text
|
||||
// is left or right.
|
||||
match &operation.operation {
|
||||
Operation::Insert { text, .. } => text
|
||||
.iter()
|
||||
.map(super::super::tokenizer::token::Token::original)
|
||||
.collect::<String>(),
|
||||
Operation::Delete {
|
||||
deleted_character_count,
|
||||
..
|
||||
} => deleted_character_count.to_string(),
|
||||
},
|
||||
)
|
||||
},
|
||||
)
|
||||
|
|
@ -232,6 +243,7 @@ where
|
|||
}
|
||||
|
||||
/// Apply the operations to the text and return the resulting text.
|
||||
#[must_use]
|
||||
pub fn apply(&self) -> String {
|
||||
let mut builder: StringBuilder<'_> = StringBuilder::new(self.text);
|
||||
|
||||
|
|
@ -282,7 +294,7 @@ mod tests {
|
|||
let original = "hello world! ...";
|
||||
let left = "Hello world! I'm Andras.";
|
||||
let right = "Hello world! How are you?";
|
||||
let expected = "Hello world! I'm Andras.How are you?";
|
||||
let expected = "Hello world! How are you? I'm Andras.";
|
||||
|
||||
let operations_1 = EditedText::from_strings(original, left);
|
||||
let operations_2 = EditedText::from_strings(original, right);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use crate::operation_transformation::Operation;
|
|||
#[derive(Clone)]
|
||||
pub struct MergeContext<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
last_operation: Option<Operation<T>>,
|
||||
pub shift: i64,
|
||||
|
|
@ -13,7 +13,7 @@ where
|
|||
|
||||
impl<T> Default for MergeContext<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
fn default() -> Self {
|
||||
MergeContext {
|
||||
|
|
@ -25,7 +25,7 @@ where
|
|||
|
||||
impl<T> Debug for MergeContext<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
f.debug_struct("MergeContext")
|
||||
|
|
@ -37,7 +37,7 @@ where
|
|||
|
||||
impl<T> MergeContext<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
pub fn last_operation(&self) -> Option<&Operation<T>> { self.last_operation.as_ref() }
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
use core::{
|
||||
fmt::{Debug, Display},
|
||||
ops::Range,
|
||||
};
|
||||
use core::fmt::{Debug, Display};
|
||||
use std::ops::Range;
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
|
@ -9,7 +7,10 @@ use serde::{Deserialize, Serialize};
|
|||
use super::merge_context::MergeContext;
|
||||
use crate::{
|
||||
Token,
|
||||
utils::{find_common_overlap::find_common_overlap, string_builder::StringBuilder},
|
||||
utils::{
|
||||
find_longest_prefix_contained_within::find_longest_prefix_contained_within,
|
||||
string_builder::StringBuilder,
|
||||
},
|
||||
};
|
||||
|
||||
/// Represents a change that can be applied to a text document.
|
||||
|
|
@ -19,7 +20,7 @@ use crate::{
|
|||
#[derive(Clone, PartialEq)]
|
||||
pub enum Operation<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
Insert {
|
||||
index: usize,
|
||||
|
|
@ -37,7 +38,7 @@ where
|
|||
|
||||
impl<T> Operation<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
/// Creates an insert operation with the given index and text.
|
||||
/// If the text is empty (meaning that the operation would be a no-op),
|
||||
|
|
@ -81,15 +82,8 @@ where
|
|||
})
|
||||
}
|
||||
|
||||
/// Tries to apply the operation to the given `ropey::Rope` text, returning
|
||||
/// the modified text.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns a `SyncLibError::OperationApplicationError` if the operation
|
||||
/// cannot be applied.
|
||||
///
|
||||
/// # Panics
|
||||
/// Applies the operation to the given `StringBuilder`, returning the
|
||||
/// modified `StringBuilder`.
|
||||
///
|
||||
/// When compiled in debug mode, panics if a delete operation is attempted
|
||||
/// on a range of text that does not match the text to be deleted.
|
||||
|
|
@ -114,7 +108,7 @@ where
|
|||
|
||||
builder.delete(self.range());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
builder
|
||||
}
|
||||
|
|
@ -122,8 +116,7 @@ where
|
|||
/// Returns the index of the first character that the operation affects.
|
||||
pub fn start_index(&self) -> usize {
|
||||
match self {
|
||||
Operation::Insert { index, .. } => *index,
|
||||
Operation::Delete { index, .. } => *index,
|
||||
Operation::Insert { index, .. } | Operation::Delete { index, .. } => *index,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -137,6 +130,7 @@ where
|
|||
}
|
||||
|
||||
/// Returns the range of indices of characters that the operation affects.
|
||||
#[allow(clippy::range_plus_one)]
|
||||
pub fn range(&self) -> Range<usize> { self.start_index()..self.end_index() + 1 }
|
||||
|
||||
/// Returns the number of affected characters. It is always greater than 0
|
||||
|
|
@ -212,17 +206,20 @@ where
|
|||
..
|
||||
}),
|
||||
) => {
|
||||
let offset_in_tokens = find_common_overlap(previous_inserted_text, &text);
|
||||
let trimmed_length_in_tokens = previous_inserted_text.len() - offset_in_tokens;
|
||||
let trimmed_length = previous_inserted_text
|
||||
// In case the current insert's prefix appears in the previously inserted text,
|
||||
// we can trim the current insert to only include the non-overlapping part.
|
||||
// This way, we don't end up duplicating text.
|
||||
let offset_in_tokens =
|
||||
find_longest_prefix_contained_within(previous_inserted_text, &text);
|
||||
let offset_in_length = text
|
||||
.iter()
|
||||
.skip(offset_in_tokens)
|
||||
.take(offset_in_tokens)
|
||||
.map(Token::get_original_length)
|
||||
.sum::<usize>();
|
||||
let trimmed_operation =
|
||||
Operation::create_insert(index, text[trimmed_length_in_tokens..].to_vec());
|
||||
Operation::create_insert(index, text[offset_in_tokens..].to_vec());
|
||||
|
||||
affecting_context.shift -= trimmed_length as i64;
|
||||
affecting_context.shift -= offset_in_length as i64;
|
||||
produced_context.shift += trimmed_operation
|
||||
.as_ref()
|
||||
.map(Operation::len)
|
||||
|
|
@ -297,7 +294,7 @@ where
|
|||
|
||||
impl<T> Display for Operation<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
match self {
|
||||
|
|
@ -341,7 +338,7 @@ where
|
|||
|
||||
impl<T> Debug for Operation<T>
|
||||
where
|
||||
T: PartialEq + Clone,
|
||||
T: PartialEq + Clone + std::fmt::Debug,
|
||||
{
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "{self}") }
|
||||
}
|
||||
|
|
@ -353,7 +350,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
#[test]
|
||||
#[should_panic]
|
||||
#[should_panic(expected = "Shifted index must be non-negative")]
|
||||
fn test_shifting_error() {
|
||||
insta::assert_debug_snapshot!(
|
||||
Operation::create_insert(1, vec!["hi".into()])
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ EditedText {
|
|||
},
|
||||
OrderedOperation {
|
||||
order: 0,
|
||||
operation: <delete 'hello world! ' from index 18>,
|
||||
operation: <delete 'hello world!' from index 17>,
|
||||
},
|
||||
OrderedOperation {
|
||||
order: 21,
|
||||
operation: <insert 'you doing? Albert' from index 26>,
|
||||
order: 20,
|
||||
operation: <insert ' you doing? Albert' from index 25>,
|
||||
},
|
||||
OrderedOperation {
|
||||
order: 21,
|
||||
order: 20,
|
||||
operation: <delete ' you? Adam' from index 43>,
|
||||
},
|
||||
],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
source: reconcile/src/tokenizer/word_tokenizer.rs
|
||||
expression: "word_tokenizer(\"\")"
|
||||
snapshot_kind: text
|
||||
---
|
||||
[]
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue