Make insert deduplication smarter

This commit is contained in:
Andras Schmelczer 2026-10-27 20:52:29 +00:00
commit c566eb7ef6

View file

@ -1,9 +1,10 @@
use std::fmt::Debug;
use std::{fmt::Debug, ops::Range};
use crate::Token;
/// Given two lists of tokens, returns `length` where the `old` list
/// somewhere within contains the `length` prefix of the `new` list
/// Returns the range in `old` containing the longest prefix of `new`.
/// Uses the first occurrence when that prefix appears more than once,
/// or `0..0` if there is no overlap.
///
/// ## Example
///
@ -11,21 +12,21 @@ use crate::Token;
/// old: [0, 1, 9, 0, 2, 5]
/// new: [9, 0, 2, 5, 1]
/// ```
/// > results in a length of 4
/// > results in a range of 2..6
///
///
/// ```not_rust
/// old: [0, 1, 9, 0, 2, 5]
/// new: [0, 2]
/// ```
/// > results in a length of 2
/// > results in a range of 3..5
///
/// ```not_rust
/// old: [0, 1, 9, 0, 2, 5]
/// new: [0, 4]
/// new: [0, 4]
/// ```
/// > results in a length of 1
pub fn find_longest_prefix_contained_within<T>(old: &[Token<T>], new: &[Token<T>]) -> usize
/// > results in a range of 0..1
pub fn find_longest_prefix_contained_within<T>(old: &[Token<T>], new: &[Token<T>]) -> Range<usize>
where
T: PartialEq + Clone + Debug,
{
@ -33,12 +34,12 @@ where
for len in (1..=max_possible).rev() {
let prefix = &new[..len];
if old.windows(len).any(|window| window == prefix) {
return len;
if let Some(start) = old.windows(len).position(|window| window == prefix) {
return start..start + len;
}
}
0
0..0
}
#[cfg(test)]
@ -51,7 +52,7 @@ mod tests {
fn test_common_overlap() {
assert_eq!(
find_longest_prefix_contained_within(&["".into()], &["".into()]),
1
0..1
);
assert_eq!(
@ -59,7 +60,7 @@ mod tests {
&["a".into(), "b".into(), "c".into()],
&["b".into(), "c".into(), "a".into()]
),
2
1..3
);
assert_eq!(
@ -67,7 +68,7 @@ mod tests {
&["a".into(), "b".into(), "c".into()],
&["b".into(), "c".into()]
),
2
1..3
);
assert_eq!(
@ -75,7 +76,7 @@ mod tests {
&["a".into(), "b".into(), "c".into()],
&["b".into()]
),
1
1..2
);
assert_eq!(
@ -83,7 +84,7 @@ mod tests {
&["a".into(), "b".into(), "c".into(), "b".into(), "a".into()],
&["b".into(), "a".into()]
),
2
3..5
);
assert_eq!(
@ -91,7 +92,15 @@ mod tests {
&["a".into(), "a".into(), "a".into()],
&["a".into(), "b".into(), "c".into()]
),
1
0..1
);
assert_eq!(
find_longest_prefix_contained_within(
&["a".into(), "b".into(), "a".into(), "b".into()],
&["a".into(), "b".into()]
),
0..2
);
assert_eq!(
@ -99,7 +108,7 @@ mod tests {
&["a".into(), "b".into(), "c".into()],
&["d".into(), "e".into(), "a".into()]
),
0
0..0
);
}
}