andras/reconcile
andras/reconcile
Watch
1
0
Fork
You've already forked reconcile
 0
Code Issues Pull requests 10 Projects Releases Packages Wiki Activity Actions

Add line tokenizer

Browse source
This commit is contained in:
Andras Schmelczer 2025-07-06 13:03:25 +01:00
commit 469e62106c
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
10 changed files with 324 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

70
src/tokenizer/line_tokenizer.rs Normal file
View file

@ -0,0 +1,70 @@
use super::token::Token;
/// Splits text into lines, preserving line endings as separate tokens.
///
/// ## Example
///
/// ```not_rust
/// "Hello\nWorld!" -> ["Hello", "\n", "World!"]
/// "Line 1\r\nLine 2" -> ["Line 1", "\r\n", "Line 2"]
/// ```
pub fn line_tokenizer(text: &str) -> Vec<Token<String>> {
let mut result = Vec::new();
let mut line_start = 0;
let mut chars = text.char_indices().peekable();
while let Some((i, c)) = chars.next() {
if c == '\n' {
// Add line content if any
if i > line_start {
result.push(text[line_start..i].into());
}
// Add newline
result.push("\n".into());
line_start = i + 1;
} else if c == '\r' && chars.peek() == Some(&(i + 1, '\n')) {
// Handle \r\n
if i > line_start {
result.push(text[line_start..i].into());
}
chars.next(); // consume \n
result.push("\r\n".into());
line_start = i + 2;
}
}
// Add final line if any
if line_start < text.len() {
result.push(text[line_start..].into());
}
result
}
#[cfg(test)]
mod tests {
use insta::assert_debug_snapshot;
use super::*;
#[test]
fn test_with_snapshots() {
assert_debug_snapshot!(line_tokenizer(""));
assert_debug_snapshot!(line_tokenizer("Hello"));
assert_debug_snapshot!(line_tokenizer("Hello\nWorld"));
assert_debug_snapshot!(line_tokenizer("Hello\nWorld\n"));
assert_debug_snapshot!(line_tokenizer("Line 1\r\nLine 2"));
assert_debug_snapshot!(line_tokenizer("Multi\nLine\nText\nHere"));
assert_debug_snapshot!(line_tokenizer("\n"));
assert_debug_snapshot!(line_tokenizer("\n\n"));
assert_debug_snapshot!(line_tokenizer("Start\n\nEnd"));
}
}

13
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-2.snap Normal file
View file

@ -0,0 +1,13 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Hello\")"
snapshot_kind: text
---
[
Token {
normalised: "Hello",
original: "Hello",
is_left_joinable: true,
is_right_joinable: true,
},
]

25
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-3.snap Normal file
View file

@ -0,0 +1,25 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Hello\\nWorld\")"
snapshot_kind: text
---
[
Token {
normalised: "Hello",
original: "Hello",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "World",
original: "World",
is_left_joinable: true,
is_right_joinable: true,
},
]

31
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-4.snap Normal file
View file

@ -0,0 +1,31 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Hello\\nWorld\\n\")"
snapshot_kind: text
---
[
Token {
normalised: "Hello",
original: "Hello",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "World",
original: "World",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
]

25
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-5.snap Normal file
View file

@ -0,0 +1,25 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Line 1\\r\\nLine 2\")"
snapshot_kind: text
---
[
Token {
normalised: "Line 1",
original: "Line 1",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\r\n",
original: "\r\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Line 2",
original: "Line 2",
is_left_joinable: true,
is_right_joinable: true,
},
]

49
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-6.snap Normal file
View file

@ -0,0 +1,49 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Multi\\nLine\\nText\\nHere\")"
snapshot_kind: text
---
[
Token {
normalised: "Multi",
original: "Multi",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Line",
original: "Line",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Text",
original: "Text",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Here",
original: "Here",
is_left_joinable: true,
is_right_joinable: true,
},
]

13
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-7.snap Normal file
View file

@ -0,0 +1,13 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"\\n\")"
snapshot_kind: text
---
[
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
]

19
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-8.snap Normal file
View file

@ -0,0 +1,19 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"\\n\\n\")"
snapshot_kind: text
---
[
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
]

31
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-9.snap Normal file
View file

@ -0,0 +1,31 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Start\\n\\nEnd\")"
snapshot_kind: text
---
[
Token {
normalised: "Start",
original: "Start",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "End",
original: "End",
is_left_joinable: true,
is_right_joinable: true,
},
]

6
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots.snap Normal file
View file

@ -0,0 +1,6 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"\")"
snapshot_kind: text
---
[]