andras/reconcile
andras/reconcile
Watch
1
0
Fork
You've already forked reconcile
 0
Code Issues Pull requests 10 Projects Releases Packages Wiki Activity Actions

Add character tokenizer

Browse source
This commit is contained in:
Andras Schmelczer 2025-06-29 19:35:21 +01:00
commit 9cb73680f8
2 changed files with 177 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

144
src/tokenizer/snapshots/reconcile__tokenizer__character_tokenizer__tests__with_snapshots-2.snap Normal file
View file

@ -0,0 +1,144 @@
---
source: src/tokenizer/character_tokenizer.rs
expression: "character_tokenizer(\" hello, \\nwhere are you?\")"
---
[
Token {
normalised: " ",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "h",
original: "h",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "e",
original: "e",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "l",
original: "l",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "l",
original: "l",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "o",
original: "o",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: ",",
original: ",",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: " ",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "w",
original: "w",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "h",
original: "h",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "e",
original: "e",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "r",
original: "r",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "e",
original: "e",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: " ",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "r",
original: "r",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "e",
original: "e",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: " ",
original: " ",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "y",
original: "y",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "o",
original: "o",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "u",
original: "u",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "?",
original: "?",
is_left_joinable: true,
is_right_joinable: true,
},
]

5
src/tokenizer/snapshots/reconcile__tokenizer__character_tokenizer__tests__with_snapshots.snap Normal file
View file

@ -0,0 +1,5 @@
---
source: src/tokenizer/character_tokenizer.rs
expression: "character_tokenizer(\"\")"
---
[]