No known key found for this signature in database
GPG key ID:
FC8F2C3D3D1A718C
16 changed files with
58 additions and
209 deletions
|
|
|
|
@ -1,39 +0,0 @@
|
|
|
|
|
---
|
|
|
|
|
source: reconcile/src/tokenizer/word_tokenizer.rs
|
|
|
|
|
expression: "word_tokenizer(\" hello, \\nwhere are you?\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalised: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: "hello,",
|
|
|
|
|
original: "hello,",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: " \n",
|
|
|
|
|
original: " \n",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: "where",
|
|
|
|
|
original: "where",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: "are",
|
|
|
|
|
original: "are",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalised: "you?",
|
|
|
|
|
original: "you?",
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,145 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/character_tokenizer.rs
|
|
|
|
|
expression: "character_tokenizer(\" hello, \\nwhere are you?\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "h",
|
|
|
|
|
original: "h",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "e",
|
|
|
|
|
original: "e",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "l",
|
|
|
|
|
original: "l",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "l",
|
|
|
|
|
original: "l",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "o",
|
|
|
|
|
original: "o",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: ",",
|
|
|
|
|
original: ",",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "w",
|
|
|
|
|
original: "w",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "h",
|
|
|
|
|
original: "h",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "e",
|
|
|
|
|
original: "e",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "r",
|
|
|
|
|
original: "r",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "e",
|
|
|
|
|
original: "e",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "a",
|
|
|
|
|
original: "a",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "r",
|
|
|
|
|
original: "r",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "e",
|
|
|
|
|
original: "e",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "y",
|
|
|
|
|
original: "y",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "o",
|
|
|
|
|
original: "o",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "u",
|
|
|
|
|
original: "u",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "?",
|
|
|
|
|
original: "?",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,6 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/character_tokenizer.rs
|
|
|
|
|
expression: "character_tokenizer(\"\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[]
|
|
|
|
|
@ -0,0 +1,13 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"Hello\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Hello",
|
|
|
|
|
original: "Hello",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,25 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"Hello\\nWorld\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Hello",
|
|
|
|
|
original: "Hello",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "World",
|
|
|
|
|
original: "World",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,31 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"Hello\\nWorld\\n\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Hello",
|
|
|
|
|
original: "Hello",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "World",
|
|
|
|
|
original: "World",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,25 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"Line 1\\r\\nLine 2\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Line 1",
|
|
|
|
|
original: "Line 1",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\r\n",
|
|
|
|
|
original: "\r\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Line 2",
|
|
|
|
|
original: "Line 2",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,49 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"Multi\\nLine\\nText\\nHere\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Multi",
|
|
|
|
|
original: "Multi",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Line",
|
|
|
|
|
original: "Line",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Text",
|
|
|
|
|
original: "Text",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Here",
|
|
|
|
|
original: "Here",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,13 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"\\n\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,19 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"\\n\\n\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,31 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"Start\\n\\nEnd\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Start",
|
|
|
|
|
original: "Start",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "\n",
|
|
|
|
|
original: "\n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "End",
|
|
|
|
|
original: "End",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,6 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/line_tokenizer.rs
|
|
|
|
|
expression: "line_tokenizer(\"\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[]
|
|
|
|
|
@ -0,0 +1,6 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/word_tokenizer.rs
|
|
|
|
|
expression: "word_tokenizer(\"\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[]
|
|
|
|
|
@ -0,0 +1,25 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/word_tokenizer.rs
|
|
|
|
|
expression: "word_tokenizer(\" what? \")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " what?",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "what?",
|
|
|
|
|
original: "what?",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " ",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,55 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/word_tokenizer.rs
|
|
|
|
|
expression: "word_tokenizer(\" hello, \\nwhere are you?\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " hello,",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "hello,",
|
|
|
|
|
original: "hello,",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " \nwhere",
|
|
|
|
|
original: " \n",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "where",
|
|
|
|
|
original: "where",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " are",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "are",
|
|
|
|
|
original: "are",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " you?",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "you?",
|
|
|
|
|
original: "you?",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
@ -0,0 +1,25 @@
|
|
|
|
|
---
|
|
|
|
|
source: src/tokenizer/word_tokenizer.rs
|
|
|
|
|
expression: "word_tokenizer(\"Hi there!\")"
|
|
|
|
|
snapshot_kind: text
|
|
|
|
|
---
|
|
|
|
|
[
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "Hi",
|
|
|
|
|
original: "Hi",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: " there!",
|
|
|
|
|
original: " ",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
Token {
|
|
|
|
|
normalized: "there!",
|
|
|
|
|
original: "there!",
|
|
|
|
|
is_left_joinable: true,
|
|
|
|
|
is_right_joinable: true,
|
|
|
|
|
},
|
|
|
|
|
]
|