andras/reconcile
andras/reconcile
Watch
1
0
Fork
You've already forked reconcile
 0
Code Issues Pull requests 10 Projects Releases Packages Wiki Activity Actions

Add line tokenizer

Browse source
This commit is contained in:
Andras Schmelczer 2025-07-06 13:03:25 +01:00
commit 469e62106c
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
9 changed files with 324 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-2.snap Normal file
View file

@ -0,0 +1,13 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Hello\")"
snapshot_kind: text
---
[
Token {
normalised: "Hello",
original: "Hello",
is_left_joinable: true,
is_right_joinable: true,
},
]

25
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-3.snap Normal file
View file

@ -0,0 +1,25 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Hello\\nWorld\")"
snapshot_kind: text
---
[
Token {
normalised: "Hello",
original: "Hello",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "World",
original: "World",
is_left_joinable: true,
is_right_joinable: true,
},
]

31
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-4.snap Normal file
View file

@ -0,0 +1,31 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Hello\\nWorld\\n\")"
snapshot_kind: text
---
[
Token {
normalised: "Hello",
original: "Hello",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "World",
original: "World",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
]

25
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-5.snap Normal file
View file

@ -0,0 +1,25 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Line 1\\r\\nLine 2\")"
snapshot_kind: text
---
[
Token {
normalised: "Line 1",
original: "Line 1",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\r\n",
original: "\r\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Line 2",
original: "Line 2",
is_left_joinable: true,
is_right_joinable: true,
},
]

49
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-6.snap Normal file
View file

@ -0,0 +1,49 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Multi\\nLine\\nText\\nHere\")"
snapshot_kind: text
---
[
Token {
normalised: "Multi",
original: "Multi",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Line",
original: "Line",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Text",
original: "Text",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "Here",
original: "Here",
is_left_joinable: true,
is_right_joinable: true,
},
]

13
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-7.snap Normal file
View file

@ -0,0 +1,13 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"\\n\")"
snapshot_kind: text
---
[
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
]

19
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-8.snap Normal file
View file

@ -0,0 +1,19 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"\\n\\n\")"
snapshot_kind: text
---
[
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
]

31
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots-9.snap Normal file
View file

@ -0,0 +1,31 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"Start\\n\\nEnd\")"
snapshot_kind: text
---
[
Token {
normalised: "Start",
original: "Start",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "\n",
original: "\n",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "End",
original: "End",
is_left_joinable: true,
is_right_joinable: true,
},
]

6
src/tokenizer/snapshots/reconcile__tokenizer__line_tokenizer__tests__with_snapshots.snap Normal file
View file

@ -0,0 +1,6 @@
---
source: src/tokenizer/line_tokenizer.rs
expression: "line_tokenizer(\"\")"
snapshot_kind: text
---
[]