Add left/right joinability for tokens

This commit is contained in:
Andras Schmelczer 2025-04-05 13:48:02 +01:00
commit b230d34b88
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
5 changed files with 312 additions and 74 deletions

View file

@ -9,6 +9,8 @@ snapshot_kind: text
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -17,6 +19,8 @@ snapshot_kind: text
Token {
normalised: "x",
original: "x",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -25,6 +29,8 @@ snapshot_kind: text
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -33,6 +39,8 @@ snapshot_kind: text
Token {
normalised: "c",
original: "c",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -41,6 +49,8 @@ snapshot_kind: text
Token {
normalised: "y",
original: "y",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -49,6 +59,8 @@ snapshot_kind: text
Token {
normalised: "d",
original: "d",
is_left_joinable: true,
is_right_joinable: true,
},
],
),

View file

@ -9,10 +9,14 @@ snapshot_kind: text
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),

View file

@ -9,14 +9,20 @@ snapshot_kind: text
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "c",
original: "c",
is_left_joinable: true,
is_right_joinable: true,
},
],
),

View file

@ -9,10 +9,14 @@ snapshot_kind: text
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
],
),

View file

@ -9,6 +9,8 @@ snapshot_kind: text
Token {
normalised: "a",
original: "a",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -17,10 +19,14 @@ snapshot_kind: text
Token {
normalised: "b",
original: "b",
is_left_joinable: true,
is_right_joinable: true,
},
Token {
normalised: "c",
original: "c",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -29,6 +35,8 @@ snapshot_kind: text
Token {
normalised: "x",
original: "x",
is_left_joinable: true,
is_right_joinable: true,
},
],
),
@ -37,6 +45,8 @@ snapshot_kind: text
Token {
normalised: "d",
original: "d",
is_left_joinable: true,
is_right_joinable: true,
},
],
),