andras/reconcile
andras/reconcile
Watch
1
0
Fork
You've already forked reconcile
 0
Code Issues Pull requests 10 Projects Releases Packages Wiki Activity Actions

Add character tokenizer

Browse source
This commit is contained in:
Andras Schmelczer 2025-06-29 19:35:21 +01:00
commit 9cb73680f8
1 changed files with 177 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/tokenizer.rs
View file

@ -1,4 +1,5 @@
mod word_tokenizer;
mod character_tokenizer;
use std::ops::Deref;
@ -35,7 +36,7 @@ impl Deref for BuiltinTokenizer {
fn deref(&self) -> &Self::Target {
match self {
BuiltinTokenizer::Character => todo!(),
BuiltinTokenizer::Character =>&character_tokenizer::character_tokenizer,
BuiltinTokenizer::Word => &word_tokenizer::word_tokenizer,
#[cfg(feature = "wasm")]
BuiltinTokenizer::__Invalid => panic!("Unexpected tokenizer type"),