token/token.rs
1pub fn main(){
2 let tokenizer=token_dict::file_lines_to_dict("words.txt").unwrap();
3 let tokens:Vec<u32>=tokenizer.tokenize_str("some text to tokenize").collect();
4 let detokens:String=tokenizer.detokenize_str(&tokens).collect();
5
6 print!("[");
7 for id in tokens.iter().take(tokens.len().saturating_sub(1)){print!("{id}, ")}
8 if let Some(id)=tokens.last(){print!("{id}")}
9 println!("]");
10
11 println!("\"{detokens}\"");
12}