token-dict 1.3.0

basic dictionary based tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12

pub fn main(){
	let tokenizer=token_dict::file_lines_to_dict("words.txt").unwrap();
	let tokens:Vec<u32>=tokenizer.tokenize_str("some text to tokenize").collect();
	let detokens:String=tokenizer.detokenize_str(&tokens).collect();

	print!("[");
	for id in tokens.iter().take(tokens.len().saturating_sub(1)){print!("{id}, ")}
	if let Some(id)=tokens.last(){print!("{id}")}
	println!("]");

	println!("\"{detokens}\"");
}