langextract_rust/
tokenizer.rs

1//! Text tokenization functionality.
2
3/// Placeholder for tokenization functionality
4pub struct Tokenizer;
5
6impl Tokenizer {
7    /// Create a new tokenizer
8    pub fn new() -> Self {
9        Self
10    }
11}
12
13impl Default for Tokenizer {
14    fn default() -> Self {
15        Self::new()
16    }
17}