code_splitter/sizer.rs
1mod chars;
2pub use chars::CharCounter;
3
4#[cfg(feature = "tokenizers")]
5mod huggingface;
6
7#[cfg(feature = "tiktoken-rs")]
8mod tiktoken;
9
10mod words;
11pub use words::WordCounter;
12
13use crate::error::Result;
14
15/// An interface for counting the size of a code chunk.
16pub trait Sizer {
17 fn size(&self, text: &str) -> Result<usize>;
18}