1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
mod chars;
pub use chars::CharCounter;

#[cfg(feature = "tokenizers")]
mod huggingface;

#[cfg(feature = "tiktoken-rs")]
mod tiktoken;

mod words;
pub use words::WordCounter;

use crate::error::Result;

/// An interface for counting the size of a code chunk.
pub trait Sizer {
    fn size(&self, text: &str) -> Result<usize>;
}