code_splitter/sizer/
words.rs

1use crate::error::Result;
2use crate::sizer::Sizer;
3
4/// A marker struct for counting words in code chunks.
5///
6/// ```
7/// use code_splitter::{Splitter, WordCounter};
8///
9/// let lang = tree_sitter_md::language();
10/// let splitter = Splitter::new(lang, WordCounter).unwrap();
11///
12/// let code = b"hello, world!";
13/// let chunks = splitter.split(code).unwrap();
14/// ```
15pub struct WordCounter;
16
17impl Sizer for WordCounter {
18    /// Count the number of words in the given text.
19    fn size(&self, text: &str) -> Result<usize> {
20        Ok(text.split_whitespace().count())
21    }
22}
23
24#[cfg(test)]
25mod tests {
26    use super::*;
27
28    #[test]
29    fn test_size() {
30        let counter = WordCounter;
31        let text = "hello, world!";
32        let size = counter.size(text).unwrap();
33        assert_eq!(size, 2);
34    }
35}
code_splitter/sizer/words.rs

code_splitter/sizer/
words.rs
