code-splitter 0.1.5

Split code into semantic chunks using tree-sitter
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

use tokenizers::Tokenizer;

use crate::error::Result;
use crate::sizer::Sizer;

impl Sizer for Tokenizer {
    /// Count the number of tokens in the given text.
    fn size(&self, text: &str) -> Result<usize> {
        let encoding = self.encode(text, false)?;
        Ok(encoding.get_ids().len())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_tokenizer_size() {
        let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None).unwrap();
        let text = "Hello, y'all! How are you 😁 ?";
        let size = tokenizer.size(text).unwrap();
        // ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"]
        assert_eq!(size, 11);
    }
}