code-splitter 0.1.5

Split code into semantic chunks using tree-sitter
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

use crate::error::Result;
use crate::sizer::Sizer;

/// A marker struct for counting characters in code chunks.
///
/// ```
/// use code_splitter::{CharCounter, Splitter};
///
/// let lang = tree_sitter_md::language();
/// let splitter = Splitter::new(lang, CharCounter).unwrap();
///
/// let code = b"hello, world!";
/// let chunks = splitter.split(code).unwrap();
/// ```
pub struct CharCounter;

impl Sizer for CharCounter {
    /// Count the number of characters in the given text.
    fn size(&self, text: &str) -> Result<usize> {
        Ok(text.chars().count())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_size() {
        let counter = CharCounter;
        let text = "café";
        let size = counter.size(text).unwrap();
        assert_eq!(size, 4);
    }
}