chonkier 0.0.2

🦛 Chonkie, now in Rust 🦀: No-nonsense, ultra-fast, ultra-light chunking library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

/*
    The Chunk type contains a bunch of important metadata about the chunk.
    It contains the chunk text, indices based on the original text, and the
    token count, along with other important metadata.
*/

// Import the Display trait
use serde::{Deserialize, Serialize};
use std::fmt;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Chunk {
    // The text of the chunk
    pub text: String,

    // The start index of the chunk in the original text
    pub start_index: usize,

    // The end index of the chunk in the original text
    pub end_index: usize,

    // The token count of the chunk
    pub token_count: usize,
}

impl Chunk {
    // Create a new chunk (Constructor)
    pub fn new(text: String, start_index: usize, end_index: usize, token_count: usize) -> Self {
        Self {
            text,
            start_index,
            end_index,
            token_count,
        }
    }

    // Get the length of the chunk
    pub fn len(&self) -> usize {
        self.text.len()
    }

    pub fn is_empty(&self) -> bool{
        self.text.is_empty()
    }
}

impl fmt::Display for Chunk {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "Chunk(text: {}, start_index: {}, end_index: {}, token_count: {})",
            self.text, self.start_index, self.end_index, self.token_count
        )
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_chunk_new() {
        let chunk = Chunk::new("Hello, world!".to_string(), 0, 12, 2);
        assert_eq!(chunk.text, "Hello, world!");
        assert_eq!(chunk.start_index, 0);
        assert_eq!(chunk.end_index, 12);
        assert_eq!(chunk.token_count, 2);
    }

    #[test]
    fn test_chunk_clone() {
        let chunk = Chunk::new("Hello, world!".to_string(), 0, 12, 2);
        let cloned_chunk = chunk.clone();
        assert_eq!(cloned_chunk.text, "Hello, world!");
        assert_eq!(cloned_chunk.start_index, 0);
        assert_eq!(cloned_chunk.end_index, 12);
        assert_eq!(cloned_chunk.token_count, 2);
    }

    #[test]
    fn test_chunk_len() {
        let chunk = Chunk::new("Hello, world!".to_string(), 0, 12, 2);
        assert_eq!(chunk.len(), 13);
    }

    #[test]
    fn test_chunk_display() {
        let chunk = Chunk::new("Hello, world!".to_string(), 0, 12, 2);
        assert_eq!(
            format!("{}", chunk),
            "Chunk(text: Hello, world!, start_index: 0, end_index: 12, token_count: 2)"
        );
    }

    #[test]
    #[cfg(feature = "json")]
    fn test_chunk_serde() {
        let chunk = Chunk::new("Hello, world!".to_string(), 0, 12, 2);
        let serialized = serde_json::to_string(&chunk).unwrap();
        let deserialized: Chunk = serde_json::from_str(&serialized).unwrap();
        assert_eq!(deserialized.text, "Hello, world!");
        assert_eq!(deserialized.start_index, 0);
        assert_eq!(deserialized.end_index, 12);
        assert_eq!(deserialized.token_count, 2);
    }
}