pharia_skill/csi/
chunking.rs

1use serde::Serialize;
2
3/// Chunking parameters
4#[derive(Clone, Debug, Serialize)]
5pub struct ChunkParams {
6    /// The name of the model the chunk is intended to be used for.
7    /// This must be a known model.
8    pub model: String,
9    /// The maximum number of tokens that should be returned per chunk.
10    pub max_tokens: u32,
11    /// The amount of allowed overlap between chunks.
12    /// overlap must be less than max-tokens.
13    pub overlap: u32,
14}
15
16impl ChunkParams {
17    pub fn new(model: impl Into<String>, max_tokens: u32) -> Self {
18        Self {
19            model: model.into(),
20            max_tokens,
21            overlap: 0,
22        }
23    }
24
25    #[must_use]
26    pub fn with_overlap(mut self, overlap: u32) -> Self {
27        self.overlap = overlap;
28        self
29    }
30}
31
32#[derive(Clone, Debug, Serialize)]
33pub struct ChunkRequest {
34    pub text: String,
35    pub params: ChunkParams,
36}
37
38impl ChunkRequest {
39    pub fn new(text: impl Into<String>, params: ChunkParams) -> Self {
40        Self {
41            text: text.into(),
42            params,
43        }
44    }
45}
pharia_skill/csi/chunking.rs

pharia_skill/csi/
chunking.rs
