ragit 0.4.5

git-like rag pipeline
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use super::{Chunk, MultiModalContent, into_multi_modal_contents};
use crate::error::Error;
use crate::index::Index;
use crate::uid::Uid;
use ragit_pdl::{encode_base64, escape_pdl_tokens};
use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct RenderedChunk {
    // You can feed `pdl_data` directly to a pdl context.
    // Make sure to use a safe filter, like `{{chunk.pdl_data|safe}}`.
    pub pdl_data: String,

    // Human-readable data. It's used in `rag cat-file`, ragit-server's
    // file-content api, agent's file reader, and many more places.
    pub human_data: String,
    pub raw_data: Vec<MultiModalContent>,
    pub source: String,
}

impl RenderedChunk {
    pub fn fake(data: String, source: String) -> Self {
        RenderedChunk {
            pdl_data: data.to_string(),
            human_data: data.to_string(),
            raw_data: vec![MultiModalContent::Text { content: data }],
            source,
        }
    }
}

impl Chunk {
    pub fn render(self, index: &Index) -> Result<RenderedChunk, Error> {
        let human_data = self.data.clone();
        let raw_data = into_multi_modal_contents(&self.data, &self.images);
        let mut pdl_data = vec![];

        for c in raw_data.iter() {
            match c {
                MultiModalContent::Text { content } => {
                    pdl_data.push(escape_pdl_tokens(content));
                },
                MultiModalContent::Image { uid } => {
                    pdl_data.push(format!("<|raw_media(png:{})|>", encode_base64(&index.get_image_bytes_by_uid(uid.parse::<Uid>()?)?)));
                },
            }
        }

        Ok(RenderedChunk {
            human_data,
            pdl_data: pdl_data.join(""),
            raw_data,
            source: self.render_source(),
        })
    }
}