rs-chunks 0.1.0

Fast, high-fidelity document chunking for RAG — a pure-Rust engine covering 36 file formats (Office, OpenDocument, PDF, email, ebooks, notebooks, and more).
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125


use super::text_extractor::{DocParagraph, ParagraphType};

fn render_table_row(content: &str) -> String {
    let cells: Vec<String> = content.split('|').map(|c| c.trim().to_string()).collect();

    if cells.is_empty() {
        return String::new();
    }

    let row = format!("| {} |", cells.join(" | "));
    let sep = format!("| {} |", vec!["---"; cells.len()].join(" | "));
    format!("{row}\n{sep}")
}

/// Render a paragraph list to Markdown. Shared by `.doc` and `.ppt`, both of
/// which produce `Vec<DocParagraph>`.
pub(crate) fn render_paragraphs_markdown(paragraphs: Vec<DocParagraph>) -> String {
    let mut rendered: Vec<String> = Vec::new();

    for p in paragraphs {
        match p.paragraph_type {
            ParagraphType::PageBreak => rendered.push("\n\n---\n\n".to_string()),
            ParagraphType::Heading(level) => {
                let content = p.content.trim();
                if content.is_empty() {
                    continue;
                }
                let line = match level {
                    1 => format!("# {content}"),
                    2 | 3 => format!("## {content}"),
                    _ => format!("### {content}"),
                };
                rendered.push(line);
            }
            ParagraphType::ListItem => {
                let content = p.content.trim();
                if content.is_empty() {
                    continue;
                }
                rendered.push(format!("- {content}"));
            }
            ParagraphType::Table => {
                let content = p.content.trim();
                if content.is_empty() {
                    continue;
                }
                let table = render_table_row(content);
                if !table.trim().is_empty() {
                    rendered.push(table);
                }
            }
            ParagraphType::Normal => {
                let content = p.content.trim();
                if content.is_empty() {
                    continue;
                }
                rendered.push(content.to_string());
            }
        }
    }

    let mut out = rendered.join("\n\n").trim().to_string();

    while out.ends_with("\n\n---") {
        out.truncate(out.len() - "\n\n---".len());
        out = out.trim_end().to_string();
    }

    out.trim().to_string()
}


/// `.doc` → Markdown with image markers interleaved by raw paragraph ordinal,
/// plus extracted image bytes.
pub(super) fn to_markdown_with_images(file_path: &str) -> Result<(String, Vec<(String, Vec<u8>)>), String> {
    super::structural::validate_doc_path(file_path)?;
    let bytes = std::fs::read(file_path).map_err(|e| format!("Failed to read .doc file: {e}"))?;
    to_markdown_with_images_bytes(&bytes)
}

pub(super) fn to_markdown_with_images_bytes(bytes: &[u8]) -> Result<(String, Vec<(String, Vec<u8>)>), String> {
    let indexed = super::structural::load_doc_paragraphs_indexed_bytes(bytes)?;
    let images = super::images::extract_doc_images_bytes(bytes).unwrap_or_default();

    let mut anchored: Vec<(usize, &str)> = images
        .iter()
        .filter_map(|img| img.raw_para.map(|rp| (rp, img.hash_name.as_str())))
        .collect();
    anchored.sort_by_key(|(rp, _)| *rp);
    let mut anchored_iter = anchored.into_iter().peekable();

    let image_paragraph = |hash_name: &str| DocParagraph {
        content: format!("![]({hash_name})"),
        paragraph_type: ParagraphType::Normal,
        heading_level: None,
    };

    let mut paragraphs: Vec<DocParagraph> = Vec::with_capacity(indexed.len() + images.len());
    for (raw_idx, para) in indexed {
        while let Some((rp, _)) = anchored_iter.peek() {
            if *rp < raw_idx {
                let (_, hash) = anchored_iter.next().unwrap();
                paragraphs.push(image_paragraph(hash));
            } else {
                break;
            }
        }
        paragraphs.push(para);
    }
    for (_, hash) in anchored_iter {
        paragraphs.push(image_paragraph(hash));
    }
    for img in images.iter().filter(|i| i.raw_para.is_none()) {
        paragraphs.push(image_paragraph(&img.hash_name));
    }

    let mut image_out: Vec<(String, Vec<u8>)> = Vec::new();
    for img in &images {
        if !image_out.iter().any(|(n, _)| n == &img.hash_name) {
            image_out.push((img.hash_name.clone(), img.bytes.clone()));
        }
    }
    Ok((render_paragraphs_markdown(paragraphs), image_out))
}