Skip to main content

byob_docs/
lib.rs

1//! Text out of documents, and PDF pages as pictures, in pure Rust.
2//!
3//! - **PDF** ([`pdf`]): text per page (hayro's interpreter, a text-collecting device), page
4//!   count, and a page rendered to PNG (hayro's rasteriser) for a vision model.
5//! - **Word and PowerPoint** ([`ooxml`]): `.docx` and `.pptx` are zip files of XML; their text
6//!   comes out as rough Markdown (headings, lists, tables), slides one by one.
7//!
8//! Every input is hostile: sizes are capped (zip entries are read with a limit, so a zip bomb
9//! stops early), parser panics are caught, and errors are short sentences a model can act on.
10#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
11
12#[doc(hidden)]
13pub mod fixtures;
14pub mod ooxml;
15pub mod pdf;
16
17use std::panic::{AssertUnwindSafe, catch_unwind};
18
19/// Largest document file read (bytes).
20pub const MAX_FILE_BYTES: u64 = 64 * 1024 * 1024;
21
22/// What a file is, by its name and first bytes.
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum Kind {
25    Pdf,
26    Docx,
27    Pptx,
28    /// .xlsx, .xls, .ods: the spreadsheet tools read these.
29    Spreadsheet,
30    /// .doc, .ppt, .odt, .odp, .rtf: known but not read here.
31    OtherDocument,
32}
33
34impl Kind {
35    /// The kind of `name` (its extension), checked against the first bytes where they tell.
36    pub fn of(name: &str, head: &[u8]) -> Option<Self> {
37        let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase()).unwrap_or_default();
38        let zip = head.starts_with(b"PK\x03\x04");
39        match ext.as_str() {
40            "pdf" if find(head, b"%PDF-").is_some() => Some(Self::Pdf),
41            "docx" | "docm" if zip => Some(Self::Docx),
42            "pptx" | "pptm" if zip => Some(Self::Pptx),
43            "xlsx" | "xlsm" | "xls" | "ods" => Some(Self::Spreadsheet),
44            "doc" | "ppt" | "odt" | "odp" | "rtf" => Some(Self::OtherDocument),
45            _ if head.starts_with(b"%PDF-") => Some(Self::Pdf),
46            _ => None,
47        }
48    }
49}
50
51/// Where `needle` first occurs in `hay`.
52fn find(hay: &[u8], needle: &[u8]) -> Option<usize> {
53    hay.windows(needle.len()).position(|w| w == needle)
54}
55
56/// Run parser code that might panic on a malformed file.
57fn guarded<T>(what: &str, f: impl FnOnce() -> Result<T, String>) -> Result<T, String> {
58    catch_unwind(AssertUnwindSafe(f)).unwrap_or_else(|_| Err(format!("{what} is damaged or uses something byob can't read")))
59}
60
61/// Parse a page list like `"3"`, `"1-3,5"`, `"4-"` or `"all"` against `total` pages.
62/// Pages count from 1; the result is sorted, without duplicates, at most `max` long.
63pub fn parse_pages(spec: &str, total: usize, max: usize) -> Result<Vec<usize>, String> {
64    let bad = || format!("pages must look like \"3\", \"1-3,5\" or \"4-\" (the document has {total})");
65    let spec = spec.trim();
66    if spec.is_empty() || spec.eq_ignore_ascii_case("all") {
67        return Ok((1..=total.min(max)).collect());
68    }
69    let mut out = Vec::new();
70    for part in spec.split([',', ' ', ';']).map(str::trim).filter(|p| !p.is_empty()) {
71        // en and em dashes too: models copy them from "pages 1–3 of 9".
72        let part = part.replace(['–', '—'], "-");
73        let (a, b) = match part.split_once('-') {
74            Some((a, b)) => {
75                let a = if a.trim().is_empty() { 1 } else { a.trim().parse::<usize>().map_err(|_| bad())? };
76                let b = if b.trim().is_empty() { total } else { b.trim().parse::<usize>().map_err(|_| bad())? };
77                (a, b)
78            }
79            None => {
80                let n = part.parse::<usize>().map_err(|_| bad())?;
81                (n, n)
82            }
83        };
84        if a == 0 || a > b {
85            return Err(bad());
86        }
87        if a > total {
88            return Err(format!("page {a} is past the end: the document has {total}"));
89        }
90        for n in a..=b.min(total) {
91            out.push(n);
92            if out.len() > max.saturating_mul(4) {
93                break;
94            }
95        }
96    }
97    out.sort_unstable();
98    out.dedup();
99    out.truncate(max);
100    if out.is_empty() {
101        return Err(bad());
102    }
103    Ok(out)
104}
105
106/// Pages as compact ranges: `[1,2,3,5]` → `"1-3,5"`.
107pub fn page_ranges(pages: &[usize]) -> String {
108    let mut out: Vec<String> = Vec::new();
109    let mut i = 0;
110    while let Some(&start) = pages.get(i) {
111        let mut end = start;
112        while pages.get(i.saturating_add(1)) == Some(&end.saturating_add(1)) {
113            end = end.saturating_add(1);
114            i = i.saturating_add(1);
115        }
116        out.push(if start == end { start.to_string() } else { format!("{start}-{end}") });
117        i = i.saturating_add(1);
118    }
119    out.join(",")
120}
121
122#[cfg(test)]
123mod tests {
124    use super::*;
125
126    #[test]
127    fn kinds() {
128        assert_eq!(Kind::of("a/report.PDF", b"%PDF-1.7\n"), Some(Kind::Pdf));
129        assert_eq!(Kind::of("noext", b"%PDF-1.4"), Some(Kind::Pdf));
130        assert_eq!(Kind::of("fake.pdf", b"hello"), None);
131        assert_eq!(Kind::of("x.docx", b"PK\x03\x04rest"), Some(Kind::Docx));
132        assert_eq!(Kind::of("x.docx", b"not a zip"), None);
133        assert_eq!(Kind::of("deck.pptx", b"PK\x03\x04"), Some(Kind::Pptx));
134        assert_eq!(Kind::of("b.xlsx", b"PK\x03\x04"), Some(Kind::Spreadsheet));
135        assert_eq!(Kind::of("old.doc", b"\xd0\xcf"), Some(Kind::OtherDocument));
136        assert_eq!(Kind::of("notes.txt", b"hi"), None);
137    }
138
139    #[test]
140    fn page_lists() {
141        assert_eq!(parse_pages("", 3, 50).unwrap(), [1, 2, 3]);
142        assert_eq!(parse_pages("all", 100, 5).unwrap(), [1, 2, 3, 4, 5]);
143        assert_eq!(parse_pages("2", 3, 50).unwrap(), [2]);
144        assert_eq!(parse_pages("5,1-3, 2", 9, 50).unwrap(), [1, 2, 3, 5]);
145        assert_eq!(parse_pages("7-", 9, 50).unwrap(), [7, 8, 9]);
146        assert_eq!(parse_pages("2–4", 9, 50).unwrap(), [2, 3, 4]);
147        assert_eq!(parse_pages("8-20", 9, 50).unwrap(), [8, 9]);
148        assert_eq!(parse_pages("1-1000000000000", 3, 50).unwrap(), [1, 2, 3]);
149        for bad in ["0", "x", "3-1", "10", "1-2-3"] {
150            assert!(parse_pages(bad, 9, 50).is_err(), "{bad}");
151        }
152    }
153
154    #[test]
155    fn ranges() {
156        assert_eq!(page_ranges(&[1, 2, 3, 5, 7, 8]), "1-3,5,7-8");
157        assert_eq!(page_ranges(&[4]), "4");
158        assert_eq!(page_ranges(&[]), "");
159    }
160}