pagesift_api/types/
parse.rs

1//! `/v2/parse`（上传文件解析）的输入类型：上传的文件、文件类型判断、`parsers` 里的 PDF 选项。
2//! 字段和取值照 Firecrawl 文档 <https://docs.firecrawl.dev/api-reference/endpoint/parse>
3//! 与源码 `apps/api/src/controllers/v2/parse.ts`。
4
5use serde_json::Value;
6
7use crate::error::ApiError;
8
9/// 调用方上传的文件。字节只有这一份，交给引擎后不再复制。
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Upload {
12    pub bytes: Vec<u8>,
13    /// multipart 里的文件名（没有时为 `upload`），作为文档的 `metadata.sourceURL`。
14    pub filename: String,
15    /// multipart 里这个文件的 Content-Type。
16    pub content_type: Option<String>,
17    pub kind: UploadKind,
18}
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum UploadKind {
22    Pdf,
23    Html,
24    /// Word、Excel、PowerPoint、EPUB、CSV（见 pagesift-office）。
25    Office(pagesift_office::Format),
26}
27
28impl UploadKind {
29    /// 判断上传的文件类型：内容以 `%PDF-` 开头就是 PDF；ZIP 容器按里面的部件认 Word、Excel、
30    /// PowerPoint、EPUB（不管文件名怎么写）；否则看扩展名，再看 Content-Type。
31    /// 和 Firecrawl 一样，认不出的类型不猜，直接拒绝。
32    pub fn detect(bytes: &[u8], filename: &str, content_type: Option<&str>) -> Option<Self> {
33        if bytes.starts_with(b"%PDF-") {
34            return Some(UploadKind::Pdf);
35        }
36        if let Some(f) = pagesift_office::Format::detect(bytes, Some(filename), content_type) {
37            return Some(UploadKind::Office(f));
38        }
39        let ext = filename
40            .rsplit_once('.')
41            .map(|(_, e)| e.to_ascii_lowercase())
42            .unwrap_or_default();
43        let mime = content_type
44            .and_then(|c| c.split(';').next())
45            .map(|c| c.trim().to_ascii_lowercase())
46            .unwrap_or_default();
47        if ext == "pdf" || matches!(mime.as_str(), "application/pdf" | "application/x-pdf") {
48            return Some(UploadKind::Pdf);
49        }
50        if matches!(ext.as_str(), "html" | "htm" | "xhtml")
51            || matches!(mime.as_str(), "text/html" | "application/xhtml+xml")
52        {
53            return Some(UploadKind::Html);
54        }
55        None
56    }
57
58    pub fn mime(self) -> &'static str {
59        match self {
60            UploadKind::Pdf => "application/pdf",
61            UploadKind::Html => "text/html",
62            UploadKind::Office(f) => f.mime(),
63        }
64    }
65}
66
67/// `parsers` 里 `{type: "pdf"}` 的选项。
68#[derive(Debug, Clone, Default, PartialEq, Eq)]
69pub struct PdfParser {
70    /// `fast`、`auto`（默认）或 `ocr`。
71    pub mode: Option<String>,
72    /// 只输出前这么多页（1–10000）。
73    pub max_pages: Option<usize>,
74    /// 另给每页的 Markdown（`data.pages`）。
75    pub pages: bool,
76    /// 给了 `parsers` 却没有 `pdf`（比如 `[]`）：和 Firecrawl 一样不解析，把原文件按 base64
77    /// 放进 `markdown`。
78    pub raw: bool,
79}
80
81impl PdfParser {
82    /// 从 `parsers` 取 PDF 选项。接受 Firecrawl 的写法：`["pdf"]`、`[{type: "pdf", ...}]`，
83    /// 以及 `"image"`（图片 OCR，这里本来就不支持图片，照收不报错，免得 Firecrawl SDK 的默认值被拒）。
84    /// 列表里没有 `pdf` 时 PDF 不解析（[`Self::raw`]）。
85    /// 做不到的选项（OCR、版面块、分页标记）明确报 UNSUPPORTED_OPTION，不悄悄忽略。
86    pub fn from_parsers(parsers: Option<&Value>) -> Result<Self, ApiError> {
87        let items = match parsers {
88            None | Some(Value::Null) => return Ok(Self::default()),
89            Some(Value::Array(items)) => items,
90            Some(_) => return Err(ApiError::invalid("parsers must be an array")),
91        };
92        let mut out = Self {
93            raw: true,
94            ..Self::default()
95        };
96        for item in items {
97            let kind = match item {
98                Value::String(t) => t.as_str(),
99                Value::Object(o) => o.get("type").and_then(Value::as_str).unwrap_or_default(),
100                _ => "",
101            };
102            let obj = match (kind, item) {
103                ("image", _) => continue,
104                ("pdf", Value::String(_)) => {
105                    out.raw = false;
106                    continue;
107                }
108                ("pdf", Value::Object(o)) => {
109                    out.raw = false;
110                    o
111                }
112                _ => {
113                    return Err(ApiError::invalid(
114                        "parsers items must be \"pdf\", \"image\" or {\"type\": \"pdf\" | \"image\", ...}",
115                    ));
116                }
117            };
118            for (key, v) in obj {
119                match key.as_str() {
120                    "type" => {}
121                    "mode" => match v.as_str() {
122                        Some(m @ ("fast" | "auto")) => out.mode = Some(m.into()),
123                        Some("ocr") => {
124                            return Err(ApiError::unsupported(
125                                "parsers[].mode \"ocr\" is not supported: pagesift reads the PDF text layer only",
126                            ));
127                        }
128                        _ => {
129                            return Err(ApiError::invalid(
130                                "parsers[].mode must be fast, auto or ocr",
131                            ));
132                        }
133                    },
134                    "maxPages" => match v.as_u64() {
135                        Some(n @ 1..=10_000) => out.max_pages = Some(n as usize),
136                        _ => {
137                            return Err(ApiError::invalid(
138                                "parsers[].maxPages must be an integer from 1 to 10000",
139                            ));
140                        }
141                    },
142                    "pages" => out.pages = flag(key, v)?,
143                    "blocks" | "pageMarkers" => {
144                        if flag(key, v)? {
145                            return Err(ApiError::unsupported(format!(
146                                "parsers[].{key} is not supported by this build"
147                            )));
148                        }
149                    }
150                    _ => {
151                        return Err(ApiError::invalid(format!(
152                            "unknown PDF parser option {key:?}"
153                        )));
154                    }
155                }
156            }
157        }
158        Ok(out)
159    }
160}
161
162fn flag(key: &str, v: &Value) -> Result<bool, ApiError> {
163    v.as_bool()
164        .ok_or_else(|| ApiError::invalid(format!("parsers[].{key} must be a boolean")))
165}
166
167#[cfg(test)]
168mod tests {
169    use super::*;
170    use crate::error::ErrorCode;
171    use serde_json::json;
172
173    #[test]
174    fn kind_prefers_content_then_name_then_mime() {
175        let d = UploadKind::detect;
176        assert_eq!(
177            d(b"%PDF-1.7", "a.html", Some("text/html")),
178            Some(UploadKind::Pdf)
179        );
180        assert_eq!(d(b"x", "Report.PDF", None), Some(UploadKind::Pdf));
181        assert_eq!(
182            d(b"x", "upload", Some("application/pdf; q=1")),
183            Some(UploadKind::Pdf)
184        );
185        assert_eq!(d(b"<p>", "a.htm", None), Some(UploadKind::Html));
186        assert_eq!(
187            d(b"<p>", "blob", Some("Text/HTML; charset=utf-8")),
188            Some(UploadKind::Html)
189        );
190        // 内容不是 ZIP 的 .docx 认不出；ZIP 按里面的部件认
191        assert_eq!(d(b"PK", "a.docx", None), None);
192        assert_eq!(
193            d(b"a,b\n1,2", "t.csv", None),
194            Some(UploadKind::Office(pagesift_office::Format::Csv))
195        );
196        assert_eq!(
197            d(b"a,b", "upload", Some("text/csv")),
198            Some(UploadKind::Office(pagesift_office::Format::Csv))
199        );
200        assert_eq!(d(b"x", "upload", Some("application/octet-stream")), None);
201    }
202
203    #[test]
204    fn pdf_parser_options() {
205        let p = |v| PdfParser::from_parsers(Some(&v));
206        assert_eq!(p(json!(["pdf"])).unwrap(), PdfParser::default());
207        assert_eq!(p(json!(["pdf", "image"])).unwrap(), PdfParser::default());
208        assert_eq!(
209            p(json!([{"type": "pdf", "mode": "fast", "maxPages": 3, "pages": true}])).unwrap(),
210            PdfParser {
211                mode: Some("fast".into()),
212                max_pages: Some(3),
213                pages: true,
214                raw: false,
215            }
216        );
217        // 列表里没有 pdf：不解析，原样返回
218        for v in [json!([]), json!(["image"]), json!([{"type": "image"}])] {
219            assert!(p(v.clone()).unwrap().raw, "{v}");
220        }
221        assert!(!PdfParser::from_parsers(None).unwrap().raw);
222        // 关着的 blocks / pageMarkers 不算请求
223        assert!(p(json!([{"type": "pdf", "blocks": false, "pageMarkers": false}])).is_ok());
224        for (v, code) in [
225            (
226                json!([{"type": "pdf", "mode": "ocr"}]),
227                ErrorCode::UnsupportedOption,
228            ),
229            (
230                json!([{"type": "pdf", "blocks": true}]),
231                ErrorCode::UnsupportedOption,
232            ),
233            (
234                json!([{"type": "pdf", "maxPages": 0}]),
235                ErrorCode::InvalidRequest,
236            ),
237            (
238                json!([{"type": "pdf", "maxPages": 10001}]),
239                ErrorCode::InvalidRequest,
240            ),
241            (
242                json!([{"type": "pdf", "mode": "x"}]),
243                ErrorCode::InvalidRequest,
244            ),
245            (json!([{"type": "docx"}]), ErrorCode::InvalidRequest),
246            (json!({"type": "pdf"}), ErrorCode::InvalidRequest),
247        ] {
248            assert_eq!(p(v.clone()).unwrap_err().code, code, "{v}");
249        }
250    }
251}
pagesift_api/types/parse.rs

pagesift_api/types/
parse.rs
