pagesift_api/types/
parse.rs

1//! `/v2/parse`（上传文件解析）的输入类型：上传的文件、文件类型判断、`parsers` 里的 PDF 选项。
2//! 字段和取值照 Firecrawl 文档 <https://docs.firecrawl.dev/api-reference/endpoint/parse>
3//! 与源码 `apps/api/src/controllers/v2/parse.ts`。
4
5use serde_json::Value;
6
7use crate::error::ApiError;
8
9/// 调用方上传的文件。字节只有这一份，交给引擎后不再复制。
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Upload {
12    pub bytes: Vec<u8>,
13    /// multipart 里的文件名（没有时为 `upload`），作为文档的 `metadata.sourceURL`。
14    pub filename: String,
15    /// multipart 里这个文件的 Content-Type。
16    pub content_type: Option<String>,
17    pub kind: UploadKind,
18    /// 调用方给的页面原地址（`url` 字段）：HTML 的相对链接按它解析，也作为 `metadata.sourceURL`。
19    /// 只用来解析地址，不会去抓。
20    pub url: Option<url::Url>,
21}
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum UploadKind {
25    Pdf,
26    Html,
27    /// Word、Excel、PowerPoint、EPUB、CSV（见 pagesift-office）。
28    Office(pagesift_office::Format),
29}
30
31impl UploadKind {
32    /// 判断上传的文件类型：内容以 `%PDF-` 开头就是 PDF；ZIP 容器按里面的部件认 Word、Excel、
33    /// PowerPoint、EPUB（不管文件名怎么写）；否则看扩展名，再看 Content-Type。
34    /// 和 Firecrawl 一样，认不出的类型不猜，直接拒绝。
35    pub fn detect(bytes: &[u8], filename: &str, content_type: Option<&str>) -> Option<Self> {
36        if bytes.starts_with(b"%PDF-") {
37            return Some(UploadKind::Pdf);
38        }
39        if let Some(f) = pagesift_office::Format::detect(bytes, Some(filename), content_type) {
40            return Some(UploadKind::Office(f));
41        }
42        let ext = filename
43            .rsplit_once('.')
44            .map(|(_, e)| e.to_ascii_lowercase())
45            .unwrap_or_default();
46        let mime = content_type
47            .and_then(|c| c.split(';').next())
48            .map(|c| c.trim().to_ascii_lowercase())
49            .unwrap_or_default();
50        if ext == "pdf" || matches!(mime.as_str(), "application/pdf" | "application/x-pdf") {
51            return Some(UploadKind::Pdf);
52        }
53        if matches!(ext.as_str(), "html" | "htm" | "xhtml")
54            || matches!(mime.as_str(), "text/html" | "application/xhtml+xml")
55        {
56            return Some(UploadKind::Html);
57        }
58        None
59    }
60
61    pub fn mime(self) -> &'static str {
62        match self {
63            UploadKind::Pdf => "application/pdf",
64            UploadKind::Html => "text/html",
65            UploadKind::Office(f) => f.mime(),
66        }
67    }
68}
69
70/// `parsers` 里 `{type: "pdf"}` 的选项。
71#[derive(Debug, Clone, Default, PartialEq, Eq)]
72pub struct PdfParser {
73    /// `fast`、`auto`（默认）或 `ocr`。
74    pub mode: Option<String>,
75    /// 只输出前这么多页（1–10000）。
76    pub max_pages: Option<usize>,
77    /// 另给每页的 Markdown（`data.pages`）。
78    pub pages: bool,
79    /// 给了 `parsers` 却没有 `pdf`（比如 `[]`）：和 Firecrawl 一样不解析，把原文件按 base64
80    /// 放进 `markdown`。
81    pub raw: bool,
82}
83
84impl PdfParser {
85    /// 从 `parsers` 取 PDF 选项。接受 Firecrawl 的写法：`["pdf"]`、`[{type: "pdf", ...}]`，
86    /// 以及 `"image"`（图片 OCR，这里本来就不支持图片，照收不报错，免得 Firecrawl SDK 的默认值被拒）。
87    /// 列表里没有 `pdf` 时 PDF 不解析（[`Self::raw`]）。
88    /// 做不到的选项（版面块、分页标记）明确报 UNSUPPORTED_OPTION，不悄悄忽略；`mode: "ocr"` 要
89    /// 服务端打开了 OCR，这里不知道，由解析 PDF 时检查。
90    pub fn from_parsers(parsers: Option<&Value>) -> Result<Self, ApiError> {
91        let items = match parsers {
92            None | Some(Value::Null) => return Ok(Self::default()),
93            Some(Value::Array(items)) => items,
94            Some(_) => return Err(ApiError::invalid("parsers must be an array")),
95        };
96        let mut out = Self {
97            raw: true,
98            ..Self::default()
99        };
100        for item in items {
101            let kind = match item {
102                Value::String(t) => t.as_str(),
103                Value::Object(o) => o.get("type").and_then(Value::as_str).unwrap_or_default(),
104                _ => "",
105            };
106            let obj = match (kind, item) {
107                ("image", _) => continue,
108                ("pdf", Value::String(_)) => {
109                    out.raw = false;
110                    continue;
111                }
112                ("pdf", Value::Object(o)) => {
113                    out.raw = false;
114                    o
115                }
116                _ => {
117                    return Err(ApiError::invalid(
118                        "parsers items must be \"pdf\", \"image\" or {\"type\": \"pdf\" | \"image\", ...}",
119                    ));
120                }
121            };
122            for (key, v) in obj {
123                match key.as_str() {
124                    "type" => {}
125                    "mode" => match v.as_str() {
126                        Some(m @ ("fast" | "auto" | "ocr")) => out.mode = Some(m.into()),
127                        _ => {
128                            return Err(ApiError::invalid(
129                                "parsers[].mode must be fast, auto or ocr",
130                            ));
131                        }
132                    },
133                    "maxPages" => match v.as_u64() {
134                        Some(n @ 1..=10_000) => out.max_pages = Some(n as usize),
135                        _ => {
136                            return Err(ApiError::invalid(
137                                "parsers[].maxPages must be an integer from 1 to 10000",
138                            ));
139                        }
140                    },
141                    "pages" => out.pages = flag(key, v)?,
142                    "blocks" | "pageMarkers" => {
143                        if flag(key, v)? {
144                            return Err(ApiError::unsupported(format!(
145                                "parsers[].{key} is not supported by this build"
146                            )));
147                        }
148                    }
149                    _ => {
150                        return Err(ApiError::invalid(format!(
151                            "unknown PDF parser option {key:?}"
152                        )));
153                    }
154                }
155            }
156        }
157        Ok(out)
158    }
159}
160
161fn flag(key: &str, v: &Value) -> Result<bool, ApiError> {
162    v.as_bool()
163        .ok_or_else(|| ApiError::invalid(format!("parsers[].{key} must be a boolean")))
164}
165
166#[cfg(test)]
167mod tests {
168    use super::*;
169    use crate::error::ErrorCode;
170    use serde_json::json;
171
172    #[test]
173    fn kind_prefers_content_then_name_then_mime() {
174        let d = UploadKind::detect;
175        assert_eq!(
176            d(b"%PDF-1.7", "a.html", Some("text/html")),
177            Some(UploadKind::Pdf)
178        );
179        assert_eq!(d(b"x", "Report.PDF", None), Some(UploadKind::Pdf));
180        assert_eq!(
181            d(b"x", "upload", Some("application/pdf; q=1")),
182            Some(UploadKind::Pdf)
183        );
184        assert_eq!(d(b"<p>", "a.htm", None), Some(UploadKind::Html));
185        assert_eq!(
186            d(b"<p>", "blob", Some("Text/HTML; charset=utf-8")),
187            Some(UploadKind::Html)
188        );
189        // 内容不是 ZIP 的 .docx 认不出；ZIP 按里面的部件认
190        assert_eq!(d(b"PK", "a.docx", None), None);
191        assert_eq!(
192            d(b"a,b\n1,2", "t.csv", None),
193            Some(UploadKind::Office(pagesift_office::Format::Csv))
194        );
195        assert_eq!(
196            d(b"a,b", "upload", Some("text/csv")),
197            Some(UploadKind::Office(pagesift_office::Format::Csv))
198        );
199        assert_eq!(d(b"x", "upload", Some("application/octet-stream")), None);
200    }
201
202    #[test]
203    fn pdf_parser_options() {
204        let p = |v| PdfParser::from_parsers(Some(&v));
205        assert_eq!(p(json!(["pdf"])).unwrap(), PdfParser::default());
206        assert_eq!(p(json!(["pdf", "image"])).unwrap(), PdfParser::default());
207        assert_eq!(
208            p(json!([{"type": "pdf", "mode": "fast", "maxPages": 3, "pages": true}])).unwrap(),
209            PdfParser {
210                mode: Some("fast".into()),
211                max_pages: Some(3),
212                pages: true,
213                raw: false,
214            }
215        );
216        // 列表里没有 pdf：不解析，原样返回
217        for v in [json!([]), json!(["image"]), json!([{"type": "image"}])] {
218            assert!(p(v.clone()).unwrap().raw, "{v}");
219        }
220        assert!(!PdfParser::from_parsers(None).unwrap().raw);
221        // 关着的 blocks / pageMarkers 不算请求
222        assert!(p(json!([{"type": "pdf", "blocks": false, "pageMarkers": false}])).is_ok());
223        assert_eq!(
224            p(json!([{"type": "pdf", "mode": "ocr"}]))
225                .unwrap()
226                .mode
227                .as_deref(),
228            Some("ocr")
229        );
230        for (v, code) in [
231            (
232                json!([{"type": "pdf", "blocks": true}]),
233                ErrorCode::UnsupportedOption,
234            ),
235            (
236                json!([{"type": "pdf", "maxPages": 0}]),
237                ErrorCode::InvalidRequest,
238            ),
239            (
240                json!([{"type": "pdf", "maxPages": 10001}]),
241                ErrorCode::InvalidRequest,
242            ),
243            (
244                json!([{"type": "pdf", "mode": "x"}]),
245                ErrorCode::InvalidRequest,
246            ),
247            (json!([{"type": "docx"}]), ErrorCode::InvalidRequest),
248            (json!({"type": "pdf"}), ErrorCode::InvalidRequest),
249        ] {
250            assert_eq!(p(v.clone()).unwrap_err().code, code, "{v}");
251        }
252    }
253}
pagesift_api/types/parse.rs

pagesift_api/types/
parse.rs
