pagesift_api/types/
parse.rs

1//! `/v2/parse`（上传文件解析）的输入类型：上传的文件、文件类型判断、`parsers` 里的 PDF 选项。
2//! 字段和取值照 Firecrawl 文档 <https://docs.firecrawl.dev/api-reference/endpoint/parse>
3//! 与源码 `apps/api/src/controllers/v2/parse.ts`。
4
5use serde_json::Value;
6
7use crate::error::ApiError;
8
9/// 调用方上传的文件。字节只有这一份，交给引擎后不再复制。
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Upload {
12    pub bytes: Vec<u8>,
13    /// multipart 里的文件名（没有时为 `upload`），作为文档的 `metadata.sourceURL`。
14    pub filename: String,
15    /// multipart 里这个文件的 Content-Type。
16    pub content_type: Option<String>,
17    pub kind: UploadKind,
18    /// 调用方给的页面原地址（`url` 字段）：HTML 的相对链接按它解析，也作为 `metadata.sourceURL`。
19    /// 只用来解析地址，不会去抓。
20    pub url: Option<url::Url>,
21}
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum UploadKind {
25    Pdf,
26    Html,
27    /// Word、Excel、PowerPoint、EPUB、CSV（见 pagesift-office）。
28    Office(pagesift_office::Format),
29}
30
31impl UploadKind {
32    /// 判断上传的文件类型：内容以 `%PDF-` 开头就是 PDF；ZIP 容器按里面的部件认 Word、Excel、
33    /// PowerPoint、EPUB（不管文件名怎么写）；否则看扩展名，再看 Content-Type。
34    /// 和 Firecrawl 一样，认不出的类型不猜，直接拒绝。
35    pub fn detect(bytes: &[u8], filename: &str, content_type: Option<&str>) -> Option<Self> {
36        if bytes.starts_with(b"%PDF-") {
37            return Some(UploadKind::Pdf);
38        }
39        if let Some(f) = pagesift_office::Format::detect(bytes, Some(filename), content_type) {
40            return Some(UploadKind::Office(f));
41        }
42        let ext = filename
43            .rsplit_once('.')
44            .map(|(_, e)| e.to_ascii_lowercase())
45            .unwrap_or_default();
46        let mime = content_type
47            .and_then(|c| c.split(';').next())
48            .map(|c| c.trim().to_ascii_lowercase())
49            .unwrap_or_default();
50        if ext == "pdf" || matches!(mime.as_str(), "application/pdf" | "application/x-pdf") {
51            return Some(UploadKind::Pdf);
52        }
53        if matches!(ext.as_str(), "html" | "htm" | "xhtml")
54            || matches!(mime.as_str(), "text/html" | "application/xhtml+xml")
55        {
56            return Some(UploadKind::Html);
57        }
58        None
59    }
60
61    pub fn mime(self) -> &'static str {
62        match self {
63            UploadKind::Pdf => "application/pdf",
64            UploadKind::Html => "text/html",
65            UploadKind::Office(f) => f.mime(),
66        }
67    }
68}
69
70/// `parsers` 里 `{type: "pdf"}` 的选项。
71#[derive(Debug, Clone, Default, PartialEq, Eq)]
72pub struct PdfParser {
73    /// `fast`、`auto`（默认）或 `ocr`。
74    pub mode: Option<String>,
75    /// 只输出前这么多页（1–10000）。
76    pub max_pages: Option<usize>,
77    /// 另给每页的 Markdown（`data.pages`）。
78    pub pages: bool,
79    /// 给了 `parsers` 却没有 `pdf`（比如 `[]`）：和 Firecrawl 一样不解析，把原文件按 base64
80    /// 放进 `markdown`。
81    pub raw: bool,
82}
83
84impl PdfParser {
85    /// 从 `parsers` 取 PDF 选项。接受 Firecrawl 的写法：`["pdf"]`、`[{type: "pdf", ...}]`，
86    /// 以及 `"image"`（图片 OCR，这里本来就不支持图片，照收不报错，免得 Firecrawl SDK 的默认值被拒）。
87    /// 列表里没有 `pdf` 时 PDF 不解析（[`Self::raw`]）。
88    /// 做不到的选项（OCR、版面块、分页标记）明确报 UNSUPPORTED_OPTION，不悄悄忽略。
89    pub fn from_parsers(parsers: Option<&Value>) -> Result<Self, ApiError> {
90        let items = match parsers {
91            None | Some(Value::Null) => return Ok(Self::default()),
92            Some(Value::Array(items)) => items,
93            Some(_) => return Err(ApiError::invalid("parsers must be an array")),
94        };
95        let mut out = Self {
96            raw: true,
97            ..Self::default()
98        };
99        for item in items {
100            let kind = match item {
101                Value::String(t) => t.as_str(),
102                Value::Object(o) => o.get("type").and_then(Value::as_str).unwrap_or_default(),
103                _ => "",
104            };
105            let obj = match (kind, item) {
106                ("image", _) => continue,
107                ("pdf", Value::String(_)) => {
108                    out.raw = false;
109                    continue;
110                }
111                ("pdf", Value::Object(o)) => {
112                    out.raw = false;
113                    o
114                }
115                _ => {
116                    return Err(ApiError::invalid(
117                        "parsers items must be \"pdf\", \"image\" or {\"type\": \"pdf\" | \"image\", ...}",
118                    ));
119                }
120            };
121            for (key, v) in obj {
122                match key.as_str() {
123                    "type" => {}
124                    "mode" => match v.as_str() {
125                        Some(m @ ("fast" | "auto")) => out.mode = Some(m.into()),
126                        Some("ocr") => {
127                            return Err(ApiError::unsupported(
128                                "parsers[].mode \"ocr\" is not supported: pagesift reads the PDF text layer only",
129                            ));
130                        }
131                        _ => {
132                            return Err(ApiError::invalid(
133                                "parsers[].mode must be fast, auto or ocr",
134                            ));
135                        }
136                    },
137                    "maxPages" => match v.as_u64() {
138                        Some(n @ 1..=10_000) => out.max_pages = Some(n as usize),
139                        _ => {
140                            return Err(ApiError::invalid(
141                                "parsers[].maxPages must be an integer from 1 to 10000",
142                            ));
143                        }
144                    },
145                    "pages" => out.pages = flag(key, v)?,
146                    "blocks" | "pageMarkers" => {
147                        if flag(key, v)? {
148                            return Err(ApiError::unsupported(format!(
149                                "parsers[].{key} is not supported by this build"
150                            )));
151                        }
152                    }
153                    _ => {
154                        return Err(ApiError::invalid(format!(
155                            "unknown PDF parser option {key:?}"
156                        )));
157                    }
158                }
159            }
160        }
161        Ok(out)
162    }
163}
164
165fn flag(key: &str, v: &Value) -> Result<bool, ApiError> {
166    v.as_bool()
167        .ok_or_else(|| ApiError::invalid(format!("parsers[].{key} must be a boolean")))
168}
169
170#[cfg(test)]
171mod tests {
172    use super::*;
173    use crate::error::ErrorCode;
174    use serde_json::json;
175
176    #[test]
177    fn kind_prefers_content_then_name_then_mime() {
178        let d = UploadKind::detect;
179        assert_eq!(
180            d(b"%PDF-1.7", "a.html", Some("text/html")),
181            Some(UploadKind::Pdf)
182        );
183        assert_eq!(d(b"x", "Report.PDF", None), Some(UploadKind::Pdf));
184        assert_eq!(
185            d(b"x", "upload", Some("application/pdf; q=1")),
186            Some(UploadKind::Pdf)
187        );
188        assert_eq!(d(b"<p>", "a.htm", None), Some(UploadKind::Html));
189        assert_eq!(
190            d(b"<p>", "blob", Some("Text/HTML; charset=utf-8")),
191            Some(UploadKind::Html)
192        );
193        // 内容不是 ZIP 的 .docx 认不出；ZIP 按里面的部件认
194        assert_eq!(d(b"PK", "a.docx", None), None);
195        assert_eq!(
196            d(b"a,b\n1,2", "t.csv", None),
197            Some(UploadKind::Office(pagesift_office::Format::Csv))
198        );
199        assert_eq!(
200            d(b"a,b", "upload", Some("text/csv")),
201            Some(UploadKind::Office(pagesift_office::Format::Csv))
202        );
203        assert_eq!(d(b"x", "upload", Some("application/octet-stream")), None);
204    }
205
206    #[test]
207    fn pdf_parser_options() {
208        let p = |v| PdfParser::from_parsers(Some(&v));
209        assert_eq!(p(json!(["pdf"])).unwrap(), PdfParser::default());
210        assert_eq!(p(json!(["pdf", "image"])).unwrap(), PdfParser::default());
211        assert_eq!(
212            p(json!([{"type": "pdf", "mode": "fast", "maxPages": 3, "pages": true}])).unwrap(),
213            PdfParser {
214                mode: Some("fast".into()),
215                max_pages: Some(3),
216                pages: true,
217                raw: false,
218            }
219        );
220        // 列表里没有 pdf：不解析，原样返回
221        for v in [json!([]), json!(["image"]), json!([{"type": "image"}])] {
222            assert!(p(v.clone()).unwrap().raw, "{v}");
223        }
224        assert!(!PdfParser::from_parsers(None).unwrap().raw);
225        // 关着的 blocks / pageMarkers 不算请求
226        assert!(p(json!([{"type": "pdf", "blocks": false, "pageMarkers": false}])).is_ok());
227        for (v, code) in [
228            (
229                json!([{"type": "pdf", "mode": "ocr"}]),
230                ErrorCode::UnsupportedOption,
231            ),
232            (
233                json!([{"type": "pdf", "blocks": true}]),
234                ErrorCode::UnsupportedOption,
235            ),
236            (
237                json!([{"type": "pdf", "maxPages": 0}]),
238                ErrorCode::InvalidRequest,
239            ),
240            (
241                json!([{"type": "pdf", "maxPages": 10001}]),
242                ErrorCode::InvalidRequest,
243            ),
244            (
245                json!([{"type": "pdf", "mode": "x"}]),
246                ErrorCode::InvalidRequest,
247            ),
248            (json!([{"type": "docx"}]), ErrorCode::InvalidRequest),
249            (json!({"type": "pdf"}), ErrorCode::InvalidRequest),
250        ] {
251            assert_eq!(p(v.clone()).unwrap_err().code, code, "{v}");
252        }
253    }
254}
pagesift_api/types/parse.rs

pagesift_api/types/
parse.rs
