pagesift_engine/engine/
parse.rs

1//! `/v2/parse`：解析调用方上传的文件。PDF 走和抓取相同的 pagesift-pdf，HTML 走相同的正文提取，
2//! 组装成和 scrape 相同的文档。不联网，所以没有缓存、robots、浏览器渲染这些环节；
3//! CPU 照样经过解析阶段的闸门和 [`cpu_task`] 的核数限制。
4
5use pagesift_api::types::{Upload, UploadKind};
6
7use super::*;
8
9/// [协议] 选项里没给 `timeout` 时整个请求的截止时间，和 Firecrawl `/parse` 的默认值一样（接口兼容）。
10/// 只是上限：PDF 的解析时限按文件大小估计，见 `pdf.rs`。
11const DEFAULT_PARSE_TIMEOUT: Duration = Duration::from_secs(30);
12
13impl Scraper {
14    /// 解析上传的文件并写审计记录（`kind: parse`）。不存快照：上传的文件往往不公开。
15    pub async fn parse(&self, upload: Upload, opts: &ScrapeOptions) -> Result<Document, ApiError> {
16        let Some(audit) = &self.audit else {
17            return self.scrape_inner(Source::Upload(upload), opts).await;
18        };
19        let name = format!("upload:{}", upload.filename);
20        let size = upload.bytes.len();
21        let started = Instant::now();
22        let result = self.scrape_inner(Source::Upload(upload), opts).await;
23        audit.record_parse(&name, size, opts, &result, started.elapsed());
24        result
25    }
26
27    /// 上传文件的基础文档（正文、元数据），相当于抓取时的 `scrape_cached`。
28    pub(super) async fn parse_upload(
29        &self,
30        upload: Upload,
31        opts: &ScrapeOptions,
32    ) -> Result<Document, ApiError> {
33        let started = Instant::now();
34        let deadline = started
35            + opts
36                .timeout
37                .map_or(DEFAULT_PARSE_TIMEOUT, Duration::from_millis);
38        let content_type = match upload.kind {
39            UploadKind::Html => upload
40                .content_type
41                .clone()
42                .filter(|c| c.to_ascii_lowercase().contains("html")),
43            UploadKind::Pdf | UploadKind::Office(_) | UploadKind::Image(_) => None,
44        };
45        // 调用方给了页面的原地址时，它就是来源；否则用文件名
46        let source = upload
47            .url
48            .as_ref()
49            .map_or_else(|| upload.filename.clone(), Url::to_string);
50        let meta = Metadata {
51            source_url: Some(source),
52            status_code: 200,
53            content_type: Some(
54                content_type
55                    .clone()
56                    .unwrap_or_else(|| upload.kind.mime().into()),
57            ),
58            proxy_used: "basic".into(),
59            ..Default::default()
60        };
61        let ext = PagesiftMeta {
62            engine: Some("upload".into()),
63            ..Default::default()
64        };
65        let progress = Progress {
66            started,
67            timings: BTreeMap::new(),
68            warnings: Vec::new(),
69        };
70        match upload.kind {
71            UploadKind::Pdf => {
72                self.pdf_document(upload.bytes, opts, (meta, ext), progress, Some(deadline))
73                    .await
74            }
75            UploadKind::Office(format) => {
76                let input = office::OfficeInput {
77                    bytes: upload.bytes,
78                    format,
79                    content_type: upload.content_type,
80                };
81                self.office_document(input, opts, (meta, ext), progress, Some(deadline))
82                    .await
83            }
84            UploadKind::Image(_) => {
85                self.image_document(upload.bytes, opts, (meta, ext), progress, Some(deadline))
86                    .await
87            }
88            UploadKind::Html => {
89                let html = pagesift_net::decode_owned(upload.bytes, content_type.as_deref(), None);
90                let base = upload.url.unwrap_or_else(|| upload_base(&upload.filename));
91                self.parse_html(html, &base, opts, (meta, ext), progress, deadline)
92                    .await
93            }
94        }
95    }
96
97    async fn parse_html(
98        &self,
99        decoded: pagesift_net::Decoded,
100        base: &Url,
101        opts: &ScrapeOptions,
102        (mut meta, mut ext): (Metadata, PagesiftMeta),
103        mut p: Progress,
104        deadline: Instant,
105    ) -> Result<Document, ApiError> {
106        if let Some(warning) = decoded.warning() {
107            p.warnings.push(warning);
108        }
109        let stage = self
110            .policy
111            .fetcher()
112            .acquire_stage(pagesift_net::NetworkStage::Parse)
113            .await;
114        let t = Instant::now();
115        let work = process_html(decoded.text, base, Some(200), opts);
116        let (doc, html) = match tokio::time::timeout_at(deadline.into(), work).await {
117            Ok(Ok(result)) => {
118                stage.finish(pagesift_net::StageOutcome::Success);
119                result
120            }
121            Ok(Err(error)) => {
122                stage.finish(pagesift_net::StageOutcome::DeterministicFailure);
123                return Err(error);
124            }
125            Err(_) => {
126                stage.finish(pagesift_net::StageOutcome::DeterministicFailure);
127                return Err(ApiError::new(
128                    ErrorCode::Timeout,
129                    "HTML extraction did not finish within the timeout",
130                ));
131            }
132        };
133        p.timings.insert("extract".into(), ms(t.elapsed()));
134        if doc.metadata.video_fallback {
135            p.warnings.push(
136                "video_metadata: no readable body text; used the page's embedded video metadata"
137                    .into(),
138            );
139        }
140        // 抓取时拦截页要报错；上传的文件是调用方自己给的，照样提取，只提醒一句
141        if doc.barrier.is_some() {
142            p.warnings.push(
143                "access_barrier: the uploaded HTML looks like a bot-check or access-barrier page, not the content"
144                    .into(),
145            );
146        }
147        fill_metadata(&mut meta, &doc.metadata);
148        // 页面没写图标时提取器按惯例猜 `/favicon.ico`，相对于 file:/// 毫无意义
149        if meta
150            .favicon
151            .as_deref()
152            .is_some_and(|f| f.starts_with("file:"))
153        {
154            meta.favicon = None;
155        }
156        fill_structured(&mut ext, &doc);
157        fill_fingerprints(&mut ext, &doc);
158        // 正文可能不完整的警告（partial_content 等）和抓取时一样给
159        p.warnings.extend(doc.coverage.warnings());
160        let raw_bytes = Some(thin_basis_bytes(&html, &doc.markdown));
161        let video_fallback = doc.metadata.video_fallback;
162        Ok(finish(
163            document(doc, opts, html),
164            meta,
165            ext,
166            p.timings,
167            p.warnings,
168            raw_bytes,
169            video_fallback,
170            p.started,
171        ))
172    }
173}
174
175/// 上传 HTML 里相对链接的基准：调用方没给页面地址（`url` 字段）、页面也没有 `<base href>` 时
176/// 按 `file:///<文件名>` 解析，不冒充任何真实网址。
177fn upload_base(filename: &str) -> Url {
178    let mut base = Url::parse("file:///").expect("static URL");
179    base.set_path(filename);
180    base
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn upload_base_resolves_relative_links_locally() {
189        let base = upload_base("my report.html");
190        assert_eq!(base.as_str(), "file:///my%20report.html");
191        assert_eq!(
192            base.join("img/a.png").unwrap().as_str(),
193            "file:///img/a.png"
194        );
195    }
196}
pagesift_engine/engine/parse.rs

pagesift_engine/engine/
parse.rs
