pagesift_engine/engine/
parse.rs

1//! `/v2/parse`：解析调用方上传的文件。PDF 走和抓取相同的 pagesift-pdf，HTML 走相同的正文提取，
2//! 组装成和 scrape 相同的文档。不联网，所以没有缓存、robots、浏览器渲染这些环节；
3//! CPU 照样经过解析阶段的闸门和 [`cpu_task`] 的核数限制。
4
5use pagesift_api::types::{Upload, UploadKind};
6
7use super::*;
8
9/// [协议] 选项里没给 `timeout` 时整个请求的截止时间，和 Firecrawl `/parse` 的默认值一样（接口兼容）。
10/// 只是上限：PDF 的解析时限按文件大小估计，见 `pdf.rs`。
11const DEFAULT_PARSE_TIMEOUT: Duration = Duration::from_secs(30);
12
13impl Scraper {
14    /// 解析上传的文件并写审计记录（`kind: parse`）。不存快照：上传的文件往往不公开。
15    pub async fn parse(&self, upload: Upload, opts: &ScrapeOptions) -> Result<Document, ApiError> {
16        let Some(audit) = &self.audit else {
17            return self.scrape_inner(Source::Upload(upload), opts).await;
18        };
19        let name = format!("upload:{}", upload.filename);
20        let size = upload.bytes.len();
21        let started = Instant::now();
22        let result = self.scrape_inner(Source::Upload(upload), opts).await;
23        audit.record_parse(&name, size, opts, &result, started.elapsed());
24        result
25    }
26
27    /// 上传文件的基础文档（正文、元数据），相当于抓取时的 `scrape_cached`。
28    pub(super) async fn parse_upload(
29        &self,
30        upload: Upload,
31        opts: &ScrapeOptions,
32    ) -> Result<Document, ApiError> {
33        let started = Instant::now();
34        let deadline = started
35            + opts
36                .timeout
37                .map_or(DEFAULT_PARSE_TIMEOUT, Duration::from_millis);
38        let content_type = match upload.kind {
39            UploadKind::Html => upload
40                .content_type
41                .clone()
42                .filter(|c| c.to_ascii_lowercase().contains("html")),
43            UploadKind::Pdf | UploadKind::Office(_) => None,
44        };
45        // 调用方给了页面的原地址时，它就是来源；否则用文件名
46        let source = upload
47            .url
48            .as_ref()
49            .map_or_else(|| upload.filename.clone(), Url::to_string);
50        let meta = Metadata {
51            source_url: Some(source),
52            status_code: 200,
53            content_type: Some(
54                content_type
55                    .clone()
56                    .unwrap_or_else(|| upload.kind.mime().into()),
57            ),
58            proxy_used: "basic".into(),
59            ..Default::default()
60        };
61        let ext = PagesiftMeta {
62            engine: Some("upload".into()),
63            ..Default::default()
64        };
65        let progress = Progress {
66            started,
67            timings: BTreeMap::new(),
68            warnings: Vec::new(),
69        };
70        match upload.kind {
71            UploadKind::Pdf => {
72                self.pdf_document(upload.bytes, opts, (meta, ext), progress, Some(deadline))
73                    .await
74            }
75            UploadKind::Office(format) => {
76                let input = office::OfficeInput {
77                    bytes: upload.bytes,
78                    format,
79                    content_type: upload.content_type,
80                };
81                self.office_document(input, opts, (meta, ext), progress, Some(deadline))
82                    .await
83            }
84            UploadKind::Html => {
85                let html = pagesift_net::decode_owned(upload.bytes, content_type.as_deref(), None);
86                let base = upload.url.unwrap_or_else(|| upload_base(&upload.filename));
87                self.parse_html(html, &base, opts, (meta, ext), progress, deadline)
88                    .await
89            }
90        }
91    }
92
93    async fn parse_html(
94        &self,
95        decoded: pagesift_net::Decoded,
96        base: &Url,
97        opts: &ScrapeOptions,
98        (mut meta, mut ext): (Metadata, PagesiftMeta),
99        mut p: Progress,
100        deadline: Instant,
101    ) -> Result<Document, ApiError> {
102        if let Some(warning) = decoded.warning() {
103            p.warnings.push(warning);
104        }
105        let stage = self
106            .policy
107            .fetcher()
108            .acquire_stage(pagesift_net::NetworkStage::Parse)
109            .await;
110        let t = Instant::now();
111        let work = process_html(decoded.text, base, Some(200), opts);
112        let (doc, html) = match tokio::time::timeout_at(deadline.into(), work).await {
113            Ok(Ok(result)) => {
114                stage.finish(pagesift_net::StageOutcome::Success);
115                result
116            }
117            Ok(Err(error)) => {
118                stage.finish(pagesift_net::StageOutcome::DeterministicFailure);
119                return Err(error);
120            }
121            Err(_) => {
122                stage.finish(pagesift_net::StageOutcome::DeterministicFailure);
123                return Err(ApiError::new(
124                    ErrorCode::Timeout,
125                    "HTML extraction did not finish within the timeout",
126                ));
127            }
128        };
129        p.timings.insert("extract".into(), ms(t.elapsed()));
130        if doc.metadata.video_fallback {
131            p.warnings.push(
132                "video_metadata: no readable body text; used the page's embedded video metadata"
133                    .into(),
134            );
135        }
136        // 抓取时拦截页要报错；上传的文件是调用方自己给的，照样提取，只提醒一句
137        if doc.barrier.is_some() {
138            p.warnings.push(
139                "access_barrier: the uploaded HTML looks like a bot-check or access-barrier page, not the content"
140                    .into(),
141            );
142        }
143        fill_metadata(&mut meta, &doc.metadata);
144        // 页面没写图标时提取器按惯例猜 `/favicon.ico`，相对于 file:/// 毫无意义
145        if meta
146            .favicon
147            .as_deref()
148            .is_some_and(|f| f.starts_with("file:"))
149        {
150            meta.favicon = None;
151        }
152        fill_structured(&mut ext, &doc);
153        fill_fingerprints(&mut ext, &doc);
154        // 正文可能不完整的警告（partial_content 等）和抓取时一样给
155        p.warnings.extend(doc.coverage.warnings());
156        let raw_bytes = Some(thin_basis_bytes(&html, &doc.markdown));
157        let video_fallback = doc.metadata.video_fallback;
158        Ok(finish(
159            document(doc, opts, html),
160            meta,
161            ext,
162            p.timings,
163            p.warnings,
164            raw_bytes,
165            video_fallback,
166            p.started,
167        ))
168    }
169}
170
171/// 上传 HTML 里相对链接的基准：调用方没给页面地址（`url` 字段）、页面也没有 `<base href>` 时
172/// 按 `file:///<文件名>` 解析，不冒充任何真实网址。
173fn upload_base(filename: &str) -> Url {
174    let mut base = Url::parse("file:///").expect("static URL");
175    base.set_path(filename);
176    base
177}
178
179#[cfg(test)]
180mod tests {
181    use super::*;
182
183    #[test]
184    fn upload_base_resolves_relative_links_locally() {
185        let base = upload_base("my report.html");
186        assert_eq!(base.as_str(), "file:///my%20report.html");
187        assert_eq!(
188            base.join("img/a.png").unwrap().as_str(),
189            "file:///img/a.png"
190        );
191    }
192}
pagesift_engine/engine/parse.rs

pagesift_engine/engine/
parse.rs
