pagesift_engine/engine/
office.rs

1//! Office、邮件、压缩包、文本类 → 文档：抓到的（`/v2/scrape`）和上传的（`/v2/parse`）走同一段代码。
2//! 转换在 pagesift-office 里；EPUB 的 XHTML 章节和邮件的 HTML 正文交给和网页相同的 HTML 提取器
3//! （保留整页，不做正文筛选）。压缩包成员和邮件附件回到这里递归转换，内层 PDF 只读文字层。
4
5use pagesift_office::{Converted, Format};
6
7use super::*;
8
9/// [策略] 先验包 v0：0.81 的现值。没给 `timeout`、这个工人上转换的样本还不够时一份文档的转换时限；
10/// 样本够了以后按"字节数 → 耗时"的预测上界（[`pagesift_stat::size_time`]）。
11const DEFAULT_OFFICE_TIMEOUT: Duration = Duration::from_secs(30);
12
13thread_local! {
14    /// 这个 CPU 工人上 Office 转换的耗时样本（工人本地，不加锁）。
15    static OFFICE_TIME: std::cell::RefCell<pagesift_stat::size_time::SizeTime> =
16        std::cell::RefCell::default();
17}
18
19/// 转换的截止时间：学到的预测上界和请求的截止时间取早的；还没学到时和 0.81 一样
20/// （有请求截止时间就用它，没有就用先验）。
21fn office_deadline(bytes: usize, started: Instant, request: Option<Instant>) -> Instant {
22    let learned = OFFICE_TIME
23        .with_borrow(|m| m.timeout(bytes, pagesift_stat::Delta::DEFAULT))
24        .map(|t| started + t);
25    match (request, learned) {
26        (Some(r), Some(l)) => r.min(l),
27        (Some(r), None) => r,
28        (None, Some(l)) => l,
29        (None, None) => started + DEFAULT_OFFICE_TIMEOUT,
30    }
31}
32
33/// 把一份 Office/EPUB/CSV 文档转成 Markdown，供引擎和评测共用。`deadline` 到了报超时。
34/// CSV 的字符集按正文探测；知道 Content-Type 时用 [`office_markdown_from`]。
35pub fn office_markdown(
36    bytes: &[u8],
37    format: Format,
38    deadline: Option<Instant>,
39) -> Result<Converted, pagesift_office::Error> {
40    office_markdown_from(bytes, format, None, deadline)
41}
42
43/// HTML 转 Markdown：保留整页，相对路径按包里的位置解析。
44fn html_to_markdown(xhtml: &str, path: &str) -> String {
45    let base = Url::parse("file:///")
46        .ok()
47        .and_then(|b| b.join(&path.replace(' ', "%20")).ok());
48    let opts = pagesift_content::Options {
49        base_url: base,
50        only_main_content: false,
51        detect_barrier: false,
52        ..Default::default()
53    };
54    pagesift_content::process(xhtml, &opts).markdown
55}
56
57/// 压缩包成员或邮件附件：按内容和文件名认类型，转不了的返回 `None`（只留在清单里）。
58fn member_markdown(
59    bytes: &[u8],
60    name: &str,
61    room: usize,
62    deadline: Option<Instant>,
63) -> Result<Option<Converted>, pagesift_office::Error> {
64    use pagesift_office::Error as E;
65    if bytes.starts_with(b"%PDF-") {
66        let popts = pagesift_pdf::Options {
67            timeout: deadline.map(|d| d.saturating_duration_since(Instant::now())),
68            ..Default::default()
69        };
70        return match pagesift_pdf::extract_with(bytes, &popts) {
71            Ok(pdf) => Ok(Some(Converted {
72                parts: pdf.pages.len(),
73                markdown: pdf.markdown,
74                title: pdf.metadata.title,
75                ..Default::default()
76            })),
77            Err(pagesift_pdf::Error::Timeout) => Err(E::Timeout),
78            Err(e) => Err(E::Malformed(e.to_string())),
79        };
80    }
81    let ext = name.rsplit_once('.').map(|(_, e)| e.to_ascii_lowercase());
82    if matches!(ext.as_deref(), Some("html" | "htm" | "xhtml")) {
83        let decoded = pagesift_net::decode(bytes, None, None);
84        return Ok(Some(Converted {
85            markdown: html_to_markdown(&decoded.text, name),
86            ..Default::default()
87        }));
88    }
89    let Some(format) = Format::detect(bytes, Some(name), None) else {
90        return Ok(None);
91    };
92    convert_with_room(bytes, format, None, deadline, Some(room)).map(Some)
93}
94
95fn convert_with_room(
96    bytes: &[u8],
97    format: Format,
98    content_type: Option<&str>,
99    deadline: Option<Instant>,
100    room: Option<usize>,
101) -> Result<Converted, pagesift_office::Error> {
102    let member = |data: &[u8], name: &str, room: usize| member_markdown(data, name, room, deadline);
103    let mut opts = pagesift_office::Options {
104        deadline,
105        html: Some(&html_to_markdown),
106        member: Some(&member),
107        ..Default::default()
108    };
109    if let Some(room) = room {
110        opts.limits.max_output_bytes = room;
111    }
112    if format.decodes_text() {
113        let (text, warning) = csv_text::decode(bytes, content_type);
114        let mut doc = pagesift_office::convert_text(&text, format, &opts)?;
115        doc.warnings.extend(warning);
116        return Ok(doc);
117    }
118    pagesift_office::convert(bytes, format, &opts)
119}
120
121/// 同 [`office_markdown`]，另给文件来源的 Content-Type（响应头或上传时的）：CSV 按它的
122/// `charset` 解码，和网页一样；声明和正文对不上时纠正并给 `decode:` 警告。
123pub fn office_markdown_from(
124    bytes: &[u8],
125    format: Format,
126    content_type: Option<&str>,
127    deadline: Option<Instant>,
128) -> Result<Converted, pagesift_office::Error> {
129    convert_with_room(bytes, format, content_type, deadline, None)
130}
131
132/// Office/EPUB 里的媒体文件作为附件：来源是部件路径（与 Markdown 里的图片引用一致，空格写成 `%20`）。
133fn office_media(
134    bytes: &[u8],
135    format: Format,
136) -> (
137    Vec<pagesift_api::types::Attachment>,
138    Vec<String>,
139    Vec<pagesift_api::types::ImageAssetResult>,
140) {
141    let opts = pagesift_office::Options::default();
142    let mut out = Vec::new();
143    let mut warnings = Vec::new();
144    let mut failed = Vec::new();
145    let list = match pagesift_office::media(bytes, format, &opts) {
146        Ok(list) => list,
147        Err(e) => return (out, vec![format!("images_not_exported: {e}")], failed),
148    };
149    for m in list {
150        match pagesift_office::media_bytes(bytes, &m.part, &opts) {
151            Ok(Some(data)) => match pagesift_api::types::attachment::sniff_image_format(&data) {
152                Some(format) => out.push(pagesift_api::types::Attachment::new(
153                    data,
154                    format,
155                    Some(m.part.replace(' ', "%20")),
156                )),
157                None => {
158                    warnings.push(format!(
159                        "image_not_exported: {}: unrecognized image bytes",
160                        m.part
161                    ));
162                    failed.push(pagesift_api::types::ImageAssetResult {
163                        source: m.part.clone(),
164                        status: pagesift_api::types::ImageAssetStatus::Failed,
165                        id: None,
166                        mime: None,
167                        bytes: None,
168                        reason: Some("unrecognized_image_bytes".into()),
169                    });
170                }
171            },
172            Ok(None) => {
173                warnings.push(format!("image_not_exported: {}: missing part", m.part));
174                failed.push(pagesift_api::types::ImageAssetResult {
175                    source: m.part.clone(),
176                    status: pagesift_api::types::ImageAssetStatus::Failed,
177                    id: None,
178                    mime: None,
179                    bytes: None,
180                    reason: Some("missing_part".into()),
181                });
182            }
183            Err(e) => {
184                warnings.push(format!("image_not_exported: {}: {e}", m.part));
185                failed.push(pagesift_api::types::ImageAssetResult {
186                    source: m.part.clone(),
187                    status: pagesift_api::types::ImageAssetStatus::Failed,
188                    id: None,
189                    mime: None,
190                    bytes: None,
191                    reason: Some(e.to_string()),
192                });
193            }
194        }
195    }
196    (out, warnings, failed)
197}
198
199/// 文件名或网址路径的扩展名、Content-Type、内容一起判断是不是要转换的文档。
200/// 文本类不走这里（网上的 text/plain、XML、JSON 还是网页路径）；
201/// 服务器明说是 HTML 的不信扩展名（`/data.csv` 返回的也可能是网页）。
202pub(super) fn office_format(body: &[u8], name: &str, content_type: Option<&str>) -> Option<Format> {
203    let html = content_type.is_some_and(|c| c.to_ascii_lowercase().contains("html"));
204    Format::detect(body, Some(name), content_type)
205        .filter(|f| f.fetchable() && !(html && f.is_text_based()))
206}
207
208/// 一份待转换的文档：字节、格式，以及来源的 Content-Type（响应头或上传字段；CSV 按它的 `charset` 解码）。
209pub(super) struct OfficeInput {
210    pub(super) bytes: Vec<u8>,
211    pub(super) format: Format,
212    pub(super) content_type: Option<String>,
213}
214
215impl Scraper {
216    pub(super) async fn office_document(
217        &self,
218        input: OfficeInput,
219        opts: &ScrapeOptions,
220        (mut meta, ext): (Metadata, PagesiftMeta),
221        mut p: Progress,
222        deadline: Option<Instant>,
223    ) -> Result<Document, ApiError> {
224        let stage = self
225            .policy
226            .fetcher()
227            .acquire_stage(pagesift_net::NetworkStage::Parse)
228            .await;
229        let t = Instant::now();
230        let format = input.format;
231        let export = opts.export_images;
232        let (result, media) = cpu_task(move || {
233            let started = Instant::now();
234            let bytes = input.bytes.len();
235            let deadline = office_deadline(bytes, started, deadline);
236            let result = office_markdown_from(
237                &input.bytes,
238                input.format,
239                input.content_type.as_deref(),
240                Some(deadline),
241            );
242            if result.is_ok() {
243                OFFICE_TIME.with_borrow_mut(|m| m.observe(bytes, started.elapsed()));
244            }
245            // `exportImages`：文档里的媒体原样从 ZIP 取出
246            let media =
247                (export && result.is_ok()).then(|| office_media(&input.bytes, input.format));
248            (result, media)
249        })
250        .await?;
251        let doc = match result {
252            Ok(doc) => {
253                stage.finish(pagesift_net::StageOutcome::Success);
254                doc
255            }
256            Err(error) => {
257                stage.finish(pagesift_net::StageOutcome::DeterministicFailure);
258                return Err(office_error(format, error));
259            }
260        };
261        p.timings.insert(format.extension().into(), ms(t.elapsed()));
262        p.warnings.extend(doc.warnings);
263        let (attachments, media_warnings, mut image_assets) = media.unwrap_or_default();
264        if opts.image_assets.is_none() {
265            image_assets.clear();
266        }
267        p.warnings.extend(media_warnings);
268        meta.title = doc.title;
269        Ok(finish(
270            Document {
271                markdown: opts.wants("markdown").then_some(doc.markdown),
272                attachments,
273                image_assets,
274                ..Default::default()
275            },
276            meta,
277            ext,
278            p.timings,
279            p.warnings,
280            None,
281            false,
282            p.started,
283        ))
284    }
285}
286
287/// 文件本身的问题是 4xx：坏文件 400 INVALID_REQUEST，加密或用到不支持的东西 400 UNSUPPORTED_OPTION，
288/// 超限 413，超时 408。
289pub(super) fn office_error(format: Format, e: pagesift_office::Error) -> ApiError {
290    use pagesift_office::Error as E;
291    let code = match e {
292        E::Encrypted | E::Unsupported(_) => ErrorCode::UnsupportedOption,
293        E::TooLarge(_) => ErrorCode::TooLarge,
294        E::Timeout => ErrorCode::Timeout,
295        _ => ErrorCode::InvalidRequest,
296    };
297    ApiError::new(
298        code,
299        format!("could not read {}: {e}", format.extension().to_uppercase()),
300    )
301}
302
303#[cfg(test)]
304mod tests {
305    use super::*;
306
307    #[test]
308    fn office_errors_are_caller_errors() {
309        use pagesift_office::Error as E;
310        for (e, code) in [
311            (E::Encrypted, ErrorCode::UnsupportedOption),
312            (E::TooLarge("x"), ErrorCode::TooLarge),
313            (E::Timeout, ErrorCode::Timeout),
314            (E::Malformed("x".into()), ErrorCode::InvalidRequest),
315        ] {
316            assert_eq!(office_error(Format::Docx, e).code, code);
317        }
318    }
319
320    #[test]
321    fn html_responses_are_not_csv() {
322        assert_eq!(
323            office_format(b"a,b", "/data.csv", Some("text/plain")),
324            Some(Format::Csv)
325        );
326        assert_eq!(office_format(b"<p>", "/data.csv", Some("text/html")), None);
327        assert_eq!(office_format(b"<p>", "/page", None), None);
328        // 文本类不抢网页路径；邮件和 RTF 这种内容明确的才接
329        assert_eq!(office_format(b"x", "/a.txt", Some("text/plain")), None);
330        assert_eq!(
331            office_format(b"{}", "/a.json", Some("application/json")),
332            None
333        );
334        assert_eq!(office_format(b"<p>", "/m.eml", Some("text/html")), None);
335        assert_eq!(
336            office_format(b"Subject: x\n\ny", "/m.eml", None),
337            Some(Format::Eml)
338        );
339        assert_eq!(office_format(b"{\\rtf1 x}", "/f", None), Some(Format::Rtf));
340    }
341}
pagesift_engine/engine/office.rs

pagesift_engine/engine/
office.rs
