pagesift_engine/engine/
output.rs

1use super::*;
2
3pub(super) fn fetch_request(url: &str, opts: &ScrapeOptions) -> Result<FetchRequest, ApiError> {
4    let mut req = FetchRequest::new(url);
5    for (k, v) in &opts.headers {
6        req = req.header(k.clone(), v.clone());
7    }
8    if let Some(etag) = &opts.if_none_match {
9        req = req.if_none_match(etag.clone());
10    }
11    if let Some(t) = &opts.if_modified_since {
12        let t = httpdate::parse_http_date(t).map_err(|_| {
13            ApiError::invalid(format!("ifModifiedSince is not an HTTP date: {t:?}"))
14        })?;
15        req = req.if_modified_since(t);
16    }
17    req.timeout = opts.timeout.map(Duration::from_millis);
18    Ok(req)
19}
20
21/// `chunking`：正文 Markdown 分块。参数格式见 [`pagesift_extract::ChunkStrategy`]（`maxLevel` 这类驼峰写法也认）。
22pub fn chunk_markdown(
23    md: &str,
24    spec: &serde_json::Value,
25) -> Result<Vec<pagesift_api::types::DocChunk>, ApiError> {
26    // API 用驼峰，提取库用下划线：只改键名
27    let snake = match spec {
28        serde_json::Value::Object(m) => serde_json::Value::Object(
29            m.iter()
30                .map(|(k, v)| {
31                    let mut out = String::new();
32                    for c in k.chars() {
33                        if c.is_ascii_uppercase() {
34                            out.push('_');
35                            out.push(c.to_ascii_lowercase());
36                        } else {
37                            out.push(c);
38                        }
39                    }
40                    (out, v.clone())
41                })
42                .collect(),
43        ),
44        other => other.clone(),
45    };
46    let strategy: pagesift_extract::ChunkStrategy = serde_json::from_value(snake)
47        .map_err(|e| ApiError::invalid(format!("invalid chunking: {e}")))?;
48    let chunks = pagesift_extract::chunk(md, &strategy)
49        .map_err(|e| ApiError::invalid(format!("invalid chunking: {e}")))?;
50    Ok(chunks
51        .into_iter()
52        .map(|c| pagesift_api::types::DocChunk {
53            text: c.text,
54            heading_path: c.heading_path,
55            start: c.start,
56            end: c.end,
57            tokens: c.tokens,
58        })
59        .collect())
60}
61
62/// `queryFilter`：只留下和查询相关的内容，统计写进 `metadata.pagesift.queryFilter`。
63pub(super) async fn apply_query_filter(
64    doc: &mut Document,
65    q: &pagesift_api::types::QueryFilter,
66) -> Result<(), ApiError> {
67    let Some(md) = doc.markdown.take() else {
68        return Ok(());
69    };
70    let fo = pagesift_extract::QueryFilterOptions {
71        threshold: q.threshold,
72        relative: q.relative,
73        top_k: q.top_k,
74        max_tokens: q.max_tokens,
75        min_tokens: q.min_tokens,
76        headings: q.headings,
77    };
78    let query = q.query.clone();
79    let (f, before) = cpu_task(move || {
80        let f = pagesift_extract::filter_markdown_by_query(&md, &query, &fo);
81        (f, pagesift_content::estimate_tokens(&md))
82    })
83    .await?;
84    let ext = doc.metadata.pagesift.get_or_insert_with(Default::default);
85    if f.kept == 0 && f.total > 0 {
86        ext.warnings.push("query_filter_no_match".into());
87    }
88    ext.query_filter = Some(pagesift_api::types::QueryFilterStats {
89        kept_blocks: f.kept,
90        total_blocks: f.total,
91        tokens_before: before,
92        tokens_after: pagesift_content::estimate_tokens(&f.markdown),
93    });
94    doc.markdown = Some(f.markdown);
95    Ok(())
96}
97
98pub(super) async fn process_html(
99    html: String,
100    base: &Url,
101    status: Option<u16>,
102    opts: &ScrapeOptions,
103) -> Result<(pagesift_content::Document, String), ApiError> {
104    let copts = content_options(base, status, opts);
105    let tags = tag_filter(opts)?;
106    cpu_task(move || {
107        // RSS / Atom / RDF feed 不是网页：按条目解析，别让 HTML 提取器把标签剥成一锅粥。
108        if let Some(document) = super::feed::parse(&html, copts.links) {
109            return (document, html);
110        }
111        // includeTags / excludeTags 只影响提取；rawHtml 仍是原始页面
112        let filtered = tags.map(|t| t.apply(&html));
113        let source = filtered.as_deref().unwrap_or(&html);
114        let document = CONTENT_WORKSPACE.with(|workspace| {
115            pagesift_content::process_with_workspace(source, &copts, &mut workspace.borrow_mut())
116        });
117        (document, html)
118    })
119    .await
120}
121
122/// 提取参数：只按请求要的格式算（链接、图片、清洗后的 HTML、变化检测的指纹）。
123pub(super) fn content_options(
124    base: &Url,
125    status: Option<u16>,
126    opts: &ScrapeOptions,
127) -> pagesift_content::Options {
128    pagesift_content::Options {
129        base_url: Some(base.clone()),
130        only_main_content: opts.only_main_content,
131        status,
132        detect_barrier: true,
133        clean_html: opts.wants("html"),
134        links: opts.wants("links"),
135        images: opts.wants("images"),
136        // 完整指纹（原始 HTML 的哈希、再渲染一遍纯文本做规范化哈希）约占提取 CPU 的一成，只在变化检测要用时算
137        fingerprints: opts.wants("changeTracking"),
138        ..Default::default()
139    }
140}
141
142/// `includeTags` / `excludeTags` 的筛选规则；都没给时为 `None`，选择器写错时返回 `INVALID_REQUEST`。
143pub fn tag_filter(opts: &ScrapeOptions) -> Result<Option<pagesift_content::TagFilter>, ApiError> {
144    pagesift_content::TagFilter::new(&opts.include_tags, &opts.exclude_tags)
145        .map_err(ApiError::invalid)
146}
147
148pub(super) fn check_barrier(doc: &pagesift_content::Document) -> Result<(), ApiError> {
149    let Some(b) = &doc.barrier else { return Ok(()) };
150    Err(ApiError::new(
151        ErrorCode::AccessBarrier,
152        format!(
153            "the site returned an access barrier ({} {}); pagesift reports it and does not retry to get past it",
154            b.vendor,
155            b.kind.as_str()
156        ),
157    )
158    .with_details(serde_json::json!({
159        "vendor": b.vendor,
160        "kind": b.kind.as_str(),
161        "confidence": b.confidence,
162        "ruleId": b.rule_id,
163    })))
164}
165
166pub(super) fn document(
167    doc: pagesift_content::Document,
168    opts: &ScrapeOptions,
169    raw_html: String,
170) -> Document {
171    Document {
172        markdown: opts.wants("markdown").then_some(doc.markdown),
173        html: doc.html,
174        raw_html: opts.wants("rawHtml").then_some(raw_html),
175        link_text: doc
176            .links
177            .as_ref()
178            .map(|l| l.iter().map(|l| l.text.clone()).collect()),
179        alternates: doc.metadata.alternates.clone(),
180        links: doc.links.map(|l| l.into_iter().map(|l| l.url).collect()),
181        images: doc.images.map(|i| i.into_iter().map(|i| i.src).collect()),
182        ..Default::default()
183    }
184}
185
186/// 正文 Markdown 的哈希总是给（Markdown 只有几 KB，几乎不花时间）；原始 HTML 和规范化纯文本的
187/// 哈希只在请求变化检测时算（见 `process_html`），它们要整页再过一遍。
188pub(super) fn fill_fingerprints(ext: &mut PagesiftMeta, doc: &pagesift_content::Document) {
189    match &doc.fingerprints {
190        Some(f) => {
191            ext.content_hash = Some(f.markdown.clone());
192            ext.raw_hash = Some(f.raw_html.clone());
193            ext.text_hash = Some(f.text.clone());
194        }
195        None => ext.content_hash = Some(pagesift_content::hash_hex(doc.markdown.as_bytes())),
196    }
197}
198
199/// 页面自带的 JSON-LD 原样透传到 `metadata.pagesift.jsonLd`，供下游直接消费。
200pub(super) fn fill_structured(ext: &mut PagesiftMeta, doc: &pagesift_content::Document) {
201    ext.json_ld = doc.metadata.json_ld().to_vec();
202}
203
204pub(super) use pagesift_sink::meta::fill as fill_metadata;
205
206#[allow(clippy::too_many_arguments)] // 把各阶段的结果汇成文档，只在抓取流程末尾调用一处
207pub(super) fn finish(
208    mut doc: Document,
209    mut meta: Metadata,
210    mut ext: PagesiftMeta,
211    mut timings: BTreeMap<String, f64>,
212    mut warnings: Vec<String>,
213    raw_bytes: Option<usize>,
214    video_fallback: bool,
215    started: Instant,
216) -> Document {
217    timings.insert("total".into(), ms(started.elapsed()));
218    if meta.status_code >= 400 {
219        warnings.push(format!("http_error_status: HTTP {}", meta.status_code));
220    } else if (300..400).contains(&meta.status_code) {
221        // 跟随重定向之后仍是 3xx：没有 Location、Location 不可用，或 300 这类需要人工选择的响应。
222        warnings.push(format!(
223            "http_redirect_status: HTTP {} was not followed (missing or unusable Location)",
224            meta.status_code
225        ));
226    }
227    // 只数到阈值为止：4.49 MB 的正文没必要为了一个警告全量扫一遍。
228    // [策略] 警告门槛（不改结果，只提示 JS 壳页）：先验包 v0：0.81 的现值，约 80 个英文词
229    const THIN_MAX_CHARS: usize = 500;
230    let markdown_chars = doc.markdown.as_deref().map(|markdown| {
231        let mut count = 0;
232        for c in markdown.chars() {
233            if !c.is_whitespace() {
234                count += 1;
235                if count > THIN_MAX_CHARS {
236                    break;
237                }
238            }
239        }
240        count
241    });
242    if meta.status_code < 400 && markdown_chars == Some(0) {
243        warnings.push(format!(
244            "empty_content: no readable text extracted (HTTP {})",
245            meta.status_code
246        ));
247    }
248    // "过薄"与"全空"是两回事：页面有实质内容（HTML 不小）却只抽出几百字，
249    // 多半是 JS 壳，或者正文选择把主体丢了。只警告，不改结果。
250    // [策略] 警告门槛：先验包 v0：0.81 的现值
251    const THIN_MIN_HTML_BYTES: usize = 20_000;
252    if meta.status_code < 400
253        && !video_fallback
254        && let (Some(chars @ 1..THIN_MAX_CHARS), Some(bytes)) = (markdown_chars, raw_bytes)
255        && bytes >= THIN_MIN_HTML_BYTES
256    {
257        warnings.push(format!(
258            "thin_content: extracted {chars} characters from {} KB of HTML; the page may be a JavaScript shell or the main content was not selected",
259            bytes / 1024
260        ));
261    }
262    ext.timings = timings;
263    ext.warnings = warnings;
264    meta.pagesift = Some(ext);
265    doc.metadata = meta;
266    doc
267}
268
269pub(super) fn ms(d: Duration) -> f64 {
270    (d.as_secs_f64() * 1000.0 * 100.0).round() / 100.0
271}
272
273/// 判断 `thin_content` 用的页面体积。正文不薄时直接用 HTML 长度，不多扫一遍；
274/// 正文薄时扣掉 `<style>` 的内容和 `data:` 内嵌资源：一张 8 MB 的 base64 图片
275/// 不代表页面“有大量内容却没抽出来”。`<script>` 不扣：内联脚本大而正文薄正是 JS 壳页的特征。
276pub(super) fn thin_basis_bytes(html: &str, markdown: &str) -> usize {
277    // [策略] 与 THIN_MAX_CHARS 同值：正文不薄时不必扣减
278    const ENOUGH: usize = 500;
279    if markdown
280        .chars()
281        .filter(|c| !c.is_whitespace())
282        .take(ENOUGH)
283        .count()
284        >= ENOUGH
285    {
286        return html.len();
287    }
288    let lower = html.to_ascii_lowercase();
289    let mut removed = 0usize;
290    for tag in ["style"] {
291        let (open, close) = (format!("<{tag}"), format!("</{tag}"));
292        let mut at = 0;
293        while let Some(i) = lower[at..].find(&open) {
294            let start = at + i;
295            let Some(j) = lower[start..].find(&close) else {
296                removed += lower.len() - start;
297                break;
298            };
299            removed += j;
300            at = start + j + close.len();
301        }
302    }
303    let mut at = 0;
304    while let Some(i) = lower[at..].find("data:") {
305        let start = at + i;
306        let len = lower[start..]
307            .find(['"', '\'', ')', ' ', '>'])
308            .unwrap_or(lower.len() - start);
309        removed += len;
310        at = start + len.max(1);
311    }
312    html.len().saturating_sub(removed)
313}
314
315/// JSON 响应（`application/json`、`*+json`）不是网页：当 HTML 解析只会剩下碎片。
316/// 合法 JSON 格式化后放进代码块；太大的不格式化，原样放进去。不是合法 JSON 时返回 `None`，按文本继续。
317pub(super) fn json_markdown(content_type: Option<&str>, body: &[u8]) -> Option<String> {
318    // [策略] 安全上限：格式化会把 JSON 放大数倍（缩进、换行），超过它原样输出，防大响应撑爆内存
319    const PRETTY_MAX_BYTES: usize = 5 * 1024 * 1024;
320    let kind = content_type?.split(';').next()?.trim().to_ascii_lowercase();
321    if kind != "application/json" && kind != "text/json" && !kind.ends_with("+json") {
322        return None;
323    }
324    let text = std::str::from_utf8(body.strip_prefix(b"\xef\xbb\xbf").unwrap_or(body)).ok()?;
325    let value: serde_json::Value = serde_json::from_str(text).ok()?;
326    let rendered = if text.len() <= PRETTY_MAX_BYTES {
327        serde_json::to_string_pretty(&value).ok()?
328    } else {
329        text.trim().to_string()
330    };
331    // 内容里有三个反引号时加长围栏，保证代码块不被提前闭合。
332    let mut fence = String::from("```");
333    while rendered.contains(fence.as_str()) {
334        fence.push('`');
335    }
336    Some(format!("{fence}json\n{rendered}\n{fence}\n"))
337}
338
339/// 魔数优先：`%PDF-` 开头就是 PDF（不管头怎么写）。只有头说是 PDF 时，正文明显是 HTML/XML
340/// 的不算——CDN、WAF 把拦截页、登录页标成 `application/pdf` 返回时按网页处理，拦截页识别照常生效。
341pub(super) fn is_pdf(body: &[u8], content_type: Option<&str>) -> bool {
342    body.starts_with(b"%PDF-")
343        || (content_type.is_some_and(|c| c.to_ascii_lowercase().contains("application/pdf"))
344            && !looks_like_markup(body))
345}
346
347/// 非 HTML/文本内容（图片、音视频、字体、压缩包等）不应该被当作文本抽取。
348/// 返回不支持的类型名；`None` 表示可以按文本继续处理。
349///
350/// Content-Type 可能标错（例如 `application/octet-stream` 实际是 HTML），所以明显的
351/// HTML/XML 开头仍然放行；图片/音视频/字体等即使正文碰巧以 `<` 开头也拒绝。
352pub(super) fn unsupported_content_type(content_type: Option<&str>, body: &[u8]) -> Option<String> {
353    let raw = content_type?;
354    let kind = raw.split(';').next()?.trim().to_ascii_lowercase();
355    if kind.is_empty() {
356        return None;
357    }
358    if kind.starts_with("image/")
359        || kind.starts_with("audio/")
360        || kind.starts_with("video/")
361        || kind.starts_with("font/")
362    {
363        return Some(kind);
364    }
365    // `application/octet-stream`、`binary/octet-stream` 返回的 PDF 很常见：认魔数，交给 PDF 解析。
366    if kind == "application/pdf" || body.starts_with(b"%PDF-") {
367        return None;
368    }
369    if kind.starts_with("text/")
370        || kind.contains("html")
371        || kind.contains("xml")
372        || kind.contains("json")
373        || matches!(
374            kind.as_str(),
375            "application/javascript" | "application/x-javascript" | "application/ecmascript"
376        )
377    {
378        return None;
379    }
380    if looks_like_markup(body) {
381        return None;
382    }
383    Some(kind)
384}
385
386fn looks_like_markup(body: &[u8]) -> bool {
387    let head = &body[..body.len().min(1024)];
388    let text = String::from_utf8_lossy(head);
389    let text = text.trim_start_matches('\u{feff}').trim_start();
390    let lower = text.to_ascii_lowercase();
391    lower.starts_with("<!doctype html")
392        || lower.starts_with("<html")
393        || lower.starts_with("<head")
394        || lower.starts_with("<body")
395        || lower.starts_with("<?xml")
396}
397
398pub fn policy_error(e: pagesift_policy::Error) -> ApiError {
399    use pagesift_policy::ErrorCode as P;
400    let code = match e.code() {
401        P::RobotsDisallowed => ErrorCode::RobotsDisallowed,
402        P::AccessBarrier => ErrorCode::AccessBarrier,
403        P::RateLimited => ErrorCode::RateLimited,
404        P::HostCooldown => ErrorCode::HostCooldown,
405        P::Timeout => ErrorCode::Timeout,
406        P::SsrfBlocked => ErrorCode::SsrfBlocked,
407        P::TooLarge => ErrorCode::TooLarge,
408        P::UpstreamError => ErrorCode::UpstreamError,
409        P::UnsupportedOption => ErrorCode::UnsupportedOption,
410        P::InvalidRequest => ErrorCode::InvalidRequest,
411        P::DnsFailure => ErrorCode::DnsFailure,
412        P::NetworkError => ErrorCode::NetworkError,
413        P::TlsCertExpired => ErrorCode::TlsCertExpired,
414        P::TlsHostnameMismatch => ErrorCode::TlsHostnameMismatch,
415        P::TlsUntrustedRoot => ErrorCode::TlsUntrustedRoot,
416        P::TlsError => ErrorCode::TlsError,
417        _ => ErrorCode::Internal,
418    };
419    let retry = e.retry_after().map(|d| d.as_secs().max(1));
420    let mut err = ApiError::new(code, e.to_string());
421    if let Some(s) = retry {
422        err = err.with_retry_after(s);
423    }
424    err
425}
426
427#[cfg(test)]
428mod tests {
429    use super::{is_pdf, json_markdown, thin_basis_bytes, unsupported_content_type};
430
431    #[test]
432    fn thin_basis_ignores_inline_payloads() {
433        let image = format!(
434            "<p>short</p><img src=\"data:image/png;base64,{}\">",
435            "A".repeat(100_000)
436        );
437        assert!(thin_basis_bytes(&image, "short") < 1_000);
438        let style = format!("<STYLE>{}</STYLE><p>x</p>", "y".repeat(50_000));
439        assert!(thin_basis_bytes(&style, "x") < 1_000);
440        let next = format!(
441            "<script id=\"__NEXT_DATA__\">{}</script><div id=\"app\"></div>",
442            "x".repeat(50_000)
443        );
444        assert_eq!(thin_basis_bytes(&next, ""), next.len());
445        let shell = format!("<div>{}</div>", "<span></span>".repeat(5_000));
446        assert_eq!(thin_basis_bytes(&shell, ""), shell.len());
447        let long = "word ".repeat(200);
448        assert_eq!(thin_basis_bytes(&image, &long), image.len());
449    }
450
451    #[test]
452    fn json_responses_become_code_blocks() {
453        let md = json_markdown(
454            Some("application/json; charset=utf-8"),
455            br#"{"html":"<h1>x</h1>","n":[1,2]}"#,
456        )
457        .unwrap();
458        assert!(md.starts_with("```json\n{"), "{md}");
459        assert!(md.contains("\"html\": \"<h1>x</h1>\""), "{md}");
460        assert!(md.ends_with("\n```\n"), "{md}");
461        let md = json_markdown(Some("application/ld+json"), br#"{"a":"```"}"#).unwrap();
462        assert!(md.starts_with("````json"), "{md}");
463        assert_eq!(json_markdown(Some("text/html"), b"{}"), None);
464        assert_eq!(json_markdown(Some("application/json"), b"<html>"), None);
465    }
466
467    #[test]
468    fn rejects_binary_content_types() {
469        for kind in [
470            "image/png",
471            "image/jpeg",
472            "image/svg+xml",
473            "audio/mpeg",
474            "video/mp4",
475            "font/woff2",
476            "application/zip",
477            "application/octet-stream",
478        ] {
479            assert_eq!(
480                unsupported_content_type(Some(kind), b"\x89PNG\r\n"),
481                Some(kind.to_string()),
482                "{kind}"
483            );
484        }
485    }
486
487    #[test]
488    fn accepts_text_and_mislabeled_markup() {
489        for kind in [
490            "text/html; charset=utf-8",
491            "text/plain",
492            "application/xhtml+xml",
493            "application/json",
494            "application/rss+xml",
495            "application/pdf",
496        ] {
497            assert_eq!(
498                unsupported_content_type(Some(kind), b"anything"),
499                None,
500                "{kind}"
501            );
502        }
503        assert_eq!(
504            unsupported_content_type(Some("application/octet-stream"), b"<!doctype html><p>x"),
505            None
506        );
507        assert_eq!(
508            unsupported_content_type(Some("application/octet-stream"), b"  <html><p>x"),
509            None
510        );
511        assert_eq!(unsupported_content_type(None, b"\x89PNG"), None);
512        for kind in ["application/octet-stream", "binary/octet-stream"] {
513            assert_eq!(
514                unsupported_content_type(Some(kind), b"%PDF-1.7\n1 0 obj"),
515                None
516            );
517            assert!(is_pdf(b"%PDF-1.7\n1 0 obj", Some(kind)));
518        }
519        // 头说是 PDF：正文是网页（拦截页、登录页）时按网页处理；不是网页的照样交给 PDF 解析
520        let pdf = Some("application/pdf");
521        assert!(!is_pdf(
522            b"<!DOCTYPE html><html><body>Access denied</body></html>",
523            pdf
524        ));
525        assert!(!is_pdf(b"\n  <html><body>x</body></html>", pdf));
526        assert!(is_pdf(b"junk before the header %PDF-1.4", pdf));
527        assert!(is_pdf(b"%PDF-1.4", Some("text/html")));
528        assert_eq!(
529            unsupported_content_type(Some("application/octet-stream"), b"\x00\x01binary"),
530            Some("application/octet-stream".into())
531        );
532    }
533}
pagesift_engine/engine/output.rs

pagesift_engine/engine/
output.rs
