pagesift_engine/engine/
output.rs

1use super::*;
2
3pub(super) fn fetch_request(url: &str, opts: &ScrapeOptions) -> Result<FetchRequest, ApiError> {
4    let mut req = FetchRequest::new(url);
5    for (k, v) in &opts.headers {
6        req = req.header(k.clone(), v.clone());
7    }
8    if let Some(etag) = &opts.if_none_match {
9        req = req.if_none_match(etag.clone());
10    }
11    if let Some(t) = &opts.if_modified_since {
12        let t = httpdate::parse_http_date(t).map_err(|_| {
13            ApiError::invalid(format!("ifModifiedSince is not an HTTP date: {t:?}"))
14        })?;
15        req = req.if_modified_since(t);
16    }
17    req.timeout = opts.timeout.map(Duration::from_millis);
18    Ok(req)
19}
20
21/// `chunking`：正文 Markdown 分块。参数格式见 [`pagesift_extract::ChunkStrategy`]（`maxLevel` 这类驼峰写法也认）。
22pub fn chunk_markdown(
23    md: &str,
24    spec: &serde_json::Value,
25) -> Result<Vec<pagesift_api::types::DocChunk>, ApiError> {
26    // API 用驼峰，提取库用下划线：只改键名
27    let snake = match spec {
28        serde_json::Value::Object(m) => serde_json::Value::Object(
29            m.iter()
30                .map(|(k, v)| {
31                    let mut out = String::new();
32                    for c in k.chars() {
33                        if c.is_ascii_uppercase() {
34                            out.push('_');
35                            out.push(c.to_ascii_lowercase());
36                        } else {
37                            out.push(c);
38                        }
39                    }
40                    (out, v.clone())
41                })
42                .collect(),
43        ),
44        other => other.clone(),
45    };
46    let strategy: pagesift_extract::ChunkStrategy = serde_json::from_value(snake)
47        .map_err(|e| ApiError::invalid(format!("invalid chunking: {e}")))?;
48    let chunks = pagesift_extract::chunk(md, &strategy)
49        .map_err(|e| ApiError::invalid(format!("invalid chunking: {e}")))?;
50    Ok(chunks
51        .into_iter()
52        .map(|c| pagesift_api::types::DocChunk {
53            text: c.text,
54            heading_path: c.heading_path,
55            start: c.start,
56            end: c.end,
57            tokens: c.tokens,
58        })
59        .collect())
60}
61
62/// `queryFilter`：只留下和查询相关的内容，统计写进 `metadata.pagesift.queryFilter`。
63pub(super) async fn apply_query_filter(
64    doc: &mut Document,
65    q: &pagesift_api::types::QueryFilter,
66) -> Result<(), ApiError> {
67    let Some(md) = doc.markdown.take() else {
68        return Ok(());
69    };
70    let fo = pagesift_extract::QueryFilterOptions {
71        threshold: q.threshold,
72        relative: q.relative,
73        top_k: q.top_k,
74        max_tokens: q.max_tokens,
75        min_tokens: q.min_tokens,
76        headings: q.headings,
77    };
78    let query = q.query.clone();
79    let (f, before) = cpu_task(move || {
80        let f = pagesift_extract::filter_markdown_by_query(&md, &query, &fo);
81        (f, pagesift_content::estimate_tokens(&md))
82    })
83    .await?;
84    let ext = doc.metadata.pagesift.get_or_insert_with(Default::default);
85    if f.kept == 0 && f.total > 0 {
86        ext.warnings.push("query_filter_no_match".into());
87    }
88    ext.query_filter = Some(pagesift_api::types::QueryFilterStats {
89        kept_blocks: f.kept,
90        total_blocks: f.total,
91        tokens_before: before,
92        tokens_after: pagesift_content::estimate_tokens(&f.markdown),
93    });
94    doc.markdown = Some(f.markdown);
95    Ok(())
96}
97
98pub(super) async fn process_html(
99    html: String,
100    base: &Url,
101    status: Option<u16>,
102    opts: &ScrapeOptions,
103) -> Result<(pagesift_content::Document, String), ApiError> {
104    let copts = content_options(base, status, opts);
105    let tags = tag_filter(opts)?;
106    cpu_task(move || {
107        // RSS / Atom / RDF feed 不是网页：按条目解析，别让 HTML 提取器把标签剥成一锅粥。
108        if let Some(document) = super::feed::parse(&html, copts.links) {
109            return (document, html);
110        }
111        // includeTags / excludeTags 只影响提取；rawHtml 仍是原始页面
112        let filtered = tags.map(|t| t.apply(&html));
113        let source = filtered.as_deref().unwrap_or(&html);
114        let document = CONTENT_WORKSPACE.with(|workspace| {
115            pagesift_content::process_with_workspace(source, &copts, &mut workspace.borrow_mut())
116        });
117        (document, html)
118    })
119    .await
120}
121
122/// 提取参数：只按请求要的格式算（链接、图片、清洗后的 HTML、变化检测的指纹）。
123pub(super) fn content_options(
124    base: &Url,
125    status: Option<u16>,
126    opts: &ScrapeOptions,
127) -> pagesift_content::Options {
128    pagesift_content::Options {
129        base_url: Some(base.clone()),
130        only_main_content: opts.only_main_content,
131        status,
132        detect_barrier: true,
133        clean_html: opts.wants("html"),
134        links: opts.wants("links"),
135        images: opts.wants("images"),
136        // 完整指纹（原始 HTML 的哈希、再渲染一遍纯文本做规范化哈希）约占提取 CPU 的一成，只在变化检测要用时算
137        fingerprints: opts.wants("changeTracking"),
138        ..Default::default()
139    }
140}
141
142/// `includeTags` / `excludeTags` 的筛选规则；都没给时为 `None`，选择器写错时返回 `INVALID_REQUEST`。
143pub fn tag_filter(opts: &ScrapeOptions) -> Result<Option<pagesift_content::TagFilter>, ApiError> {
144    pagesift_content::TagFilter::new(&opts.include_tags, &opts.exclude_tags)
145        .map_err(ApiError::invalid)
146}
147
148pub(super) fn check_barrier(
149    doc: &pagesift_content::Document,
150    raw_html: &str,
151) -> Result<(), ApiError> {
152    let Some(b) = &doc.barrier else { return Ok(()) };
153    // A vendor challenge page can also embed an interactive widget. The
154    // primary rule may classify the page as a challenge, so inspect the raw
155    // document for widgets that need a human before saying it is automatic.
156    let interactive = {
157        let html = raw_html.to_ascii_lowercase();
158        [
159            "g-recaptcha",
160            "h-captcha",
161            "cf-turnstile",
162            "px-captcha",
163            "captcha-delivery",
164            "geetest",
165        ]
166        .iter()
167        .any(|marker| html.contains(marker))
168    };
169    Err(ApiError::new(
170        ErrorCode::AccessBarrier,
171        format!(
172            "the site returned an access barrier ({} {})",
173            b.vendor,
174            b.kind.as_str()
175        ),
176    )
177    .with_details(serde_json::json!({
178        "vendor": b.vendor,
179        "kind": b.kind.as_str(),
180        "confidence": b.confidence,
181        "ruleId": b.rule_id,
182        "needsHuman": interactive || matches!(b.kind, pagesift_content::BarrierKind::Captcha),
183    })))
184}
185
186pub(super) fn document(
187    doc: pagesift_content::Document,
188    opts: &ScrapeOptions,
189    raw_html: String,
190) -> Document {
191    Document {
192        markdown: opts.wants("markdown").then_some(doc.markdown),
193        html: doc.html,
194        raw_html: opts.wants("rawHtml").then_some(raw_html),
195        link_text: doc
196            .links
197            .as_ref()
198            .map(|l| l.iter().map(|l| l.text.clone()).collect()),
199        alternates: doc.metadata.alternates.clone(),
200        links: doc.links.map(|l| l.into_iter().map(|l| l.url).collect()),
201        images: doc.images.map(|i| i.into_iter().map(|i| i.src).collect()),
202        ..Default::default()
203    }
204}
205
206/// 正文 Markdown 的哈希总是给（Markdown 只有几 KB，几乎不花时间）；原始 HTML 和规范化纯文本的
207/// 哈希只在请求变化检测时算（见 `process_html`），它们要整页再过一遍。
208pub(super) fn fill_fingerprints(ext: &mut PagesiftMeta, doc: &pagesift_content::Document) {
209    match &doc.fingerprints {
210        Some(f) => {
211            ext.content_hash = Some(f.markdown.clone());
212            ext.raw_hash = Some(f.raw_html.clone());
213            ext.text_hash = Some(f.text.clone());
214        }
215        None => ext.content_hash = Some(pagesift_content::hash_hex(doc.markdown.as_bytes())),
216    }
217}
218
219/// 页面自带的 JSON-LD 原样透传到 `metadata.pagesift.jsonLd`，供下游直接消费。
220pub(super) fn fill_structured(ext: &mut PagesiftMeta, doc: &pagesift_content::Document) {
221    ext.json_ld = doc.metadata.json_ld().to_vec();
222}
223
224pub(super) use pagesift_sink::meta::fill as fill_metadata;
225
226#[allow(clippy::too_many_arguments)] // 把各阶段的结果汇成文档，只在抓取流程末尾调用一处
227pub(super) fn finish(
228    mut doc: Document,
229    mut meta: Metadata,
230    mut ext: PagesiftMeta,
231    mut timings: BTreeMap<String, f64>,
232    mut warnings: Vec<String>,
233    raw_bytes: Option<usize>,
234    video_fallback: bool,
235    started: Instant,
236) -> Document {
237    timings.insert("total".into(), ms(started.elapsed()));
238    if meta.status_code >= 400 {
239        warnings.push(format!("http_error_status: HTTP {}", meta.status_code));
240    } else if (300..400).contains(&meta.status_code) {
241        // 跟随重定向之后仍是 3xx：没有 Location、Location 不可用，或 300 这类需要人工选择的响应。
242        warnings.push(format!(
243            "http_redirect_status: HTTP {} was not followed (missing or unusable Location)",
244            meta.status_code
245        ));
246    }
247    // 只数到阈值为止：4.49 MB 的正文没必要为了一个警告全量扫一遍。
248    // [策略] 警告门槛（不改结果，只提示 JS 壳页）：先验包 v0：0.81 的现值，约 80 个英文词
249    const THIN_MAX_CHARS: usize = 500;
250    let markdown_chars = doc.markdown.as_deref().map(|markdown| {
251        let mut count = 0;
252        for c in markdown.chars() {
253            if !c.is_whitespace() {
254                count += 1;
255                if count > THIN_MAX_CHARS {
256                    break;
257                }
258            }
259        }
260        count
261    });
262    if meta.status_code < 400 && markdown_chars == Some(0) {
263        warnings.push(format!(
264            "empty_content: no readable text extracted (HTTP {})",
265            meta.status_code
266        ));
267    }
268    // "过薄"与"全空"是两回事：页面有实质内容（HTML 不小）却只抽出几百字，
269    // 多半是 JS 壳，或者正文选择把主体丢了。只警告，不改结果。
270    // [策略] 警告门槛：先验包 v0：0.81 的现值
271    const THIN_MIN_HTML_BYTES: usize = 20_000;
272    if meta.status_code < 400
273        && !video_fallback
274        && let (Some(chars @ 1..THIN_MAX_CHARS), Some(bytes)) = (markdown_chars, raw_bytes)
275        && bytes >= THIN_MIN_HTML_BYTES
276    {
277        warnings.push(format!(
278            "thin_content: extracted {chars} characters from {} KB of HTML; the page may be a JavaScript shell or the main content was not selected",
279            bytes / 1024
280        ));
281    }
282    ext.timings = timings;
283    ext.warnings = warnings;
284    meta.pagesift = Some(ext);
285    doc.metadata = meta;
286    doc
287}
288
289pub(super) fn ms(d: Duration) -> f64 {
290    (d.as_secs_f64() * 1000.0 * 100.0).round() / 100.0
291}
292
293/// 判断 `thin_content` 用的页面体积。正文不薄时直接用 HTML 长度，不多扫一遍；
294/// 正文薄时扣掉 `<style>` 的内容和 `data:` 内嵌资源：一张 8 MB 的 base64 图片
295/// 不代表页面“有大量内容却没抽出来”。`<script>` 不扣：内联脚本大而正文薄正是 JS 壳页的特征。
296pub(super) fn thin_basis_bytes(html: &str, markdown: &str) -> usize {
297    // [策略] 与 THIN_MAX_CHARS 同值：正文不薄时不必扣减
298    const ENOUGH: usize = 500;
299    if markdown
300        .chars()
301        .filter(|c| !c.is_whitespace())
302        .take(ENOUGH)
303        .count()
304        >= ENOUGH
305    {
306        return html.len();
307    }
308    let lower = html.to_ascii_lowercase();
309    let mut removed = 0usize;
310    for tag in ["style"] {
311        let (open, close) = (format!("<{tag}"), format!("</{tag}"));
312        let mut at = 0;
313        while let Some(i) = lower[at..].find(&open) {
314            let start = at + i;
315            let Some(j) = lower[start..].find(&close) else {
316                removed += lower.len() - start;
317                break;
318            };
319            removed += j;
320            at = start + j + close.len();
321        }
322    }
323    let mut at = 0;
324    while let Some(i) = lower[at..].find("data:") {
325        let start = at + i;
326        let len = lower[start..]
327            .find(['"', '\'', ')', ' ', '>'])
328            .unwrap_or(lower.len() - start);
329        removed += len;
330        at = start + len.max(1);
331    }
332    html.len().saturating_sub(removed)
333}
334
335/// JSON 响应（`application/json`、`*+json`）不是网页：当 HTML 解析只会剩下碎片。
336/// 合法 JSON 格式化后放进代码块；太大的不格式化，原样放进去。不是合法 JSON 时返回 `None`，按文本继续。
337pub(super) fn json_markdown(content_type: Option<&str>, body: &[u8]) -> Option<String> {
338    // [策略] 安全上限：格式化会把 JSON 放大数倍（缩进、换行），超过它原样输出，防大响应撑爆内存
339    const PRETTY_MAX_BYTES: usize = 5 * 1024 * 1024;
340    let kind = content_type?.split(';').next()?.trim().to_ascii_lowercase();
341    if kind != "application/json" && kind != "text/json" && !kind.ends_with("+json") {
342        return None;
343    }
344    let text = std::str::from_utf8(body.strip_prefix(b"\xef\xbb\xbf").unwrap_or(body)).ok()?;
345    let value: serde_json::Value = serde_json::from_str(text).ok()?;
346    let rendered = if text.len() <= PRETTY_MAX_BYTES {
347        serde_json::to_string_pretty(&value).ok()?
348    } else {
349        text.trim().to_string()
350    };
351    // 内容里有三个反引号时加长围栏，保证代码块不被提前闭合。
352    let mut fence = String::from("```");
353    while rendered.contains(fence.as_str()) {
354        fence.push('`');
355    }
356    Some(format!("{fence}json\n{rendered}\n{fence}\n"))
357}
358
359/// 魔数优先：`%PDF-` 开头就是 PDF（不管头怎么写）。只有头说是 PDF 时，正文明显是 HTML/XML
360/// 的不算——CDN、WAF 把拦截页、登录页标成 `application/pdf` 返回时按网页处理，拦截页识别照常生效。
361pub(super) fn is_pdf(body: &[u8], content_type: Option<&str>) -> bool {
362    body.starts_with(b"%PDF-")
363        || (content_type.is_some_and(|c| c.to_ascii_lowercase().contains("application/pdf"))
364            && !looks_like_markup(body))
365}
366
367/// 非 HTML/文本内容（图片、音视频、字体、压缩包等）不应该被当作文本抽取。
368/// 返回不支持的类型名；`None` 表示可以按文本继续处理。
369///
370/// Content-Type 可能标错（例如 `application/octet-stream` 实际是 HTML），所以明显的
371/// HTML/XML 开头仍然放行；图片/音视频/字体等即使正文碰巧以 `<` 开头也拒绝。
372pub(super) fn unsupported_content_type(content_type: Option<&str>, body: &[u8]) -> Option<String> {
373    let raw = content_type?;
374    let kind = raw.split(';').next()?.trim().to_ascii_lowercase();
375    if kind.is_empty() {
376        return None;
377    }
378    if kind.starts_with("image/")
379        || kind.starts_with("audio/")
380        || kind.starts_with("video/")
381        || kind.starts_with("font/")
382    {
383        return Some(kind);
384    }
385    // `application/octet-stream`、`binary/octet-stream` 返回的 PDF 很常见：认魔数，交给 PDF 解析。
386    if kind == "application/pdf" || body.starts_with(b"%PDF-") {
387        return None;
388    }
389    if kind.starts_with("text/")
390        || kind.contains("html")
391        || kind.contains("xml")
392        || kind.contains("json")
393        || matches!(
394            kind.as_str(),
395            "application/javascript" | "application/x-javascript" | "application/ecmascript"
396        )
397    {
398        return None;
399    }
400    if looks_like_markup(body) {
401        return None;
402    }
403    Some(kind)
404}
405
406fn looks_like_markup(body: &[u8]) -> bool {
407    let head = &body[..body.len().min(1024)];
408    let text = String::from_utf8_lossy(head);
409    let text = text.trim_start_matches('\u{feff}').trim_start();
410    let lower = text.to_ascii_lowercase();
411    lower.starts_with("<!doctype html")
412        || lower.starts_with("<html")
413        || lower.starts_with("<head")
414        || lower.starts_with("<body")
415        || lower.starts_with("<?xml")
416}
417
418pub fn policy_error(e: pagesift_policy::Error) -> ApiError {
419    use pagesift_policy::ErrorCode as P;
420    let code = match e.code() {
421        P::RobotsDisallowed => ErrorCode::RobotsDisallowed,
422        P::AccessBarrier => ErrorCode::AccessBarrier,
423        P::RateLimited => ErrorCode::RateLimited,
424        P::HostCooldown => ErrorCode::HostCooldown,
425        P::Timeout => ErrorCode::Timeout,
426        P::SsrfBlocked => ErrorCode::SsrfBlocked,
427        P::TooLarge => ErrorCode::TooLarge,
428        P::UpstreamError => ErrorCode::UpstreamError,
429        P::UnsupportedOption => ErrorCode::UnsupportedOption,
430        P::InvalidRequest => ErrorCode::InvalidRequest,
431        P::DnsFailure => ErrorCode::DnsFailure,
432        P::NetworkError => ErrorCode::NetworkError,
433        P::TlsCertExpired => ErrorCode::TlsCertExpired,
434        P::TlsHostnameMismatch => ErrorCode::TlsHostnameMismatch,
435        P::TlsUntrustedRoot => ErrorCode::TlsUntrustedRoot,
436        P::TlsError => ErrorCode::TlsError,
437        _ => ErrorCode::Internal,
438    };
439    let retry = e.retry_after().map(|d| d.as_secs().max(1));
440    let mut err = ApiError::new(code, e.to_string());
441    if let Some(s) = retry {
442        err = err.with_retry_after(s);
443    }
444    err
445}
446
447#[cfg(test)]
448mod tests {
449    use super::{is_pdf, json_markdown, thin_basis_bytes, unsupported_content_type};
450
451    #[test]
452    fn thin_basis_ignores_inline_payloads() {
453        let image = format!(
454            "<p>short</p><img src=\"data:image/png;base64,{}\">",
455            "A".repeat(100_000)
456        );
457        assert!(thin_basis_bytes(&image, "short") < 1_000);
458        let style = format!("<STYLE>{}</STYLE><p>x</p>", "y".repeat(50_000));
459        assert!(thin_basis_bytes(&style, "x") < 1_000);
460        let next = format!(
461            "<script id=\"__NEXT_DATA__\">{}</script><div id=\"app\"></div>",
462            "x".repeat(50_000)
463        );
464        assert_eq!(thin_basis_bytes(&next, ""), next.len());
465        let shell = format!("<div>{}</div>", "<span></span>".repeat(5_000));
466        assert_eq!(thin_basis_bytes(&shell, ""), shell.len());
467        let long = "word ".repeat(200);
468        assert_eq!(thin_basis_bytes(&image, &long), image.len());
469    }
470
471    #[test]
472    fn json_responses_become_code_blocks() {
473        let md = json_markdown(
474            Some("application/json; charset=utf-8"),
475            br#"{"html":"<h1>x</h1>","n":[1,2]}"#,
476        )
477        .unwrap();
478        assert!(md.starts_with("```json\n{"), "{md}");
479        assert!(md.contains("\"html\": \"<h1>x</h1>\""), "{md}");
480        assert!(md.ends_with("\n```\n"), "{md}");
481        let md = json_markdown(Some("application/ld+json"), br#"{"a":"```"}"#).unwrap();
482        assert!(md.starts_with("````json"), "{md}");
483        assert_eq!(json_markdown(Some("text/html"), b"{}"), None);
484        assert_eq!(json_markdown(Some("application/json"), b"<html>"), None);
485    }
486
487    #[test]
488    fn rejects_binary_content_types() {
489        for kind in [
490            "image/png",
491            "image/jpeg",
492            "image/svg+xml",
493            "audio/mpeg",
494            "video/mp4",
495            "font/woff2",
496            "application/zip",
497            "application/octet-stream",
498        ] {
499            assert_eq!(
500                unsupported_content_type(Some(kind), b"\x89PNG\r\n"),
501                Some(kind.to_string()),
502                "{kind}"
503            );
504        }
505    }
506
507    #[test]
508    fn accepts_text_and_mislabeled_markup() {
509        for kind in [
510            "text/html; charset=utf-8",
511            "text/plain",
512            "application/xhtml+xml",
513            "application/json",
514            "application/rss+xml",
515            "application/pdf",
516        ] {
517            assert_eq!(
518                unsupported_content_type(Some(kind), b"anything"),
519                None,
520                "{kind}"
521            );
522        }
523        assert_eq!(
524            unsupported_content_type(Some("application/octet-stream"), b"<!doctype html><p>x"),
525            None
526        );
527        assert_eq!(
528            unsupported_content_type(Some("application/octet-stream"), b"  <html><p>x"),
529            None
530        );
531        assert_eq!(unsupported_content_type(None, b"\x89PNG"), None);
532        for kind in ["application/octet-stream", "binary/octet-stream"] {
533            assert_eq!(
534                unsupported_content_type(Some(kind), b"%PDF-1.7\n1 0 obj"),
535                None
536            );
537            assert!(is_pdf(b"%PDF-1.7\n1 0 obj", Some(kind)));
538        }
539        // 头说是 PDF：正文是网页（拦截页、登录页）时按网页处理；不是网页的照样交给 PDF 解析
540        let pdf = Some("application/pdf");
541        assert!(!is_pdf(
542            b"<!DOCTYPE html><html><body>Access denied</body></html>",
543            pdf
544        ));
545        assert!(!is_pdf(b"\n  <html><body>x</body></html>", pdf));
546        assert!(is_pdf(b"junk before the header %PDF-1.4", pdf));
547        assert!(is_pdf(b"%PDF-1.4", Some("text/html")));
548        assert_eq!(
549            unsupported_content_type(Some("application/octet-stream"), b"\x00\x01binary"),
550            Some("application/octet-stream".into())
551        );
552    }
553}
pagesift_engine/engine/output.rs

pagesift_engine/engine/
output.rs
