pagesift_engine/engine/
output.rs

1use super::*;
2
3pub(super) fn fetch_request(url: &str, opts: &ScrapeOptions) -> Result<FetchRequest, ApiError> {
4    let mut req = FetchRequest::new(url);
5    for (k, v) in &opts.headers {
6        req = req.header(k.clone(), v.clone());
7    }
8    if let Some(etag) = &opts.if_none_match {
9        req = req.if_none_match(etag.clone());
10    }
11    if let Some(t) = &opts.if_modified_since {
12        let t = httpdate::parse_http_date(t).map_err(|_| {
13            ApiError::invalid(format!("ifModifiedSince is not an HTTP date: {t:?}"))
14        })?;
15        req = req.if_modified_since(t);
16    }
17    req.timeout = opts.timeout.map(Duration::from_millis);
18    Ok(req)
19}
20
21/// `chunking`：正文 Markdown 分块。参数格式见 [`pagesift_extract::ChunkStrategy`]（`maxLevel` 这类驼峰写法也认）。
22pub fn chunk_markdown(
23    md: &str,
24    spec: &serde_json::Value,
25) -> Result<Vec<pagesift_api::types::DocChunk>, ApiError> {
26    // API 用驼峰，提取库用下划线：只改键名
27    let snake = match spec {
28        serde_json::Value::Object(m) => serde_json::Value::Object(
29            m.iter()
30                .map(|(k, v)| {
31                    let mut out = String::new();
32                    for c in k.chars() {
33                        if c.is_ascii_uppercase() {
34                            out.push('_');
35                            out.push(c.to_ascii_lowercase());
36                        } else {
37                            out.push(c);
38                        }
39                    }
40                    (out, v.clone())
41                })
42                .collect(),
43        ),
44        other => other.clone(),
45    };
46    let strategy: pagesift_extract::ChunkStrategy = serde_json::from_value(snake)
47        .map_err(|e| ApiError::invalid(format!("invalid chunking: {e}")))?;
48    let chunks = pagesift_extract::chunk(md, &strategy)
49        .map_err(|e| ApiError::invalid(format!("invalid chunking: {e}")))?;
50    Ok(chunks
51        .into_iter()
52        .map(|c| pagesift_api::types::DocChunk {
53            text: c.text,
54            heading_path: c.heading_path,
55            start: c.start,
56            end: c.end,
57            tokens: c.tokens,
58        })
59        .collect())
60}
61
62/// `queryFilter`：只留下和查询相关的内容，统计写进 `metadata.pagesift.queryFilter`。
63pub(super) async fn apply_query_filter(
64    doc: &mut Document,
65    q: &pagesift_api::types::QueryFilter,
66) -> Result<(), ApiError> {
67    let Some(md) = doc.markdown.take() else {
68        return Ok(());
69    };
70    let fo = pagesift_extract::QueryFilterOptions {
71        threshold: q.threshold,
72        relative: q.relative,
73        top_k: q.top_k,
74        max_tokens: q.max_tokens,
75        min_tokens: q.min_tokens,
76        headings: q.headings,
77    };
78    let query = q.query.clone();
79    let (f, before) = cpu_task(move || {
80        let f = pagesift_extract::filter_markdown_by_query(&md, &query, &fo);
81        (f, pagesift_content::estimate_tokens(&md))
82    })
83    .await?;
84    let ext = doc.metadata.pagesift.get_or_insert_with(Default::default);
85    if f.kept == 0 && f.total > 0 {
86        ext.warnings.push("query_filter_no_match".into());
87    }
88    ext.query_filter = Some(pagesift_api::types::QueryFilterStats {
89        kept_blocks: f.kept,
90        total_blocks: f.total,
91        tokens_before: before,
92        tokens_after: pagesift_content::estimate_tokens(&f.markdown),
93    });
94    doc.markdown = Some(f.markdown);
95    Ok(())
96}
97
98pub(super) async fn process_html(
99    html: String,
100    base: &Url,
101    status: Option<u16>,
102    opts: &ScrapeOptions,
103) -> Result<(pagesift_content::Document, String), ApiError> {
104    let copts = content_options(base, status, opts);
105    let tags = tag_filter(opts)?;
106    cpu_task(move || {
107        // RSS / Atom / RDF feed 不是网页：按条目解析，别让 HTML 提取器把标签剥成一锅粥。
108        if let Some(document) = super::feed::parse(&html, copts.links) {
109            return (document, html);
110        }
111        // includeTags / excludeTags 只影响提取；rawHtml 仍是原始页面
112        let filtered = tags.map(|t| t.apply(&html));
113        let source = filtered.as_deref().unwrap_or(&html);
114        let document = CONTENT_WORKSPACE.with(|workspace| {
115            pagesift_content::process_with_workspace(source, &copts, &mut workspace.borrow_mut())
116        });
117        (document, html)
118    })
119    .await
120}
121
122/// 提取参数：只按请求要的格式算（链接、图片、清洗后的 HTML、变化检测的指纹）。
123pub(super) fn content_options(
124    base: &Url,
125    status: Option<u16>,
126    opts: &ScrapeOptions,
127) -> pagesift_content::Options {
128    pagesift_content::Options {
129        base_url: Some(base.clone()),
130        only_main_content: opts.only_main_content,
131        keep_data_images: !opts.remove_base64_images,
132        status,
133        detect_barrier: true,
134        clean_html: opts.wants("html"),
135        links: opts.wants("links"),
136        images: opts.wants("images"),
137        // 完整指纹（原始 HTML 的哈希、再渲染一遍纯文本做规范化哈希）约占提取 CPU 的一成，只在变化检测要用时算
138        fingerprints: opts.wants("changeTracking"),
139        ..Default::default()
140    }
141}
142
143/// `includeTags` / `excludeTags` 的筛选规则；都没给时为 `None`，选择器写错时返回 `INVALID_REQUEST`。
144pub fn tag_filter(opts: &ScrapeOptions) -> Result<Option<pagesift_content::TagFilter>, ApiError> {
145    pagesift_content::TagFilter::new(&opts.include_tags, &opts.exclude_tags)
146        .map_err(ApiError::invalid)
147}
148
149pub(super) fn check_barrier(
150    doc: &pagesift_content::Document,
151    raw_html: &str,
152) -> Result<(), ApiError> {
153    let Some(b) = &doc.barrier else { return Ok(()) };
154    // A vendor challenge page can also embed an interactive widget. The
155    // primary rule may classify the page as a challenge, so inspect the raw
156    // document for widgets that need a human before saying it is automatic.
157    let interactive = {
158        let html = raw_html.to_ascii_lowercase();
159        [
160            "g-recaptcha",
161            "h-captcha",
162            "cf-turnstile",
163            "px-captcha",
164            "captcha-delivery",
165            "geetest",
166        ]
167        .iter()
168        .any(|marker| html.contains(marker))
169    };
170    Err(ApiError::new(
171        ErrorCode::AccessBarrier,
172        format!(
173            "the site returned an access barrier ({} {})",
174            b.vendor,
175            b.kind.as_str()
176        ),
177    )
178    .with_details(serde_json::json!({
179        "vendor": b.vendor,
180        "kind": b.kind.as_str(),
181        "confidence": b.confidence,
182        "ruleId": b.rule_id,
183        "needsHuman": interactive || matches!(b.kind, pagesift_content::BarrierKind::Captcha),
184    })))
185}
186
187pub(super) fn document(
188    doc: pagesift_content::Document,
189    opts: &ScrapeOptions,
190    raw_html: String,
191) -> Document {
192    Document {
193        markdown: opts.wants("markdown").then_some(doc.markdown),
194        html: doc.html,
195        raw_html: opts.wants("rawHtml").then_some(raw_html),
196        link_text: doc
197            .links
198            .as_ref()
199            .map(|l| l.iter().map(|l| l.text.clone()).collect()),
200        alternates: doc.metadata.alternates.clone(),
201        links: doc.links.map(|l| l.into_iter().map(|l| l.url).collect()),
202        images: doc.images.map(|i| i.into_iter().map(|i| i.src).collect()),
203        ..Default::default()
204    }
205}
206
207/// 正文 Markdown 的哈希总是给（Markdown 只有几 KB，几乎不花时间）；原始 HTML 和规范化纯文本的
208/// 哈希只在请求变化检测时算（见 `process_html`），它们要整页再过一遍。
209pub(super) fn fill_fingerprints(ext: &mut PagesiftMeta, doc: &pagesift_content::Document) {
210    match &doc.fingerprints {
211        Some(f) => {
212            ext.content_hash = Some(f.markdown.clone());
213            ext.raw_hash = Some(f.raw_html.clone());
214            ext.text_hash = Some(f.text.clone());
215        }
216        None => ext.content_hash = Some(pagesift_content::hash_hex(doc.markdown.as_bytes())),
217    }
218}
219
220/// 页面自带的 JSON-LD 原样透传到 `metadata.pagesift.jsonLd`，供下游直接消费。
221pub(super) fn fill_structured(ext: &mut PagesiftMeta, doc: &pagesift_content::Document) {
222    ext.json_ld = doc.metadata.json_ld().to_vec();
223}
224
225pub(super) use pagesift_sink::meta::fill as fill_metadata;
226
227#[allow(clippy::too_many_arguments)] // 把各阶段的结果汇成文档，只在抓取流程末尾调用一处
228pub(super) fn finish(
229    mut doc: Document,
230    mut meta: Metadata,
231    mut ext: PagesiftMeta,
232    mut timings: BTreeMap<String, f64>,
233    mut warnings: Vec<String>,
234    raw_bytes: Option<usize>,
235    video_fallback: bool,
236    started: Instant,
237) -> Document {
238    timings.insert("total".into(), ms(started.elapsed()));
239    if meta.status_code >= 400 {
240        warnings.push(format!("http_error_status: HTTP {}", meta.status_code));
241    } else if (300..400).contains(&meta.status_code) {
242        // 跟随重定向之后仍是 3xx：没有 Location、Location 不可用，或 300 这类需要人工选择的响应。
243        warnings.push(format!(
244            "http_redirect_status: HTTP {} was not followed (missing or unusable Location)",
245            meta.status_code
246        ));
247    }
248    // 只数到阈值为止：4.49 MB 的正文没必要为了一个警告全量扫一遍。
249    // [策略] 警告门槛（不改结果，只提示 JS 壳页）：先验包 v0：0.81 的现值，约 80 个英文词
250    const THIN_MAX_CHARS: usize = 500;
251    let markdown_chars = doc.markdown.as_deref().map(|markdown| {
252        let mut count = 0;
253        for c in markdown.chars() {
254            if !c.is_whitespace() {
255                count += 1;
256                if count > THIN_MAX_CHARS {
257                    break;
258                }
259            }
260        }
261        count
262    });
263    if meta.status_code < 400 && markdown_chars == Some(0) {
264        warnings.push(format!(
265            "empty_content: no readable text extracted (HTTP {})",
266            meta.status_code
267        ));
268    }
269    // "过薄"与"全空"是两回事：页面有实质内容（HTML 不小）却只抽出几百字，
270    // 多半是 JS 壳，或者正文选择把主体丢了。只警告，不改结果。
271    // [策略] 警告门槛：先验包 v0：0.81 的现值
272    const THIN_MIN_HTML_BYTES: usize = 20_000;
273    if meta.status_code < 400
274        && !video_fallback
275        && let (Some(chars @ 1..THIN_MAX_CHARS), Some(bytes)) = (markdown_chars, raw_bytes)
276        && bytes >= THIN_MIN_HTML_BYTES
277    {
278        warnings.push(format!(
279            "thin_content: extracted {chars} characters from {} KB of HTML; the page may be a JavaScript shell or the main content was not selected",
280            bytes / 1024
281        ));
282    }
283    ext.timings = timings;
284    ext.warnings = warnings;
285    meta.pagesift = Some(ext);
286    doc.metadata = meta;
287    doc
288}
289
290pub(super) fn ms(d: Duration) -> f64 {
291    (d.as_secs_f64() * 1000.0 * 100.0).round() / 100.0
292}
293
294/// 判断 `thin_content` 用的页面体积。正文不薄时直接用 HTML 长度，不多扫一遍；
295/// 正文薄时扣掉 `<style>` 的内容和 `data:` 内嵌资源：一张 8 MB 的 base64 图片
296/// 不代表页面“有大量内容却没抽出来”。`<script>` 不扣：内联脚本大而正文薄正是 JS 壳页的特征。
297pub(super) fn thin_basis_bytes(html: &str, markdown: &str) -> usize {
298    // [策略] 与 THIN_MAX_CHARS 同值：正文不薄时不必扣减
299    const ENOUGH: usize = 500;
300    if markdown
301        .chars()
302        .filter(|c| !c.is_whitespace())
303        .take(ENOUGH)
304        .count()
305        >= ENOUGH
306    {
307        return html.len();
308    }
309    let lower = html.to_ascii_lowercase();
310    let mut removed = 0usize;
311    for tag in ["style"] {
312        let (open, close) = (format!("<{tag}"), format!("</{tag}"));
313        let mut at = 0;
314        while let Some(i) = lower[at..].find(&open) {
315            let start = at + i;
316            let Some(j) = lower[start..].find(&close) else {
317                removed += lower.len() - start;
318                break;
319            };
320            removed += j;
321            at = start + j + close.len();
322        }
323    }
324    let mut at = 0;
325    while let Some(i) = lower[at..].find("data:") {
326        let start = at + i;
327        let len = lower[start..]
328            .find(['"', '\'', ')', ' ', '>'])
329            .unwrap_or(lower.len() - start);
330        removed += len;
331        at = start + len.max(1);
332    }
333    html.len().saturating_sub(removed)
334}
335
336/// JSON 响应（`application/json`、`*+json`）不是网页：当 HTML 解析只会剩下碎片。
337/// 合法 JSON 格式化后放进代码块；太大的不格式化，原样放进去。不是合法 JSON 时返回 `None`，按文本继续。
338pub(super) fn json_markdown(content_type: Option<&str>, body: &[u8]) -> Option<String> {
339    // [策略] 安全上限：格式化会把 JSON 放大数倍（缩进、换行），超过它原样输出，防大响应撑爆内存
340    const PRETTY_MAX_BYTES: usize = 5 * 1024 * 1024;
341    let kind = content_type?.split(';').next()?.trim().to_ascii_lowercase();
342    if kind != "application/json" && kind != "text/json" && !kind.ends_with("+json") {
343        return None;
344    }
345    let text = std::str::from_utf8(body.strip_prefix(b"\xef\xbb\xbf").unwrap_or(body)).ok()?;
346    let value: serde_json::Value = serde_json::from_str(text).ok()?;
347    let rendered = if text.len() <= PRETTY_MAX_BYTES {
348        serde_json::to_string_pretty(&value).ok()?
349    } else {
350        text.trim().to_string()
351    };
352    // 内容里有三个反引号时加长围栏，保证代码块不被提前闭合。
353    let mut fence = String::from("```");
354    while rendered.contains(fence.as_str()) {
355        fence.push('`');
356    }
357    Some(format!("{fence}json\n{rendered}\n{fence}\n"))
358}
359
360/// 魔数优先：`%PDF-` 开头就是 PDF（不管头怎么写）。只有头说是 PDF 时，正文明显是 HTML/XML
361/// 的不算——CDN、WAF 把拦截页、登录页标成 `application/pdf` 返回时按网页处理，拦截页识别照常生效。
362pub(super) fn is_pdf(body: &[u8], content_type: Option<&str>) -> bool {
363    body.starts_with(b"%PDF-")
364        || (content_type.is_some_and(|c| c.to_ascii_lowercase().contains("application/pdf"))
365            && !looks_like_markup(body))
366}
367
368/// 非 HTML/文本内容（图片、音视频、字体、压缩包等）不应该被当作文本抽取。
369/// 返回不支持的类型名；`None` 表示可以按文本继续处理。
370///
371/// Content-Type 可能标错（例如 `application/octet-stream` 实际是 HTML），所以明显的
372/// HTML/XML 开头仍然放行；图片/音视频/字体等即使正文碰巧以 `<` 开头也拒绝。
373pub(super) fn unsupported_content_type(content_type: Option<&str>, body: &[u8]) -> Option<String> {
374    let raw = content_type?;
375    let kind = raw.split(';').next()?.trim().to_ascii_lowercase();
376    if kind.is_empty() {
377        return None;
378    }
379    if kind.starts_with("image/")
380        || kind.starts_with("audio/")
381        || kind.starts_with("video/")
382        || kind.starts_with("font/")
383    {
384        return Some(kind);
385    }
386    // `application/octet-stream`、`binary/octet-stream` 返回的 PDF 很常见：认魔数，交给 PDF 解析。
387    if kind == "application/pdf" || body.starts_with(b"%PDF-") {
388        return None;
389    }
390    if kind.starts_with("text/")
391        || kind.contains("html")
392        || kind.contains("xml")
393        || kind.contains("json")
394        || matches!(
395            kind.as_str(),
396            "application/javascript" | "application/x-javascript" | "application/ecmascript"
397        )
398    {
399        return None;
400    }
401    if looks_like_markup(body) {
402        return None;
403    }
404    Some(kind)
405}
406
407fn looks_like_markup(body: &[u8]) -> bool {
408    let head = &body[..body.len().min(1024)];
409    let text = String::from_utf8_lossy(head);
410    let text = text.trim_start_matches('\u{feff}').trim_start();
411    let lower = text.to_ascii_lowercase();
412    lower.starts_with("<!doctype html")
413        || lower.starts_with("<html")
414        || lower.starts_with("<head")
415        || lower.starts_with("<body")
416        || lower.starts_with("<?xml")
417}
418
419pub fn policy_error(e: pagesift_policy::Error) -> ApiError {
420    use pagesift_policy::ErrorCode as P;
421    let code = match e.code() {
422        P::RobotsDisallowed => ErrorCode::RobotsDisallowed,
423        P::AccessBarrier => ErrorCode::AccessBarrier,
424        P::RateLimited => ErrorCode::RateLimited,
425        P::HostCooldown => ErrorCode::HostCooldown,
426        P::Timeout => ErrorCode::Timeout,
427        P::SsrfBlocked => ErrorCode::SsrfBlocked,
428        P::TooLarge => ErrorCode::TooLarge,
429        P::UpstreamError => ErrorCode::UpstreamError,
430        P::UnsupportedOption => ErrorCode::UnsupportedOption,
431        P::InvalidRequest => ErrorCode::InvalidRequest,
432        P::DnsFailure => ErrorCode::DnsFailure,
433        P::NetworkError => ErrorCode::NetworkError,
434        P::TlsCertExpired => ErrorCode::TlsCertExpired,
435        P::TlsHostnameMismatch => ErrorCode::TlsHostnameMismatch,
436        P::TlsUntrustedRoot => ErrorCode::TlsUntrustedRoot,
437        P::TlsError => ErrorCode::TlsError,
438        _ => ErrorCode::Internal,
439    };
440    let retry = e.retry_after().map(|d| d.as_secs().max(1));
441    let mut err = ApiError::new(code, e.to_string());
442    if let Some(s) = retry {
443        err = err.with_retry_after(s);
444    }
445    err
446}
447
448#[cfg(test)]
449mod tests {
450    use super::{is_pdf, json_markdown, thin_basis_bytes, unsupported_content_type};
451
452    #[test]
453    fn thin_basis_ignores_inline_payloads() {
454        let image = format!(
455            "<p>short</p><img src=\"data:image/png;base64,{}\">",
456            "A".repeat(100_000)
457        );
458        assert!(thin_basis_bytes(&image, "short") < 1_000);
459        let style = format!("<STYLE>{}</STYLE><p>x</p>", "y".repeat(50_000));
460        assert!(thin_basis_bytes(&style, "x") < 1_000);
461        let next = format!(
462            "<script id=\"__NEXT_DATA__\">{}</script><div id=\"app\"></div>",
463            "x".repeat(50_000)
464        );
465        assert_eq!(thin_basis_bytes(&next, ""), next.len());
466        let shell = format!("<div>{}</div>", "<span></span>".repeat(5_000));
467        assert_eq!(thin_basis_bytes(&shell, ""), shell.len());
468        let long = "word ".repeat(200);
469        assert_eq!(thin_basis_bytes(&image, &long), image.len());
470    }
471
472    #[test]
473    fn json_responses_become_code_blocks() {
474        let md = json_markdown(
475            Some("application/json; charset=utf-8"),
476            br#"{"html":"<h1>x</h1>","n":[1,2]}"#,
477        )
478        .unwrap();
479        assert!(md.starts_with("```json\n{"), "{md}");
480        assert!(md.contains("\"html\": \"<h1>x</h1>\""), "{md}");
481        assert!(md.ends_with("\n```\n"), "{md}");
482        let md = json_markdown(Some("application/ld+json"), br#"{"a":"```"}"#).unwrap();
483        assert!(md.starts_with("````json"), "{md}");
484        assert_eq!(json_markdown(Some("text/html"), b"{}"), None);
485        assert_eq!(json_markdown(Some("application/json"), b"<html>"), None);
486    }
487
488    #[test]
489    fn rejects_binary_content_types() {
490        for kind in [
491            "image/png",
492            "image/jpeg",
493            "image/svg+xml",
494            "audio/mpeg",
495            "video/mp4",
496            "font/woff2",
497            "application/zip",
498            "application/octet-stream",
499        ] {
500            assert_eq!(
501                unsupported_content_type(Some(kind), b"\x89PNG\r\n"),
502                Some(kind.to_string()),
503                "{kind}"
504            );
505        }
506    }
507
508    #[test]
509    fn accepts_text_and_mislabeled_markup() {
510        for kind in [
511            "text/html; charset=utf-8",
512            "text/plain",
513            "application/xhtml+xml",
514            "application/json",
515            "application/rss+xml",
516            "application/pdf",
517        ] {
518            assert_eq!(
519                unsupported_content_type(Some(kind), b"anything"),
520                None,
521                "{kind}"
522            );
523        }
524        assert_eq!(
525            unsupported_content_type(Some("application/octet-stream"), b"<!doctype html><p>x"),
526            None
527        );
528        assert_eq!(
529            unsupported_content_type(Some("application/octet-stream"), b"  <html><p>x"),
530            None
531        );
532        assert_eq!(unsupported_content_type(None, b"\x89PNG"), None);
533        for kind in ["application/octet-stream", "binary/octet-stream"] {
534            assert_eq!(
535                unsupported_content_type(Some(kind), b"%PDF-1.7\n1 0 obj"),
536                None
537            );
538            assert!(is_pdf(b"%PDF-1.7\n1 0 obj", Some(kind)));
539        }
540        // 头说是 PDF：正文是网页（拦截页、登录页）时按网页处理；不是网页的照样交给 PDF 解析
541        let pdf = Some("application/pdf");
542        assert!(!is_pdf(
543            b"<!DOCTYPE html><html><body>Access denied</body></html>",
544            pdf
545        ));
546        assert!(!is_pdf(b"\n  <html><body>x</body></html>", pdf));
547        assert!(is_pdf(b"junk before the header %PDF-1.4", pdf));
548        assert!(is_pdf(b"%PDF-1.4", Some("text/html")));
549        assert_eq!(
550            unsupported_content_type(Some("application/octet-stream"), b"\x00\x01binary"),
551            Some("application/octet-stream".into())
552        );
553    }
554}
pagesift_engine/engine/output.rs

pagesift_engine/engine/
output.rs
