pagesift_engine/engine/
mod.rs

1//! 抓取引擎：把网络层、礼貌策略、正文提取、PDF 和浏览器渲染串成一次完整的抓取。
2//!
3//! 流程：robots 模式 → HTTP 抓取（策略层负责冷却、robots、限速）→ 按内容类型分流（PDF / HTML）
4//! → `engine: auto` 时正文太少就升级到浏览器渲染 → 组装成 Firecrawl 格式的文档。
5
6use std::collections::BTreeMap;
7use std::sync::Arc;
8use std::time::{Duration, Instant};
9
10use pagesift_api::types::{Document, Metadata, PagesiftMeta, ScrapeOptions};
11use pagesift_api::{ApiError, BoxFuture, Engine, ErrorCode};
12use pagesift_cdp::{
13    AllowListGuard, BlockResources, BrowserPool, NetworkAllowList, PoolConfig, PrivateNetworkGuard,
14    RenderOptions, RenderedPage, RequestGuard, SubRequest, Verdict,
15};
16use pagesift_net::{FetchRequest, Fetcher, FetcherConfig, Url};
17use pagesift_policy::{Policy, PolicyConfig, RequestKind, RobotsMode};
18use tokio::sync::OnceCell;
19
20tokio::task_local! {
21    static ITNA_NETWORK_BUDGET: Arc<pagesift_itna::NetworkBudget>;
22    static ITNA_COOKIE_SESSION: Option<Arc<pagesift_net::CookieSession>>;
23    static ITNA_EXPECTED_DIRECT: Option<bool>;
24    static ITNA_DEFAULT_PROXY: Option<String>;
25}
26
27pub mod agent;
28mod asset_modes;
29mod autoschema;
30mod cache;
31mod cpu;
32mod csv_text;
33mod egress_route;
34mod extract;
35mod feed;
36mod fetch;
37mod frames;
38mod hooked;
39mod http_extract;
40mod image_file;
41mod images;
42mod interact;
43mod office;
44mod output;
45mod parse;
46mod pdf;
47pub mod planner;
48mod render;
49mod render_stats;
50mod scraper;
51mod speculate;
52pub use autoschema::ActionSecret;
53pub use hooked::ScraperBuilder;
54pub use interact::InteractLimits;
55mod similar;
56mod structured;
57use cpu::{CONTENT_WORKSPACE, cpu_task};
58pub use cpu::{set_cpu_capacity, set_cpu_groups};
59use extract::*;
60use frames::*;
61use http_extract::*;
62use office::*;
63pub use office::{office_markdown, office_markdown_from};
64pub use output::*;
65use render::*;
66
67/// F-1.9：HTTP 抓到的正文少于这么多（加权）字符时，`engine: auto` 升级到浏览器渲染。
68/// 中日韩文字一个字按 [`CJK_WEIGHT`] 个字符算（信息量和几个拉丁字母相当），见 [`text_weight`]。
69pub const AUTO_RENDER_MIN_CHARS: usize = 1200;
70/// 已经认出是一篇文章（有发布时间或 `og:type=article`）时，正文到这么多加权字符就不再渲染：
71/// 短新闻 HTTP 已经拿全了。AiSeek 的生产数据里，63% 的渲染拿到的文字和 HTTP 一样多，
72/// 多半就是这类短文章。
73pub const AUTO_RENDER_ARTICLE_MIN_CHARS: usize = 300;
74pub const CJK_WEIGHT: usize = 3;
75
76/// 正文的加权长度：空白不算，中日韩文字按 [`CJK_WEIGHT`] 算。
77pub fn text_weight(s: &str) -> usize {
78    s.chars()
79        .filter(|c| !c.is_whitespace())
80        .map(|c| if is_cjk(c) { CJK_WEIGHT } else { 1 })
81        .sum()
82}
83
84fn is_cjk(c: char) -> bool {
85    matches!(c as u32,
86        0x3040..=0x30FF   // 平假名、片假名
87        | 0x3400..=0x4DBF // CJK 扩展 A
88        | 0x4E00..=0x9FFF // CJK 统一汉字
89        | 0xAC00..=0xD7AF // 谚文音节
90        | 0xF900..=0xFAFF // CJK 兼容汉字
91        | 0x20000..=0x2FA1F)
92}
93
94/// `engine: auto` 时 HTTP 的结果是否还要用浏览器渲染。
95pub fn needs_render(doc: &pagesift_content::Document) -> bool {
96    let w = text_weight(&doc.markdown);
97    if w >= AUTO_RENDER_MIN_CHARS {
98        return false;
99    }
100    let m = &doc.metadata;
101    let article = m.published_time.is_some()
102        || m.extra
103            .iter()
104            .any(|(k, v)| k == "og:type" && v.eq_ignore_ascii_case("article"));
105    !(article && w >= AUTO_RENDER_ARTICLE_MIN_CHARS)
106}
107/// 浏览器渲染出的正文少于这么多字符时，换下一个内核再试（Lightpanda 的脚本支持不全，
108/// 单页应用可能渲染不出内容）。最后一个内核的结果照常返回。
109pub const RENDER_FALLBACK_MIN_CHARS: usize = 200;
110
111/// 内核链里一个内核的启动检查结果（[`Scraper::probe_engines`]）。
112#[derive(Debug, Clone)]
113pub struct EngineProbe {
114    pub engine: &'static str,
115    /// 可执行文件路径或远程地址。
116    pub location: String,
117    pub result: Result<(), String>,
118}
119
120#[derive(Clone, Default)]
121pub struct ScraperConfig {
122    pub fetcher: FetcherConfig,
123    pub policy: PolicyConfig,
124    /// Explicitly enable bounded access-barrier recovery.
125    pub itna: bool,
126    pub browser: PoolConfig,
127    /// 抓取结果缓存（`maxAge`、`minAge`、`storeInCache`）；不设时不缓存。
128    pub store: Option<pagesift_store::Store>,
129    /// `summary`、`json` 格式和 `/v2/extract` 用的 OpenAI 兼容服务。
130    pub llm: Option<pagesift_llm::LlmConfig>,
131    /// `/v2/search` 用的搜索后端。
132    pub search: Option<Arc<dyn pagesift_search::SearchBackend>>,
133    /// 命名出口（名字，代理地址）：请求的 `egress` 选其中一个，其余抓取设置和默认出口相同；
134    /// 两个及以上时，不指定 `egress` 的请求由出口池选。
135    pub egress: Vec<(String, String)>,
136    /// 命名出口的类别（没列出的是 `direct`）。代理地址为空的 `direct` 出口与默认出口相同。
137    pub egress_classes: std::collections::HashMap<String, pagesift_policy::EgressClass>,
138    /// 审计日志：每次抓取一行元数据，可疑结果留 HTML 快照（见 [`crate::audit`]）。
139    pub audit: Option<Arc<crate::audit::Audit>>,
140    /// 扩展点（`pagesift_hooks`）：代码注册的钩子和 `[[rules]]` 编译出的规则。空时不经过任何钩子。
141    pub hooks: pagesift_hooks::Hooks,
142    /// `[secrets]`：act 动作里 `{{secret:NAME}}` 在本地替换成的值（不经过模型）。
143    pub secrets: std::collections::BTreeMap<String, ActionSecret>,
144    /// 扫描页 OCR（`[ocr] enabled`）；不设时扫描页只有说明，并报 `scanned_pages_without_ocr`。
145    pub ocr: Option<Arc<dyn pagesift_pdf::PageOcr>>,
146    /// `/interact` 保留会话的时长上限（默认 Firecrawl 文档的值）。
147    pub interact: InteractLimits,
148}
149
150/// 创建 [`Scraper`] 失败：网络层或 LLM 客户端的配置不对。
151#[derive(Debug)]
152pub enum ScraperError {
153    Fetcher(pagesift_net::Error),
154    Llm(pagesift_llm::LlmError),
155}
156
157impl std::fmt::Display for ScraperError {
158    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
159        match self {
160            Self::Fetcher(e) => write!(f, "fetcher configuration: {e}"),
161            Self::Llm(e) => write!(f, "LLM configuration: {e}"),
162        }
163    }
164}
165
166impl std::error::Error for ScraperError {
167    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
168        match self {
169            Self::Fetcher(e) => Some(e),
170            Self::Llm(e) => Some(e),
171        }
172    }
173}
174
175pub struct Scraper {
176    policy: Arc<Policy>,
177    itna: bool,
178    /// Reusable identities, scoped by owner, origin, egress and fingerprint.
179    /// A session is never shared between different owners.
180    identities: pagesift_policy::identity_pool::IdentityPool,
181    store: Option<pagesift_store::Store>,
182    llm: Option<pagesift_llm::ChatClient>,
183    search: Option<Arc<dyn pagesift_search::SearchBackend>>,
184    ocr: Option<Arc<dyn pagesift_pdf::PageOcr>>,
185    /// 网络层放开了内网（`allowPrivateNetwork`）时，浏览器的子请求也一样放开。
186    browser_private_network: bool,
187    /// 浏览器子请求的细粒度内网放行名单（allow_hosts / allow_ips / allow_cidrs）。
188    browser_allow: NetworkAllowList,
189    /// 按优先级排列的内核（见 [`pagesift_cdp::engine_chain`]）。每个池第一次需要时才创建；
190    /// 创建失败时记住结果，不反复尝试。
191    browsers: Vec<(PoolConfig, OnceCell<Option<Arc<BrowserPool>>>)>,
192    /// 每个内核最近一次启动失败的时间：之后 [`LAUNCH_RETRY`] 内跳过它，不让每个请求都去试一次。
193    launch_failed: Vec<std::sync::Mutex<Option<Instant>>>,
194    /// 每个内核最近一次启动失败的原因，用来在“浏览器不可用”时告诉调用方真因（root、沙箱等）。
195    launch_notes: Vec<std::sync::Mutex<Option<String>>>,
196    /// 有内核找到了浏览器、但创建或启动失败（不是"没找到浏览器"）。
197    launch_broken: std::sync::atomic::AtomicBool,
198    /// 爬取里同时进行的抓取数，等于全局并发。排队的页面只占一个很小的等待 future，
199    /// 拿到名额才创建完整的抓取状态机（十几 KB）：同时爬上千个站时，排队的页面不再各占一份。
200    /// 上限跟着限速器的全局在途数目标走（USL 控制器，见 `pagesift_policy::RateLimiter`）。
201    crawl_slots: pagesift_policy::AdaptiveConcurrencyGate,
202    learner: crate::learn::HostLearner,
203    /// 要不要开浏览器、渲染结果怎么和 HTTP 结果合并。
204    planner: planner::RenderPlanner,
205    /// 自动升级的在线耗时分布：预算、导航耐心、早停时机都从这里来。
206    render_stats: render_stats::RenderStats,
207    audit: Option<Arc<crate::audit::Audit>>,
208    hooks: pagesift_hooks::Hooks,
209    secrets: std::collections::BTreeMap<String, ActionSecret>,
210    /// 多服务器网关：爬取里的页面交给它（工人执行，见 `pagesift` 的 `cluster` 模块）。单主机不设，路径不变。
211    pages: std::sync::OnceLock<Arc<dyn crate::PageRunner>>,
212    /// `/interact` 的会话表。
213    sessions: Arc<interact::Sessions>,
214}
215
216/// 文档从哪来：抓一个网址，或解析调用方上传的文件（`/v2/parse`）。
217pub(super) enum Source<'a> {
218    Url(&'a str, RobotsMode),
219    Upload(pagesift_api::types::Upload),
220}
221
222#[derive(Debug, Clone, Copy, PartialEq, Eq)]
223enum EngineChoice {
224    Http,
225    Browser,
226    Auto,
227}
228
229impl Engine for Scraper {
230    fn scrape<'a>(
231        &'a self,
232        url: &'a str,
233        opts: &'a ScrapeOptions,
234    ) -> BoxFuture<'a, Result<Document, ApiError>> {
235        Box::pin(Scraper::scrape(self, url, opts))
236    }
237
238    fn parse<'a>(
239        &'a self,
240        file: pagesift_api::types::Upload,
241        opts: &'a ScrapeOptions,
242    ) -> BoxFuture<'a, Result<Document, ApiError>> {
243        Box::pin(Scraper::parse(self, file, opts))
244    }
245
246    fn map<'a>(
247        &'a self,
248        req: &'a pagesift_api::types::MapRequest,
249    ) -> BoxFuture<'a, Result<Vec<pagesift_api::types::MapLink>, ApiError>> {
250        Box::pin(Scraper::map(self, req))
251    }
252
253    fn crawl<'a>(
254        &'a self,
255        req: &'a pagesift_api::types::CrawlRequest,
256        sink: Arc<dyn pagesift_api::CrawlSink>,
257    ) -> BoxFuture<'a, Result<(), ApiError>> {
258        Box::pin(Scraper::crawl(self, req, sink))
259    }
260
261    fn search<'a>(
262        &'a self,
263        req: &'a pagesift_api::types::SearchRequest,
264    ) -> BoxFuture<'a, Result<serde_json::Value, ApiError>> {
265        Box::pin(Scraper::search(self, req))
266    }
267
268    fn extract<'a>(
269        &'a self,
270        req: &'a pagesift_api::types::ExtractRequest,
271    ) -> BoxFuture<'a, Result<serde_json::Value, ApiError>> {
272        Box::pin(Scraper::extract(self, req))
273    }
274
275    fn search_budgeted<'a>(
276        &'a self,
277        req: &'a pagesift_api::types::SearchRequest,
278        budget: Arc<dyn pagesift_api::PageBudget>,
279    ) -> BoxFuture<'a, Result<serde_json::Value, ApiError>> {
280        Box::pin(async move { self.search_with_budget(req, Some(budget.as_ref())).await })
281    }
282
283    fn extract_budgeted<'a>(
284        &'a self,
285        req: &'a pagesift_api::types::ExtractRequest,
286        budget: Arc<dyn pagesift_api::PageBudget>,
287    ) -> BoxFuture<'a, Result<serde_json::Value, ApiError>> {
288        Box::pin(async move { self.extract_with_budget(req, Some(budget.as_ref())).await })
289    }
290
291    fn supports_interact(&self) -> bool {
292        self.interact_available()
293    }
294
295    /// 动作只在 Chrome 上验证过，和 interact 同一个条件。
296    fn supports_actions(&self) -> bool {
297        self.interact_available()
298    }
299
300    fn actions_unavailable(&self) -> &'static str {
301        "actions need Chrome: no Chrome or Chromium was found on this machine (run `pagesift doctor`)"
302    }
303
304    fn scrape_session<'a>(
305        &'a self,
306        url: &'a str,
307        opts: &'a ScrapeOptions,
308        keep: bool,
309        owner: Option<&'a str>,
310    ) -> BoxFuture<'a, Result<Document, ApiError>> {
311        Box::pin(Scraper::scrape_session(self, url, opts, keep, owner))
312    }
313
314    fn observe<'a>(
315        &'a self,
316        url: &'a str,
317        instruction: &'a str,
318    ) -> BoxFuture<'a, Result<pagesift_api::types::Observed, ApiError>> {
319        Box::pin(async move {
320            let (found, used) = Scraper::observe(self, url, instruction).await?;
321            let usage = format!("llm_usage: {} calls, {} tokens", used.calls, used.tokens);
322            Ok((found, vec![usage]))
323        })
324    }
325
326    fn find_similar<'a>(
327        &'a self,
328        url: &'a str,
329        selector: &'a str,
330        opts: &'a ScrapeOptions,
331    ) -> BoxFuture<'a, Result<pagesift_api::types::SimilarData, ApiError>> {
332        Box::pin(Scraper::find_similar(self, url, selector, opts))
333    }
334
335    fn interact<'a>(
336        &'a self,
337        id: &'a str,
338        req: &'a pagesift_api::types::InteractRequest,
339        owner: Option<&'a str>,
340    ) -> BoxFuture<'a, Result<pagesift_api::types::InteractResponse, ApiError>> {
341        Box::pin(Scraper::interact(self, id, req, owner))
342    }
343
344    fn stop_interact<'a>(
345        &'a self,
346        id: &'a str,
347        owner: Option<&'a str>,
348    ) -> BoxFuture<'a, Result<pagesift_api::types::InteractStopped, ApiError>> {
349        Box::pin(Scraper::stop_interact(self, id, owner))
350    }
351
352    fn agent_check(&self, req: &pagesift_api::types::AgentRequest) -> Result<(), ApiError> {
353        agent::check(self.llm.is_some(), self.search.is_some(), req)
354    }
355
356    fn agent<'a>(
357        &'a self,
358        req: &'a pagesift_api::types::AgentRequest,
359        sink: Arc<dyn pagesift_api::AgentSink>,
360    ) -> BoxFuture<'a, pagesift_api::types::AgentOutcome> {
361        Box::pin(async move {
362            match &self.llm {
363                Some(llm) => agent::run(self, llm, self.search.is_some(), req, sink.as_ref()).await,
364                None => pagesift_api::types::AgentOutcome {
365                    finish_reason: "error".into(),
366                    error: Some(pagesift_api::types::AgentFailure::from(
367                        &ApiError::unsupported("agent needs an LLM: set [llm] in pagesift.toml"),
368                    )),
369                    ..Default::default()
370                },
371            }
372        })
373    }
374
375    /// 统一准入：批量任务的并发取当前全局在途数目标（限速器的 USL 控制器）。
376    fn admitted_concurrency(&self) -> Option<usize> {
377        Some(self.concurrency())
378    }
379
380    fn supported_formats(&self) -> &'static [&'static str] {
381        match (self.llm.is_some(), self.store.is_some()) {
382            (true, true) => &[
383                "markdown",
384                "html",
385                "rawHtml",
386                "links",
387                "images",
388                "extract",
389                "summary",
390                "json",
391                "changeTracking",
392            ],
393            (true, false) => &[
394                "markdown", "html", "rawHtml", "links", "images", "extract", "summary", "json",
395            ],
396            (false, true) => &[
397                "markdown",
398                "html",
399                "rawHtml",
400                "links",
401                "images",
402                "extract",
403                "changeTracking",
404            ],
405            (false, false) => &["markdown", "html", "rawHtml", "links", "images", "extract"],
406        }
407    }
408}
409
410/// 放开内网时浏览器子请求的检查：只限制协议，不查地址。
411struct SchemeGuard;
412
413impl RequestGuard for SchemeGuard {
414    fn check<'a>(&'a self, req: &'a SubRequest) -> pagesift_cdp::BoxFuture<'a, Verdict> {
415        let ok = Url::parse(&req.url)
416            .is_ok_and(|u| matches!(u.scheme(), "http" | "https" | "data" | "blob" | "about"));
417        Box::pin(async move {
418            if ok {
419                Verdict::Allow
420            } else {
421                Verdict::Block("scheme not allowed".into())
422            }
423        })
424    }
425}
426
427#[cfg(test)]
428mod tests {
429    use super::*;
430
431    fn content(md: &str, published: bool, og_article: bool) -> pagesift_content::Document {
432        let mut d = pagesift_content::process("<p>x</p>", &Default::default());
433        d.markdown = md.to_string();
434        d.metadata.published_time = published.then(|| "2026-10-21T00:00:00Z".into());
435        if og_article {
436            d.metadata.extra.push(("og:type".into(), "article".into()));
437        }
438        d
439    }
440
441    #[test]
442    fn static_learning_cannot_suppress_client_code() {
443        for html in [
444            "<script>hydrate()</script>",
445            "<script src='/app.js'></script>",
446            "<SCRIPT TYPE='module' src='/view.js'></SCRIPT>",
447            "<body onload='hydrate()'>",
448            "<button onclick='hydrate()'>Open</button>",
449        ] {
450            assert!(has_client_code(html), "{html}");
451        }
452        for html in [
453            "<article><p>Short but complete.</p></article>",
454            "<script type='application/ld+json'>{}</script>",
455            "<script type='APPLICATION/JSON; charset=utf-8'>{}</script>",
456            "<script type='text/plain'>inert sample</script>",
457        ] {
458            assert!(!has_client_code(html), "{html}");
459        }
460    }
461
462    #[test]
463    fn render_rule_weights_cjk_and_trusts_short_articles() {
464        assert_eq!(text_weight("ab c\n"), 3);
465        assert_eq!(text_weight("日本語のニュース"), 8 * CJK_WEIGHT);
466        // 500 个汉字按 1500 算，不用渲染；500 个拉丁字母要渲染
467        assert!(!needs_render(&content(&"字".repeat(500), false, false)));
468        assert!(needs_render(&content(&"a".repeat(500), false, false)));
469        // 认出是文章且有 300 以上：不渲染；太短的文章照样渲染
470        assert!(!needs_render(&content(&"a".repeat(400), true, false)));
471        assert!(!needs_render(&content(&"a".repeat(400), false, true)));
472        assert!(needs_render(&content(&"a".repeat(100), true, true)));
473    }
474
475    #[test]
476    fn render_errors_are_classified() {
477        use pagesift_cdp::Error as E;
478        for (e, code) in [
479            (
480                E::Action {
481                    index: 0,
482                    message: "no element".into(),
483                },
484                ErrorCode::InvalidRequest,
485            ),
486            (E::InvalidOptions("x".into()), ErrorCode::InvalidRequest),
487            (E::ExecuteJavascriptDisabled, ErrorCode::UnsupportedOption),
488            (E::Timeout, ErrorCode::Timeout),
489            (E::Busy, ErrorCode::Busy),
490            (
491                E::Navigation("net::ERR_NAME_NOT_RESOLVED".into()),
492                ErrorCode::DnsFailure,
493            ),
494            (
495                E::Navigation("net::ERR_CONNECTION_RESET".into()),
496                ErrorCode::NetworkError,
497            ),
498            (
499                E::Navigation("net::ERR_CERT_DATE_INVALID".into()),
500                ErrorCode::TlsCertExpired,
501            ),
502            (
503                E::Navigation("net::ERR_CERT_COMMON_NAME_INVALID".into()),
504                ErrorCode::TlsHostnameMismatch,
505            ),
506            (
507                E::Navigation("net::ERR_CERT_AUTHORITY_INVALID".into()),
508                ErrorCode::TlsUntrustedRoot,
509            ),
510            (
511                E::Navigation("net::ERR_CERT_REVOKED".into()),
512                ErrorCode::TlsError,
513            ),
514            (E::ConnectionClosed, ErrorCode::UpstreamError),
515        ] {
516            assert_eq!(render_error_code(&e), code, "{e}");
517        }
518    }
519
520    #[test]
521    fn finds_only_visible_same_origin_and_srcdoc_iframes() {
522        let base = Url::parse("https://a.test/page").unwrap();
523        let html = r#"<body>
524            <iframe src="https://www.googletagmanager.com/ns.html"></iframe>
525            <iframe src="/pixel" style="display: none"></iframe>
526            <iframe src="/hidden" aria-hidden="true"></iframe>
527            <iframe src="/tiny" width="1" height="1"></iframe>
528            <iframe src="about:blank"></iframe>
529            <iframe src="/comments?id=1&amp;x=2" height="160"></iframe>
530            <iframe srcdoc="&lt;p&gt;inline&lt;/p&gt;"></iframe>
531            <iframe srcdoc="&lt;p&gt;x&lt;/p&gt;" style="visibility:hidden"></iframe>
532        </body>"#;
533        let frames = content_iframes(html, &base);
534        assert_eq!(frames.len(), 2, "{frames:?}");
535        assert_eq!(frames[0].0, "/comments?id=1&x=2");
536        assert_eq!(
537            frames[0].1.as_ref().map(Url::as_str),
538            Some("https://a.test/comments?id=1&x=2")
539        );
540        assert_eq!(frames[1], ("<p>inline</p>".to_string(), None));
541        assert!(content_iframes("<p>no frames</p>", &base).is_empty());
542    }
543
544    #[test]
545    fn replaces_the_matching_iframe_element() {
546        let mut html = r#"<p>a</p><IFRAME src="/x" hidden></IFRAME><iframe src="/x">fallback</iframe><p>b</p>"#
547            .to_string();
548        assert!(replace_iframe(&mut html, "<div>X</div>", |el| {
549            !hidden_frame(el) && src_is(el, "/x")
550        }));
551        assert_eq!(
552            html,
553            r#"<p>a</p><IFRAME src="/x" hidden></IFRAME><div>X</div><p>b</p>"#
554        );
555        assert!(!replace_iframe(&mut html, "", |el| src_is(el, "/missing")));
556    }
557
558    #[test]
559    fn takes_the_body_of_an_iframe_document() {
560        assert_eq!(
561            body_inner("<html><head><title>t</title></head><BODY class=x><p>hi</p></body></html>"),
562            "<p>hi</p>"
563        );
564        assert_eq!(body_inner("<p>fragment</p>"), "<p>fragment</p>");
565    }
566
567    #[test]
568    fn meta_keys_follow_firecrawl_style() {
569        assert_eq!(pagesift_sink::meta::key("og:image"), "ogImage");
570        assert_eq!(pagesift_sink::meta::key("og:site_name"), "ogSiteName");
571        assert_eq!(
572            pagesift_sink::meta::key("article:published_time"),
573            "articlePublishedTime"
574        );
575        assert_eq!(pagesift_sink::meta::key("dc.title"), "dcTitle");
576    }
577}
pagesift_engine/engine/mod.rs

pagesift_engine/engine/
mod.rs
