pagesift_engine/
lib.rs

1//! pagesift 的引擎：单页抓取的完整流程（HTTP、渲染、渲染决策、PDF/Office、结果组装）和爬取循环。
2//! 命令行、HTTP 服务、多服务器接线在上层的 `pagesift` crate；这里不依赖它们。
3
4#![forbid(unsafe_code)]
5
6pub mod change;
7pub mod crawl;
8pub mod discovery;
9pub mod engine;
10pub mod notice;
11pub mod scripts;
12
13pub use pagesift_frontier::{focus, frontier};
14pub use pagesift_plan::learn;
15pub use pagesift_sink::audit;
16
17use pagesift_api::types::{Document, ScrapeOptions};
18use pagesift_api::{ApiError, BoxFuture};
19use pagesift_policy::RobotsMode;
20
21/// 爬取里逐页抓取的执行者。网关把它装到 `Scraper` 上（[`engine::Scraper::route_pages`]），
22/// 爬取前沿留在网关，页面交给工人。
23pub trait PageRunner: Send + Sync {
24    fn run<'a>(
25        &'a self,
26        kind: &'static str,
27        url: &'a str,
28        opts: &'a ScrapeOptions,
29        mode: RobotsMode,
30    ) -> BoxFuture<'a, Result<Document, ApiError>>;
31
32    /// 执行方的准入容量（网关：在线工人报告的准入值之和）；`None` 时用本机的准入。
33    fn capacity(&self) -> Option<usize> {
34        None
35    }
36}
37
38pub use crawl::normalize as normalize_url;
39pub use engine::{
40    AUTO_RENDER_ARTICLE_MIN_CHARS, AUTO_RENDER_MIN_CHARS, EngineProbe, Scraper, ScraperConfig,
41    ScraperError, needs_render, office_markdown, office_markdown_from, set_cpu_capacity,
42    set_cpu_groups, text_weight,
43};
pagesift_engine/lib.rs

pagesift_engine/
lib.rs
