pagesift_api/
engine.rs

1//! 抓取引擎接口。API 层只做协议、校验和任务管理，真正的抓取由实现这个 trait 的引擎完成。
2
3use std::future::Future;
4use std::pin::Pin;
5use std::sync::Arc;
6
7use crate::error::ApiError;
8use crate::types::{
9    AgentFailure, AgentOutcome, AgentRequest, AgentStep, CrawlRequest, CrawlSummary, Document,
10    ExtractRequest, InteractRequest, InteractResponse, InteractStopped, MapLink, MapRequest,
11    ScrapeOptions, SearchRequest, Upload,
12};
13use serde_json::Value;
14
15pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
16
17/// 每次抓取前取得页面预算；拒绝时不得发出页面请求。
18pub trait PageBudget: Send + Sync {
19    fn acquire(&self) -> BoxFuture<'_, Result<(), ApiError>>;
20}
21
22pub trait Engine: Send + Sync + 'static {
23    fn scrape<'a>(
24        &'a self,
25        url: &'a str,
26        opts: &'a ScrapeOptions,
27    ) -> BoxFuture<'a, Result<Document, ApiError>>;
28
29    /// `/v2/parse`：解析调用方上传的文件（PDF、HTML），返回和 scrape 相同的文档。
30    /// `opts` 已经过 [`crate::check_parse_options`] 校验。
31    fn parse<'a>(
32        &'a self,
33        _file: Upload,
34        _opts: &'a ScrapeOptions,
35    ) -> BoxFuture<'a, Result<Document, ApiError>> {
36        Box::pin(async {
37            Err(ApiError::unsupported(
38                "parse is not available in this build",
39            ))
40        })
41    }
42
43    fn map<'a>(&'a self, _req: &'a MapRequest) -> BoxFuture<'a, Result<Vec<MapLink>, ApiError>> {
44        Box::pin(async { Err(ApiError::unsupported("map is not available in this build")) })
45    }
46
47    /// 整站爬取：抓到一页、出错一页、发现新 URL 时都通过 `sink` 回报，直到爬完或被取消。
48    fn crawl<'a>(
49        &'a self,
50        _req: &'a CrawlRequest,
51        _sink: Arc<dyn CrawlSink>,
52    ) -> BoxFuture<'a, Result<(), ApiError>> {
53        Box::pin(async {
54            Err(ApiError::unsupported(
55                "crawl is not available in this build",
56            ))
57        })
58    }
59
60    /// `/v2/search`：返回 Firecrawl 的 `data`（`{ "web": [...] }`）。
61    fn search<'a>(&'a self, _req: &'a SearchRequest) -> BoxFuture<'a, Result<Value, ApiError>> {
62        Box::pin(async {
63            Err(ApiError::unsupported(
64                "search is not configured: set [search] in pagesift.toml",
65            ))
66        })
67    }
68
69    /// `/v2/extract`：返回提取出的 JSON。
70    fn extract<'a>(&'a self, _req: &'a ExtractRequest) -> BoxFuture<'a, Result<Value, ApiError>> {
71        Box::pin(async {
72            Err(ApiError::unsupported(
73                "extract needs an LLM: set [llm] in pagesift.toml",
74            ))
75        })
76    }
77
78    fn search_budgeted<'a>(
79        &'a self,
80        _req: &'a SearchRequest,
81        _budget: Arc<dyn PageBudget>,
82    ) -> BoxFuture<'a, Result<Value, ApiError>> {
83        Box::pin(async {
84            Err(ApiError::unsupported(
85                "page quotas are not supported by this search engine",
86            ))
87        })
88    }
89
90    fn extract_budgeted<'a>(
91        &'a self,
92        _req: &'a ExtractRequest,
93        _budget: Arc<dyn PageBudget>,
94    ) -> BoxFuture<'a, Result<Value, ApiError>> {
95        Box::pin(async {
96            Err(ApiError::unsupported(
97                "page quotas are not supported by this extract engine",
98            ))
99        })
100    }
101
102    /// `/v2/agent` 开跑前的检查：没配 LLM、没有 `urls` 又没配搜索时，提交时就报错。
103    fn agent_check(&self, _req: &AgentRequest) -> Result<(), ApiError> {
104        Err(ApiError::unsupported(
105            "agent needs an LLM: set [llm] in pagesift.toml",
106        ))
107    }
108
109    /// `/v2/agent`：跑完一次（中途出错也回已经攒下的证据，错误在 `AgentOutcome::error`）。
110    fn agent<'a>(
111        &'a self,
112        _req: &'a AgentRequest,
113        _sink: Arc<dyn AgentSink>,
114    ) -> BoxFuture<'a, AgentOutcome> {
115        Box::pin(async {
116            AgentOutcome {
117                error: Some(AgentFailure::from(&ApiError::unsupported(
118                    "agent needs an LLM: set [llm] in pagesift.toml",
119                ))),
120                ..Default::default()
121            }
122        })
123    }
124
125    /// 引擎支持的 `formats`；请求了不支持的格式时返回 UNSUPPORTED_OPTION，不悄悄忽略。
126    fn supported_formats(&self) -> &'static [&'static str] {
127        &["markdown", "html", "rawHtml", "links", "images"]
128    }
129
130    /// 是否支持 `actions`（需要浏览器渲染）。
131    fn supports_actions(&self) -> bool {
132        false
133    }
134
135    /// 不支持 `actions` 时给调用方的说明。
136    fn actions_unavailable(&self) -> &'static str {
137        "actions need browser rendering, which is not available in this build"
138    }
139
140    /// 是否支持 `/v2/scrape/{id}/interact`。支持时 `/v2/scrape` 走 [`Engine::scrape_session`]。
141    fn supports_interact(&self) -> bool {
142        false
143    }
144
145    /// `/v2/scrape`：抓一页，结果带 `metadata.scrapeId`。`keep` 时保留浏览器页面（`keepSession`），
146    /// 否则只记下回放信息。`owner` 是调用方的账号，之后的 interact 只认同一个账号。
147    fn scrape_session<'a>(
148        &'a self,
149        _url: &'a str,
150        _opts: &'a ScrapeOptions,
151        _keep: bool,
152        _owner: Option<&'a str>,
153    ) -> BoxFuture<'a, Result<Document, ApiError>> {
154        Box::pin(async {
155            Err(ApiError::unsupported(
156                "keepSession is not available in this build",
157            ))
158        })
159    }
160
161    /// `/v2/observe`：页面上和指令对应的候选元素（选择器、角色、名称），返回候选和用量警告。
162    fn observe<'a>(
163        &'a self,
164        _url: &'a str,
165        _instruction: &'a str,
166    ) -> BoxFuture<'a, Result<crate::types::Observed, ApiError>> {
167        Box::pin(async {
168            Err(ApiError::unsupported(
169                "observe is not available in this build",
170            ))
171        })
172    }
173
174    /// `/v2/similar`：和例子结构相同的兄弟元素。
175    fn find_similar<'a>(
176        &'a self,
177        _url: &'a str,
178        _selector: &'a str,
179        _opts: &'a ScrapeOptions,
180    ) -> BoxFuture<'a, Result<crate::types::SimilarData, ApiError>> {
181        Box::pin(async {
182            Err(ApiError::unsupported(
183                "find_similar is not available in this build",
184            ))
185        })
186    }
187
188    fn interact<'a>(
189        &'a self,
190        _id: &'a str,
191        _req: &'a InteractRequest,
192        _owner: Option<&'a str>,
193    ) -> BoxFuture<'a, Result<InteractResponse, ApiError>> {
194        Box::pin(async {
195            Err(ApiError::unsupported(
196                "interact is not available in this build",
197            ))
198        })
199    }
200
201    fn stop_interact<'a>(
202        &'a self,
203        _id: &'a str,
204        _owner: Option<&'a str>,
205    ) -> BoxFuture<'a, Result<InteractStopped, ApiError>> {
206        Box::pin(async {
207            Err(ApiError::unsupported(
208                "interact is not available in this build",
209            ))
210        })
211    }
212
213    /// 统一准入现在允许的在途抓取数：批量任务的并发按它定，用户的
214    /// `max_batch_concurrency` 只作上限。引擎还没接准入时返回 `None`，API 用配置或先验。
215    fn admitted_concurrency(&self) -> Option<usize> {
216        None
217    }
218}
219
220/// agent 运行的回报和额度，由 API 层的任务管理实现；`()` 是不回报、不限额的实现。
221pub trait AgentSink: Send + Sync {
222    /// 一次工具调用结束；`credits` 是到现在为止用掉的额度。
223    fn step<'a>(&'a self, _step: &'a AgentStep, _credits: u64) -> BoxFuture<'a, ()> {
224        Box::pin(async {})
225    }
226    /// 调用方要求取消：在两步之间生效。
227    fn cancelled(&self) -> bool {
228        false
229    }
230    /// 抓到一页：计入调用方的页面配额。
231    fn charge_page(&self) -> BoxFuture<'_, ()> {
232        Box::pin(async {})
233    }
234    /// 调用方今天还能抓多少页；没有配额时 `None`。
235    fn pages_left(&self) -> Option<u64> {
236        None
237    }
238}
239
240impl AgentSink for () {}
241
242/// 整站爬取的进度回报，由 API 层的任务管理实现。
243pub trait CrawlSink: Send + Sync {
244    fn page(&self, doc: Document) -> BoxFuture<'_, ()>;
245    fn error<'a>(&'a self, url: &'a str, err: ApiError) -> BoxFuture<'a, ()>;
246    /// 已经确定要抓的页面总数（随着发现新链接增长）。
247    fn total(&self, total: usize) -> BoxFuture<'_, ()>;
248    fn cancelled(&self) -> BoxFuture<'_, bool>;
249
250    /// 断点续爬：任务是从中断处恢复的时候返回上次的进度，引擎据此重建队列和去重表，
251    /// 不再抓已经完成的页面。新任务返回 `None`。
252    fn resume(&self) -> Option<CrawlResume> {
253        None
254    }
255
256    /// 是否需要 [`CrawlSink::enqueued`] 回报（持久化的任务需要）。不需要时引擎不必收集。
257    fn tracks_frontier(&self) -> bool {
258        false
259    }
260
261    /// 新进入队列的 URL 和发现深度，按入队顺序。只在 [`CrawlSink::tracks_frontier`] 为真时调用。
262    fn enqueued(&self, _urls: Vec<(String, usize)>) -> BoxFuture<'_, ()> {
263        Box::pin(async {})
264    }
265
266    /// 爬取结束时的说明（停止原因等）；只有带查询的爬取会调用。
267    fn summary(&self, _summary: CrawlSummary) -> BoxFuture<'_, ()> {
268        Box::pin(async {})
269    }
270}
271
272/// 中断的爬取任务恢复时的进度。完成的页面以文档的 `metadata.sourceURL`（失败的以出错的 URL）为准。
273#[derive(Debug, Clone, Default, PartialEq)]
274pub struct CrawlResume {
275    /// 入过队的全部 URL，按入队顺序（去重表）。
276    pub enqueued: Vec<String>,
277    /// 入了队但还没完成的 URL 和发现深度，按入队顺序。中断时正在抓的页面也在这里。
278    pub pending: Vec<(String, usize)>,
279    /// 已经完成的页数（成功和失败都算），计入 `limit`。
280    pub done: usize,
281}
pagesift_api/engine.rs

pagesift_api/
engine.rs
