pagesift_api/
lib.rs

1//! pagesift 的 HTTP 接口：Firecrawl v2 兼容层、pagesift 扩展、健康检查和监控指标，
2//! 以及部署成服务要用的任务持久化、webhook、按 API key 的限流和配额。
3//!
4//! 协议、校验和任务管理在这里；真正的抓取由实现 [`Engine`] 的引擎完成。
5
6#![forbid(unsafe_code)]
7
8mod agent;
9pub mod attachments;
10mod engine;
11mod error;
12pub mod fields;
13pub(crate) mod firecrawl;
14pub mod jina;
15mod jobs;
16mod limits;
17mod metrics;
18pub mod monitor;
19pub(crate) mod server;
20mod stream_body;
21#[cfg(test)]
22mod test_engine;
23pub mod types;
24mod validate;
25mod webhook;
26
27pub use agent::{Agents, agent_body, agent_body_with_evidence, check_agent_request};
28pub use engine::{AgentSink, BoxFuture, CrawlResume, CrawlSink, Engine};
29pub use error::{ApiError, ErrorCode};
30pub use jobs::mirror::{JobLog, JobMirror, apply_op};
31pub use jobs::{JOB_TTL, Job, JobKind, JobState, Jobs, rfc3339};
32pub use limits::KeyLimits;
33pub use server::{Config, ExtraMetrics, Timeouts, router, serve, serve_with};
34pub use validate::{
35    MAX_CRAWL_DEPTH, MAX_CRAWL_LIMIT, MAX_MAP_LIMIT, MAX_PARSE_FILE_BYTES, MAX_SCRAPE_TIMEOUT_MS,
36    MAX_URL_LEN, MAX_URLS, check_parse_options, check_scrape_options, check_url,
37};
38pub use webhook::{WebhookConfig, WebhookSettings};
pagesift_api/lib.rs

pagesift_api/
lib.rs
