crw-core 0.24.0

Core types, config, and error handling for the CRW web scraper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

//! Core types, configuration, and error handling for the CRW web scraper.
//!
//! This crate provides the foundational building blocks shared across all CRW crates:
//!
//! - [`config`] — Layered TOML configuration with environment variable overrides
//! - [`error`] — Unified error types ([`CrwError`]) and result alias ([`CrwResult`])
//! - [`types`] — Shared data structures (`ScrapeData`, `FetchResult`, `OutputFormat`, etc.)
//! - [`url_safety`] — SSRF protection (blocks private IPs, cloud metadata, non-HTTP schemes)
//!
//! # Example
//!
//! ```rust
//! use crw_core::{AppConfig, CrwError, CrwResult};
//!
//! let config = AppConfig::load().unwrap();
//! assert!(config.server.port > 0);
//! ```

pub mod config;
pub mod deadline;
pub mod error;
pub mod evidence;
pub mod mcp;
pub mod metrics;
pub mod proxy;
pub mod research_types;
pub mod reserved_sem;
pub mod scrape_class;
pub mod types;
pub mod url_safety;

pub use config::AppConfig;
pub use deadline::Deadline;
pub use error::{CrwError, CrwResult};
pub use proxy::{ProxyEntry, ProxyRotation, ProxyRotator};
pub use reserved_sem::{BatchGate, LanePermit, ReservedSemaphore};
pub use scrape_class::{REQUEST_CLASS, ScrapeClass, current_scrape_class};