pub use crate::engines::types::{
validate_session_id, CapturedResponse, HttpMethod, InternalCapturedResponse,
InternalPageAction, InternalScrapeRequest, InternalScrapeResponse, InternalScreenshotConfig,
PageAction, ScrapeOptions, ScrapeOptionsBuilder, ScrapeRequest, ScrapeResponse,
ScreenshotConfig, ScrollDirection, WaitFor, MAX_SESSION_ID_LEN,
};
use crate::engines::health_monitor::{AggregateHealthStatus, EngineHealthMonitor};
use crate::engines::router::{EngineRouter, EngineRouterTrait};
use crate::engines::validators::validate_url;
use crate::utils::retry::RetryReason;
use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;
#[derive(Error, Debug, Clone)]
pub enum EngineError {
#[error("Request failed: {0}")]
RequestFailed(String),
#[error("Request timed out after {0:?}")]
Timeout(Duration),
#[error("All engines failed: {0}")]
AllEnginesFailed(String),
#[error("No engines available")]
NoEnginesAvailable,
#[error("Invalid URL: {0}")]
InvalidUrl(String),
#[error("SSRF protection: {0}")]
SsrfProtection(String),
#[error("Memory constrained: {0}")]
MemoryConstrained(String),
#[error("Unsupported operation: {0}")]
Unsupported(String),
#[error("Browser error: {0}")]
BrowserError(String),
#[error("Anti-bot detected: {0}")]
AntiBotDetected(String),
#[error("Feature toggle: {0}")]
FeatureToggle(String),
#[error("Request expired")]
Expired,
#[error("Engine {engine} exceeded MRT of {mrt:?}")]
EngineMrtExceeded {
engine: String,
mrt: Duration,
},
#[error("Other error: {0}")]
Other(String),
#[error("Internal error: {0}")]
Internal(String),
}
impl From<String> for EngineError {
fn from(msg: String) -> Self {
EngineError::RequestFailed(msg)
}
}
impl From<&str> for EngineError {
fn from(msg: &str) -> Self {
EngineError::RequestFailed(msg.to_string())
}
}
impl From<anyhow::Error> for EngineError {
fn from(err: anyhow::Error) -> Self {
EngineError::Internal(err.to_string())
}
}
impl EngineError {
pub fn is_retryable(&self) -> bool {
match self {
Self::RequestFailed(_) => true,
Self::Timeout(_) => true,
Self::NoEnginesAvailable => false,
Self::InvalidUrl(_) => false,
Self::SsrfProtection(_) => false,
Self::BrowserError(_) => true,
Self::AntiBotDetected(_) => true,
Self::FeatureToggle(_) => true,
Self::Internal(_) => false,
Self::AllEnginesFailed(_) => false,
Self::Expired => false,
Self::Other(_) => false,
Self::MemoryConstrained(_) => false,
Self::Unsupported(_) => false,
Self::EngineMrtExceeded { .. } => true,
}
}
pub fn retry_reason(&self) -> RetryReason {
match self {
Self::AntiBotDetected(_) => RetryReason::AntiBot,
Self::FeatureToggle(_) => RetryReason::FeatureToggle,
Self::RequestFailed(_)
| Self::Timeout(_)
| Self::BrowserError(_)
| Self::NoEnginesAvailable
| Self::InvalidUrl(_)
| Self::SsrfProtection(_)
| Self::Internal(_)
| Self::AllEnginesFailed(_)
| Self::Expired
| Self::Other(_)
| Self::MemoryConstrained(_)
| Self::Unsupported(_)
| Self::EngineMrtExceeded { .. } => RetryReason::Transient,
}
}
}
use async_trait::async_trait;
#[async_trait]
pub trait ScraperEngine: Send + Sync {
async fn scrape(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError>;
fn support_score(&self, request: &InternalScrapeRequest) -> u8;
fn name(&self) -> &'static str;
fn supports_tls_fingerprint(&self) -> bool {
false
}
fn max_response_time(&self) -> Duration {
Duration::from_secs(30)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum EngineHealthStatus {
#[default]
Healthy,
Degraded {
unhealthy_engines: Vec<String>,
message: String,
},
Unavailable {
message: String,
},
}
#[async_trait]
pub trait EngineClientTrait: Send + Sync {
async fn scrape(&self, request: &ScrapeRequest) -> Result<ScrapeResponse, EngineError>;
async fn health_check(&self) -> EngineHealthStatus;
fn engine_count(&self) -> usize;
fn registered_engines(&self) -> Vec<String>;
}
#[derive(Clone)]
pub struct EngineClient {
router: Arc<dyn EngineRouterTrait>,
#[cfg(not(feature = "auth"))]
health_monitor: Arc<EngineHealthMonitor>,
#[cfg(feature = "auth")]
health_registry: Arc<garrison::health::HealthRegistry>,
}
impl EngineClient {
pub fn new() -> Self {
let router: Arc<dyn EngineRouterTrait> = Arc::new(EngineRouter::new(Vec::new()));
Self::with_router(router)
}
pub fn with_router(router: Arc<dyn EngineRouterTrait>) -> Self {
let health_monitor = Arc::new(EngineHealthMonitor::new(Vec::new()));
#[cfg(feature = "auth")]
let health_registry = Arc::new(
crate::engines::health_monitor::garrison_health::build_health_registry(&health_monitor),
);
Self {
router,
#[cfg(not(feature = "auth"))]
health_monitor,
#[cfg(feature = "auth")]
health_registry,
}
}
pub fn with_engines(engines: Vec<Arc<dyn ScraperEngine>>) -> Self {
let router: Arc<dyn EngineRouterTrait> = Arc::new(EngineRouter::new(engines.clone()));
let engines_for_health = Vec::new(); let health_monitor = Arc::new(EngineHealthMonitor::new(engines_for_health));
#[cfg(feature = "auth")]
let health_registry = Arc::new(
crate::engines::health_monitor::garrison_health::build_health_registry(&health_monitor),
);
Self {
router,
#[cfg(not(feature = "auth"))]
health_monitor,
#[cfg(feature = "auth")]
health_registry,
}
}
pub async fn scrape(&self, request: &ScrapeRequest) -> Result<ScrapeResponse, EngineError> {
if let Err(e) = validate_url(&request.url).await {
return Err(EngineError::SsrfProtection(e.to_string()));
}
let internal_request = request.to_internal()?;
match self.router.route(&internal_request).await {
Ok(response) => Ok(response.to_public(&request.url)),
Err(e) => Err(convert_error(e)),
}
}
pub async fn health_check(&self) -> EngineHealthStatus {
#[cfg(feature = "auth")]
let status = {
use crate::engines::health_monitor::garrison_health;
let report = self.health_registry.check_all().await;
match garrison_health::map_report(&report) {
AggregateHealthStatus::Healthy => EngineHealthStatus::Healthy,
AggregateHealthStatus::Degraded(unhealthy_engines) => {
let count = unhealthy_engines.len();
EngineHealthStatus::Degraded {
unhealthy_engines,
message: format!("{} engines degraded", count),
}
}
AggregateHealthStatus::Unavailable => EngineHealthStatus::Unavailable {
message: "All engines unavailable".to_string(),
},
}
};
#[cfg(not(feature = "auth"))]
let status = {
self.health_monitor.perform_all_health_checks().await;
let status = self.health_monitor.get_aggregate_status().await;
match status {
AggregateHealthStatus::Healthy => EngineHealthStatus::Healthy,
AggregateHealthStatus::Degraded(unhealthy_engines) => {
let count = unhealthy_engines.len();
EngineHealthStatus::Degraded {
unhealthy_engines,
message: format!("{} engines degraded", count),
}
}
AggregateHealthStatus::Unavailable => EngineHealthStatus::Unavailable {
message: "All engines unavailable".to_string(),
},
}
};
status
}
#[cfg(feature = "auth")]
pub fn health_registry(&self) -> Arc<garrison::health::HealthRegistry> {
Arc::clone(&self.health_registry)
}
pub fn engine_count(&self) -> usize {
self.router.registered_engines().len()
}
pub fn registered_engines(&self) -> Vec<String> {
self.router.registered_engines()
}
pub fn set_crawl_delay(&self, host: &str, delay: Duration) {
self.router.set_crawl_delay(host, delay);
}
}
impl Default for EngineClient {
fn default() -> Self {
Self::new()
}
}
#[async_trait]
impl EngineClientTrait for EngineClient {
async fn scrape(&self, request: &ScrapeRequest) -> Result<ScrapeResponse, EngineError> {
self.scrape(request).await
}
async fn health_check(&self) -> EngineHealthStatus {
self.health_check().await
}
fn engine_count(&self) -> usize {
self.engine_count()
}
fn registered_engines(&self) -> Vec<String> {
self.registered_engines()
}
}
fn convert_error(e: EngineError) -> EngineError {
match e {
EngineError::RequestFailed(msg) => EngineError::RequestFailed(msg),
EngineError::Timeout(duration) => EngineError::Timeout(duration),
EngineError::AllEnginesFailed(msg) => {
if msg.contains("No suitable engines") {
EngineError::NoEnginesAvailable
} else {
EngineError::RequestFailed(msg)
}
}
EngineError::SsrfProtection(msg) => EngineError::SsrfProtection(msg),
EngineError::BrowserError(msg) => EngineError::BrowserError(msg),
EngineError::AntiBotDetected(msg) => EngineError::AntiBotDetected(msg),
EngineError::FeatureToggle(msg) => EngineError::FeatureToggle(msg),
EngineError::Expired => EngineError::Internal("Request expired".to_string()),
EngineError::Other(msg) => EngineError::Internal(msg),
EngineError::MemoryConstrained(msg) => EngineError::MemoryConstrained(msg),
EngineError::Unsupported(msg) => EngineError::Unsupported(msg),
EngineError::NoEnginesAvailable => EngineError::NoEnginesAvailable,
EngineError::InvalidUrl(msg) => EngineError::InvalidUrl(msg),
EngineError::Internal(msg) => EngineError::Internal(msg),
EngineError::EngineMrtExceeded { engine, mrt } => {
EngineError::EngineMrtExceeded { engine, mrt }
}
}
}
#[cfg(test)]
#[path = "tests/engine_client_test.rs"]
mod tests;