use std::time::Duration;
use thiserror::Error;
pub mod config;
pub mod prompt;
pub mod scraper;
pub mod search;
pub mod types;
pub mod llm;
pub use config::ScraperConfig;
pub use types::{ScrapedContent, SearchResult};
#[derive(Error, Debug)]
pub enum ScraperError {
#[error("HTTP request failed: {0}")]
RequestError(#[from] reqwest::Error),
#[error("Rate limit exceeded")]
RateLimitError,
#[error("Content extraction failed: {0}")]
ExtractionError(String),
#[error("LLM processing failed: {0}")]
LLMError(String),
#[error("Search failed: {0}")]
SearchError(String),
}
pub type Result<T> = std::result::Result<T, ScraperError>;
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_CONCURRENT_REQUESTS: usize = 5;
pub const DEFAULT_MAX_RETRIES: u32 = 3;