use crate::error::{FerrisFetcherError, Result};
use crate::types::{HttpMethod, RateLimit, RetryPolicy};
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use std::time::Duration;
use url::Url;
#[derive(Debug, Clone)]
pub struct Config {
pub user_agent: String,
pub timeout: Duration,
pub max_concurrent_requests: usize,
pub rate_limit: Option<RateLimit>,
pub retry_policy: RetryPolicy,
pub headers: HeaderMap,
pub follow_redirects: bool,
pub max_redirects: usize,
pub cookie_jar: bool,
pub proxy: Option<Url>,
pub default_method: HttpMethod,
pub connection_pool_size: usize,
pub connect_timeout: Duration,
pub keep_alive_timeout: Duration,
pub tcp_keep_alive: Duration,
pub http2: bool,
pub compression: bool,
pub brotli: bool,
pub gzip: bool,
pub deflate: bool,
}
impl Default for Config {
fn default() -> Self {
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_str(&format!("FerrisFetcher/{}", env!("CARGO_PKG_VERSION")))
.expect("Invalid user agent"),
);
headers.insert("Accept", HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
headers.insert("Accept-Language", HeaderValue::from_static("en-US,en;q=0.5"));
headers.insert("Accept-Encoding", HeaderValue::from_static("gzip, deflate, br"));
headers.insert("DNT", HeaderValue::from_static("1"));
headers.insert("Connection", HeaderValue::from_static("keep-alive"));
headers.insert("Upgrade-Insecure-Requests", HeaderValue::from_static("1"));
Self {
user_agent: format!("FerrisFetcher/{}", env!("CARGO_PKG_VERSION")),
timeout: Duration::from_secs(30),
max_concurrent_requests: 10,
rate_limit: Some(RateLimit::default()),
retry_policy: RetryPolicy::default(),
headers,
follow_redirects: true,
max_redirects: 5,
cookie_jar: true,
proxy: None,
default_method: HttpMethod::Get,
connection_pool_size: 100,
connect_timeout: Duration::from_secs(10),
keep_alive_timeout: Duration::from_secs(60),
tcp_keep_alive: Duration::from_secs(60),
http2: true,
compression: true,
brotli: true,
gzip: true,
deflate: true,
}
}
}
impl Config {
pub fn new() -> Self {
Self::default()
}
pub fn with_user_agent(mut self, user_agent: impl Into<String>) -> Self {
self.user_agent = user_agent.into();
if let Ok(value) = HeaderValue::from_str(&self.user_agent) {
self.headers.insert(USER_AGENT, value);
}
self
}
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn with_max_concurrent_requests(mut self, max: usize) -> Self {
self.max_concurrent_requests = max;
self
}
pub fn with_rate_limit(mut self, rate_limit: RateLimit) -> Self {
self.rate_limit = Some(rate_limit);
self
}
pub fn without_rate_limit(mut self) -> Self {
self.rate_limit = None;
self
}
pub fn with_retry_policy(mut self, retry_policy: RetryPolicy) -> Self {
self.retry_policy = retry_policy;
self
}
pub fn with_header(mut self, name: &str, value: &str) -> Result<Self> {
let header_name = name.parse::<reqwest::header::HeaderName>()
.map_err(|e| FerrisFetcherError::ConfigError(format!("Invalid header name '{}': {}", name, e)))?;
let header_value = HeaderValue::from_str(value)
.map_err(|e| FerrisFetcherError::ConfigError(format!("Invalid header value for '{}': {}", name, e)))?;
self.headers.insert(header_name, header_value);
Ok(self)
}
pub fn with_proxy(mut self, proxy: Url) -> Self {
self.proxy = Some(proxy);
self
}
pub fn without_redirects(mut self) -> Self {
self.follow_redirects = false;
self
}
pub fn with_max_redirects(mut self, max: usize) -> Self {
self.max_redirects = max;
self
}
pub fn without_cookies(mut self) -> Self {
self.cookie_jar = false;
self
}
pub fn with_default_method(mut self, method: HttpMethod) -> Self {
self.default_method = method;
self
}
pub fn with_connection_pool_size(mut self, size: usize) -> Self {
self.connection_pool_size = size;
self
}
pub fn without_http2(mut self) -> Self {
self.http2 = false;
self
}
pub fn without_compression(mut self) -> Self {
self.compression = false;
self.brotli = false;
self.gzip = false;
self.deflate = false;
self
}
pub fn validate(&self) -> Result<()> {
if self.timeout.is_zero() {
return Err(FerrisFetcherError::ConfigError("Timeout cannot be zero".to_string()));
}
if self.max_concurrent_requests == 0 {
return Err(FerrisFetcherError::ConfigError("Max concurrent requests must be greater than 0".to_string()));
}
if self.max_redirects == 0 && self.follow_redirects {
return Err(FerrisFetcherError::ConfigError("Max redirects must be greater than 0 when following redirects".to_string()));
}
if let Some(rate_limit) = &self.rate_limit {
if rate_limit.requests_per_period == 0 {
return Err(FerrisFetcherError::ConfigError("Rate limit requests per period must be greater than 0".to_string()));
}
}
if self.retry_policy.max_attempts == 0 {
return Err(FerrisFetcherError::ConfigError("Retry policy max attempts must be greater than 0".to_string()));
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_default_config() {
let config = Config::default();
assert!(config.validate().is_ok());
assert_eq!(config.user_agent, format!("FerrisFetcher/{}", crate::VERSION));
assert!(config.follow_redirects);
assert!(config.cookie_jar);
assert!(config.http2);
assert!(config.compression);
}
#[test]
fn test_config_builder() {
let config = Config::new()
.with_timeout(Duration::from_secs(60))
.with_max_concurrent_requests(20)
.without_rate_limit()
.without_redirects();
assert!(config.validate().is_ok());
assert_eq!(config.timeout, Duration::from_secs(60));
assert_eq!(config.max_concurrent_requests, 20);
assert!(config.rate_limit.is_none());
assert!(!config.follow_redirects);
}
#[test]
fn test_invalid_config() {
let config = Config::new().with_timeout(Duration::from_secs(0));
assert!(config.validate().is_err());
}
#[test]
fn test_custom_headers() {
let config = Config::new()
.with_header("X-Custom-Header", "test-value")
.unwrap();
assert!(config.headers.contains_key("x-custom-header"));
assert_eq!(
config.headers.get("x-custom-header").unwrap(),
"test-value"
);
}
}