use serde::{Deserialize, Serialize};
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__DATABASE__")]
pub struct DatabaseSettings {
pub(crate) url: String,
#[config(default = Some(150))]
pub max_connections: Option<u32>,
#[config(default = Some(20))]
pub min_connections: Option<u32>,
#[config(default = Some(15))]
pub connect_timeout: Option<u64>,
#[config(default = Some(300))]
pub idle_timeout: Option<u64>,
#[config(default = Some(1800))]
pub max_lifetime: Option<u64>,
#[config(default = Some(30))]
pub connection_keepalive: Option<u64>,
#[config(default = Some(60))]
pub health_check_interval: Option<u64>,
#[config(default = Some(128))]
pub prepare_cache_capacity: Option<usize>,
#[serde(default)]
#[config(default = false)]
pub auto_migrate: bool,
}
impl std::fmt::Debug for DatabaseSettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DatabaseSettings")
.field("url", &"***REDACTED***")
.field("max_connections", &self.max_connections)
.field("min_connections", &self.min_connections)
.field("connect_timeout", &self.connect_timeout)
.field("idle_timeout", &self.idle_timeout)
.field("max_lifetime", &self.max_lifetime)
.field("connection_keepalive", &self.connection_keepalive)
.field("health_check_interval", &self.health_check_interval)
.field("prepare_cache_capacity", &self.prepare_cache_capacity)
.field("auto_migrate", &self.auto_migrate)
.finish()
}
}
impl DatabaseSettings {
pub fn url(&self) -> &str {
&self.url
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SERVER__")]
pub struct ServerSettings {
#[config(default = "0.0.0.0".to_string())]
pub host: String,
#[config(default = 8899)]
pub port: u16,
#[config(default = true)]
pub enable_port_detection: bool,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__WEBSOCKET__")]
pub struct WebSocketSettings {
#[config(default = 100)]
pub max_connections: usize,
#[config(default = 300)]
pub idle_timeout_secs: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__RATE_LIMITING__")]
pub struct RateLimitingSettings {
#[config(default = true)]
pub enabled: bool,
#[config(default = 100)]
pub default_rpm: u32,
#[config(default = 20)]
pub burst_size: u32,
#[config(default = 10)]
pub anon_ip_rpm: u32,
#[config(default = 600)]
pub auth_ip_rpm: u32,
#[config(default = "memory".to_string())]
pub storage_backend: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CONCURRENCY__")]
pub struct ConcurrencySettings {
#[config(default = 10)]
pub default_team_limit: u64,
#[config(default = 300)]
pub task_lock_duration_seconds: u64,
#[config(default = 0.8)]
pub mem_pressure_threshold: f64,
#[config(default = 0.9)]
pub mem_critical_threshold: f64,
#[config(default = 30)]
pub critical_timeout_seconds: u64,
#[config(default = false)]
pub adaptive_enabled: bool,
}
impl ConcurrencySettings {
pub fn validate(&self) {
assert!(
self.mem_pressure_threshold > 0.0 && self.mem_pressure_threshold < 1.0,
"concurrency.mem_pressure_threshold must be in (0.0, 1.0), got {}",
self.mem_pressure_threshold
);
assert!(
self.mem_critical_threshold > 0.0 && self.mem_critical_threshold < 1.0,
"concurrency.mem_critical_threshold must be in (0.0, 1.0), got {}",
self.mem_critical_threshold
);
assert!(
self.mem_pressure_threshold < self.mem_critical_threshold,
"concurrency.mem_pressure_threshold ({}) must be strictly less than \
mem_critical_threshold ({})",
self.mem_pressure_threshold,
self.mem_critical_threshold
);
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__RETENTION__")]
pub struct RetentionSettings {
#[config(default = 30)]
pub scrape_results_days: i64,
}
#[cfg(test)]
mod tests {
use crate::config::WorkerCount;
#[test]
fn test_worker_count_fixed() {
let count = WorkerCount::Fixed(10);
assert_eq!(count.resolve(), 10);
}
#[test]
fn test_worker_count_auto_returns_positive() {
let count = WorkerCount::Auto("auto".to_string());
let resolved = count.resolve();
assert!(
resolved > 0,
"Auto mode should return positive value, got {}",
resolved
);
}
#[test]
fn test_worker_count_default_is_auto() {
let count = WorkerCount::default();
match count {
WorkerCount::Auto(_) => {} WorkerCount::Fixed(n) => panic!("Expected Auto, got Fixed({})", n),
}
}
#[test]
fn test_database_settings_url() {
let settings = super::DatabaseSettings {
url: "postgresql://user:pass@localhost/db".to_string(),
max_connections: Some(100),
min_connections: Some(10),
connect_timeout: Some(10),
idle_timeout: Some(300),
max_lifetime: Some(1800),
connection_keepalive: Some(30),
health_check_interval: Some(60),
prepare_cache_capacity: None,
auto_migrate: false,
};
assert_eq!(settings.url(), "postgresql://user:pass@localhost/db");
}
#[test]
fn test_database_settings_debug_redacts_url() {
let settings = super::DatabaseSettings {
url: "postgresql://secret:password@localhost/db".to_string(),
max_connections: Some(100),
min_connections: Some(10),
connect_timeout: Some(10),
idle_timeout: Some(300),
max_lifetime: Some(1800),
connection_keepalive: Some(30),
health_check_interval: Some(60),
prepare_cache_capacity: None,
auto_migrate: false,
};
let debug_str = format!("{:?}", settings);
assert!(debug_str.contains("***REDACTED***"));
assert!(!debug_str.contains("secret:password"));
}
#[test]
fn test_server_settings_construction() {
let settings = super::ServerSettings {
host: "127.0.0.1".to_string(),
port: 8080,
enable_port_detection: false,
};
assert_eq!(settings.host, "127.0.0.1");
assert_eq!(settings.port, 8080);
assert!(!settings.enable_port_detection);
}
#[test]
fn test_rate_limiting_settings_construction() {
let settings = super::RateLimitingSettings {
enabled: false,
default_rpm: 200,
burst_size: 50,
anon_ip_rpm: 10,
auth_ip_rpm: 600,
storage_backend: "memory".to_string(),
};
assert!(!settings.enabled);
assert_eq!(settings.default_rpm, 200);
assert_eq!(settings.burst_size, 50);
assert_eq!(settings.storage_backend, "memory");
}
#[test]
fn test_concurrency_settings_construction() {
let settings = super::ConcurrencySettings {
default_team_limit: 20,
task_lock_duration_seconds: 600,
mem_pressure_threshold: 0.75,
mem_critical_threshold: 0.85,
critical_timeout_seconds: 45,
adaptive_enabled: true,
};
assert_eq!(settings.default_team_limit, 20);
assert_eq!(settings.task_lock_duration_seconds, 600);
assert!((settings.mem_pressure_threshold - 0.75).abs() < f64::EPSILON);
assert!((settings.mem_critical_threshold - 0.85).abs() < f64::EPSILON);
assert_eq!(settings.critical_timeout_seconds, 45);
assert!(settings.adaptive_enabled);
}
#[test]
fn test_concurrency_settings_memory_defaults() {
let settings = super::ConcurrencySettings::default();
assert!((settings.mem_pressure_threshold - 0.8).abs() < f64::EPSILON);
assert!((settings.mem_critical_threshold - 0.9).abs() < f64::EPSILON);
assert_eq!(settings.critical_timeout_seconds, 30);
assert!(settings.mem_pressure_threshold < settings.mem_critical_threshold);
}
#[test]
fn test_concurrency_settings_adaptive_enabled_default_false() {
let settings = super::ConcurrencySettings::default();
assert!(
!settings.adaptive_enabled,
"adaptive_enabled must default to false for backward compatibility"
);
}
}