use serde::{Deserialize, Serialize};
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__BING_SEARCH__")]
pub struct BingSearchSettings {
pub(crate) api_key: Option<String>,
}
impl BingSearchSettings {
pub fn api_key(&self) -> Option<&str> {
self.api_key.as_deref()
}
}
impl std::fmt::Debug for BingSearchSettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BingSearchSettings")
.field("api_key", &"[REDACTED]")
.finish()
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__")]
pub struct SearchSettings {
#[config(default = 30)]
pub timeout_seconds: u64,
#[config(default = true)]
pub rate_limiting_enabled: bool,
#[config(default = false)]
pub test_data_enabled: bool,
#[config(default = 3)]
pub max_retries: u32,
#[config(default = 1000)]
pub retry_delay_ms: u64,
#[config(default = 50)]
pub max_results: u32,
#[serde(default)]
pub default_engine: Option<String>,
#[serde(default)]
pub engines: SearchEnginesSettings,
#[serde(default)]
pub exa: ExaEngineSettings,
#[serde(default)]
pub tavily: TavilyEngineSettings,
#[serde(default)]
pub searxng: SearxngSearchSettings,
pub fallback: SearchFallbackConfig,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__ENGINES__")]
pub struct SearchEnginesSettings {
#[config(default = true)]
pub bing_enabled: bool,
#[config(default = true)]
pub baidu_enabled: bool,
#[config(default = true)]
pub sogou_enabled: bool,
}
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__EXA__")]
pub struct ExaEngineSettings {
pub(crate) api_key: Option<String>,
pub(crate) base_url: Option<String>,
}
impl ExaEngineSettings {
pub fn api_key(&self) -> Option<&str> {
self.api_key.as_deref()
}
pub fn base_url(&self) -> Option<&str> {
self.base_url.as_deref()
}
}
impl std::fmt::Debug for ExaEngineSettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ExaEngineSettings")
.field("api_key", &"[REDACTED]")
.field("base_url", &self.base_url)
.finish()
}
}
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__TAVILY__")]
pub struct TavilyEngineSettings {
pub(crate) api_key: Option<String>,
pub(crate) base_url: Option<String>,
}
impl TavilyEngineSettings {
pub fn api_key(&self) -> Option<&str> {
self.api_key.as_deref()
}
pub fn base_url(&self) -> Option<&str> {
self.base_url.as_deref()
}
}
impl std::fmt::Debug for TavilyEngineSettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TavilyEngineSettings")
.field("api_key", &"[REDACTED]")
.field("base_url", &self.base_url)
.finish()
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__SEARXNG__")]
#[serde(default)]
pub struct SearxngSearchSettings {
#[config(default = false)]
pub enabled: bool,
#[config(default = 3)]
pub pool_target: usize,
#[config(default = 2)]
pub max_failover_attempts: u32,
#[config(default = Vec::<String>::new())]
pub seed_urls: Vec<String>,
#[config(default = "crawlrs-searxng/0.1 (+https://github.com/Kirky-X/crawlrs)".to_string())]
pub user_agent: String,
#[serde(default)]
pub collector: SearxngCollectorSettings,
#[serde(default)]
pub health: SearxngHealthSettings,
#[serde(default)]
pub breaker: SearxngBreakerSettings,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__SEARXNG__COLLECTOR__")]
#[serde(default)]
pub struct SearxngCollectorSettings {
#[config(default = 21600)]
pub interval_seconds: u64,
#[config(default = "https://searx.space/data/instances.json".to_string())]
pub data_url: String,
#[config(default = 30)]
pub request_timeout_seconds: u64,
#[config(default = 95.0)]
pub min_uptime_day: f64,
#[config(default = 4)]
pub retire_miss_cycles: i32,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__SEARXNG__HEALTH__")]
#[serde(default)]
pub struct SearxngHealthSettings {
#[config(default = 300)]
pub interval_seconds: u64,
#[config(default = 8)]
pub probe_timeout_seconds: u64,
#[config(default = "crawlrs".to_string())]
pub probe_query: String,
#[config(default = 10)]
pub candidate_batch: u64,
#[config(default = 5)]
pub failure_retire_threshold: i32,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__SEARXNG__BREAKER__")]
#[serde(default)]
pub struct SearxngBreakerSettings {
#[config(default = 3)]
pub failure_threshold: u32,
#[config(default = 120)]
pub recovery_timeout_seconds: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__FALLBACK__")]
pub struct SearchFallbackConfig {
#[config(default = false)]
pub enabled: bool,
#[config(default = vec!["exa".to_string(), "parallel".to_string(), "tavily".to_string()])]
pub engines: Vec<String>,
pub exa: FallbackEngineConfig,
pub parallel: FallbackEngineConfig,
pub tavily: FallbackEngineConfig,
}
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SEARCH__FALLBACK__")]
pub struct FallbackEngineConfig {
#[config(default = String::new())]
pub api_key: String,
#[config(default = String::new())]
pub endpoint: String,
}
impl std::fmt::Debug for FallbackEngineConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FallbackEngineConfig")
.field("api_key", &"[REDACTED]")
.field("endpoint", &self.endpoint)
.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_bing_default_api_key_is_none() {
let settings = BingSearchSettings::default();
assert!(
settings.api_key().is_none(),
"default api_key should be None"
);
}
#[test]
fn test_bing_api_key_returns_some_value() {
let settings = BingSearchSettings {
api_key: Some("secret-bing-key".to_string()),
};
assert_eq!(
settings.api_key(),
Some("secret-bing-key"),
"api_key() should return the stored key"
);
}
#[test]
fn test_bing_api_key_returns_none_when_empty() {
let settings = BingSearchSettings { api_key: None };
assert!(
settings.api_key().is_none(),
"api_key() should return None when not set"
);
}
#[test]
fn test_bing_serde_roundtrip_with_key() {
let settings = BingSearchSettings {
api_key: Some("key-123".to_string()),
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: BingSearchSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(
back.api_key(),
Some("key-123"),
"serde roundtrip should preserve api_key"
);
}
#[test]
fn test_bing_serde_roundtrip_without_key() {
let settings = BingSearchSettings::default();
let json = serde_json::to_string(&settings).expect("serialize");
let back: BingSearchSettings = serde_json::from_str(&json).expect("deserialize");
assert!(
back.api_key().is_none(),
"serde roundtrip should preserve None api_key"
);
}
#[test]
fn test_bing_clone_preserves_api_key() {
let settings = BingSearchSettings {
api_key: Some("cloned-key".to_string()),
};
let cloned = settings.clone();
assert_eq!(
cloned.api_key(),
Some("cloned-key"),
"clone should preserve api_key"
);
}
#[test]
fn test_bing_debug_does_not_panic() {
let settings = BingSearchSettings {
api_key: Some("debug-key".to_string()),
};
let debug = format!("{:?}", settings);
assert!(
debug.contains("BingSearchSettings"),
"Debug output should contain struct name"
);
}
#[test]
fn test_bing_debug_redacts_api_key() {
let settings = BingSearchSettings {
api_key: Some("fixture-bing-value".to_string()), };
let debug = format!("{settings:?}");
assert!(debug.contains("[REDACTED]"), "debug: {debug}");
assert!(
!debug.contains("fixture-bing-value"),
"api key leaked: {debug}"
);
}
#[test]
fn test_search_fallback_debug_redacts_api_keys() {
let settings = SearchFallbackConfig {
enabled: true,
engines: vec!["exa".to_string()],
exa: FallbackEngineConfig {
api_key: "fixture-exa-value".to_string(), endpoint: "https://exa.example".to_string(),
},
parallel: FallbackEngineConfig {
api_key: "fixture-parallel-value".to_string(), endpoint: String::new(),
},
tavily: FallbackEngineConfig {
api_key: "fixture-tavily-value".to_string(), endpoint: String::new(),
},
};
let debug = format!("{settings:?}");
assert!(debug.contains("SearchFallbackConfig"), "debug: {debug}");
assert!(
!debug.contains("fixture-exa-value"),
"exa key leaked: {debug}"
);
assert!(
!debug.contains("fixture-parallel-value"),
"parallel key leaked: {debug}"
);
assert!(
!debug.contains("fixture-tavily-value"),
"tavily key leaked: {debug}"
);
assert_eq!(
debug.matches("[REDACTED]").count(),
3,
"all three fallback keys must be redacted: {debug}"
);
assert!(
debug.contains("https://exa.example"),
"non-sensitive endpoint should stay visible: {debug}"
);
}
#[test]
fn test_search_default_timeout_seconds() {
let settings = SearchSettings::default();
assert_eq!(
settings.timeout_seconds, 30,
"default timeout_seconds should be 30"
);
}
#[test]
fn test_search_default_rate_limiting_enabled() {
let settings = SearchSettings::default();
assert!(
settings.rate_limiting_enabled,
"default rate_limiting_enabled should be true"
);
}
#[test]
fn test_search_default_test_data_disabled() {
let settings = SearchSettings::default();
assert!(
!settings.test_data_enabled,
"default test_data_enabled should be false"
);
}
#[test]
fn test_search_default_max_retries() {
let settings = SearchSettings::default();
assert_eq!(settings.max_retries, 3, "default max_retries should be 3");
}
#[test]
fn test_search_default_retry_delay_ms() {
let settings = SearchSettings::default();
assert_eq!(
settings.retry_delay_ms, 1000,
"default retry_delay_ms should be 1000"
);
}
#[test]
fn test_search_default_max_results() {
let settings = SearchSettings::default();
assert_eq!(settings.max_results, 50, "default max_results should be 50");
}
#[test]
fn test_search_serde_roundtrip_default() {
let settings = SearchSettings::default();
let json = serde_json::to_string(&settings).expect("serialize");
let back: SearchSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(
back.timeout_seconds, settings.timeout_seconds,
"serde roundtrip should preserve timeout_seconds"
);
assert_eq!(
back.max_retries, settings.max_retries,
"serde roundtrip should preserve max_retries"
);
}
#[test]
fn test_search_serde_roundtrip_custom_values() {
let settings = SearchSettings {
timeout_seconds: 60,
rate_limiting_enabled: false,
test_data_enabled: true,
max_retries: 10,
retry_delay_ms: 2000,
max_results: 100,
default_engine: None,
engines: SearchEnginesSettings::default(),
exa: ExaEngineSettings::default(),
tavily: TavilyEngineSettings::default(),
searxng: SearxngSearchSettings::default(),
fallback: SearchFallbackConfig::default(),
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: SearchSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.timeout_seconds, 60);
assert!(
!back.rate_limiting_enabled,
"rate_limiting_enabled should survive roundtrip"
);
assert!(back.test_data_enabled);
assert_eq!(back.max_retries, 10);
assert_eq!(back.retry_delay_ms, 2000);
assert_eq!(back.max_results, 100);
}
#[test]
fn test_search_clone_preserves_all_fields() {
let settings = SearchSettings {
timeout_seconds: 45,
rate_limiting_enabled: false,
test_data_enabled: true,
max_retries: 5,
retry_delay_ms: 500,
max_results: 75,
default_engine: None,
engines: SearchEnginesSettings::default(),
exa: ExaEngineSettings::default(),
tavily: TavilyEngineSettings::default(),
searxng: SearxngSearchSettings::default(),
fallback: SearchFallbackConfig::default(),
};
let cloned = settings.clone();
assert_eq!(cloned.timeout_seconds, settings.timeout_seconds);
assert_eq!(cloned.rate_limiting_enabled, settings.rate_limiting_enabled);
assert_eq!(cloned.test_data_enabled, settings.test_data_enabled);
assert_eq!(cloned.max_retries, settings.max_retries);
assert_eq!(cloned.retry_delay_ms, settings.retry_delay_ms);
assert_eq!(cloned.max_results, settings.max_results);
}
#[test]
fn test_fallback_default_disabled() {
let settings = SearchFallbackConfig::default();
assert!(!settings.enabled, "fallback should be disabled by default");
}
#[test]
fn test_fallback_default_engines_order() {
let settings = SearchFallbackConfig::default();
assert_eq!(settings.engines, vec!["exa", "parallel", "tavily"]);
}
#[test]
fn test_fallback_engine_config_default_empty() {
let config = FallbackEngineConfig::default();
assert_eq!(config.api_key, "");
assert_eq!(config.endpoint, "");
}
#[test]
fn test_fallback_serde_roundtrip() {
let settings = SearchFallbackConfig {
enabled: true,
engines: vec!["tavily".to_string(), "exa".to_string()],
exa: FallbackEngineConfig {
api_key: "exa-key".to_string(),
endpoint: "https://mcp.exa.ai/mcp".to_string(),
},
parallel: FallbackEngineConfig {
api_key: String::new(),
endpoint: "https://search.parallel.ai/mcp".to_string(),
},
tavily: FallbackEngineConfig {
api_key: "tavily-key".to_string(),
endpoint: "https://api.tavily.com".to_string(),
},
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: SearchFallbackConfig = serde_json::from_str(&json).expect("deserialize");
assert!(back.enabled);
assert_eq!(back.engines, vec!["tavily", "exa"]);
assert_eq!(back.exa.api_key, "exa-key");
assert_eq!(back.tavily.endpoint, "https://api.tavily.com");
}
#[test]
fn test_fallback_clone_preserves_fields() {
let settings = SearchFallbackConfig {
enabled: true,
engines: vec!["exa".to_string()],
exa: FallbackEngineConfig {
api_key: "key".to_string(),
endpoint: "https://example.com".to_string(),
},
parallel: FallbackEngineConfig::default(),
tavily: FallbackEngineConfig::default(),
};
let cloned = settings.clone();
assert_eq!(cloned.enabled, settings.enabled);
assert_eq!(cloned.engines, settings.engines);
assert_eq!(cloned.exa.api_key, settings.exa.api_key);
}
#[test]
fn test_search_settings_has_fallback_field() {
let settings = SearchSettings::default();
assert!(!settings.fallback.enabled);
assert_eq!(settings.fallback.engines.len(), 3);
}
#[test]
fn test_exa_engine_settings_default_is_none() {
let settings = ExaEngineSettings::default();
assert!(
settings.api_key().is_none(),
"default exa api_key should be None"
);
assert!(
settings.base_url().is_none(),
"default exa base_url should be None"
);
}
#[test]
fn test_exa_engine_settings_accessors_return_values() {
let settings = ExaEngineSettings {
api_key: Some("secret-exa-key".to_string()),
base_url: Some("https://mcp.example.com/mcp".to_string()),
};
assert_eq!(settings.api_key(), Some("secret-exa-key"));
assert_eq!(settings.base_url(), Some("https://mcp.example.com/mcp"));
}
#[test]
fn test_exa_engine_settings_serde_roundtrip() {
let settings = ExaEngineSettings {
api_key: Some("key-456".to_string()),
base_url: None,
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: ExaEngineSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.api_key(), Some("key-456"));
assert!(back.base_url().is_none());
}
#[test]
fn test_exa_engine_settings_missing_env_key_deserializes_to_none() {
let back: ExaEngineSettings = serde_json::from_str("{}").expect("deserialize");
assert!(back.api_key().is_none());
assert!(back.base_url().is_none());
}
#[test]
fn test_exa_engine_settings_debug_redacts_api_key() {
let settings = ExaEngineSettings {
api_key: Some("fixture-exa-engine".to_string()), base_url: Some("https://mcp.example.com/mcp".to_string()),
};
let debug = format!("{settings:?}");
assert!(debug.contains("ExaEngineSettings"), "debug: {debug}");
assert!(debug.contains("[REDACTED]"), "debug: {debug}");
assert!(
!debug.contains("fixture-exa-engine"),
"api key leaked: {debug}"
);
assert!(
debug.contains("https://mcp.example.com/mcp"),
"non-sensitive field should stay visible: {debug}"
);
}
#[test]
fn test_tavily_engine_settings_default_is_none() {
let settings = TavilyEngineSettings::default();
assert!(
settings.api_key().is_none(),
"default tavily api_key should be None"
);
assert!(
settings.base_url().is_none(),
"default tavily base_url should be None"
);
}
#[test]
fn test_tavily_engine_settings_accessors_return_values() {
let settings = TavilyEngineSettings {
api_key: Some("secret-tavily-key".to_string()),
base_url: Some("https://api.example.com".to_string()),
};
assert_eq!(settings.api_key(), Some("secret-tavily-key"));
assert_eq!(settings.base_url(), Some("https://api.example.com"));
}
#[test]
fn test_tavily_engine_settings_debug_redacts_api_key() {
let settings = TavilyEngineSettings {
api_key: Some("fixture-tavily-engine".to_string()), base_url: Some("https://api.tavily.com".to_string()),
};
let debug = format!("{settings:?}");
assert!(debug.contains("TavilyEngineSettings"), "debug: {debug}");
assert!(debug.contains("[REDACTED]"), "debug: {debug}");
assert!(
!debug.contains("fixture-tavily-engine"),
"api key leaked: {debug}"
);
assert!(
debug.contains("https://api.tavily.com"),
"non-sensitive field should stay visible: {debug}"
);
}
#[test]
fn test_tavily_engine_settings_serde_roundtrip() {
let settings = TavilyEngineSettings {
api_key: Some("key-789".to_string()),
base_url: Some("https://api.tavily.com".to_string()),
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: TavilyEngineSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.api_key(), Some("key-789"));
assert_eq!(back.base_url(), Some("https://api.tavily.com"));
}
#[test]
fn test_search_settings_default_engine_credentials_are_none() {
let settings = SearchSettings::default();
assert!(
settings.exa.api_key().is_none(),
"default SearchSettings should not configure exa api_key"
);
assert!(
settings.tavily.api_key().is_none(),
"default SearchSettings should not configure tavily api_key"
);
}
#[test]
fn test_search_settings_serde_roundtrip_preserves_none_credentials() {
let settings = SearchSettings::default();
let json = serde_json::to_string(&settings).expect("serialize");
let back: SearchSettings = serde_json::from_str(&json).expect("deserialize");
assert!(back.exa.api_key().is_none());
assert!(back.tavily.api_key().is_none());
}
#[test]
fn test_searxng_settings_defaults() {
let settings = SearxngSearchSettings::default();
assert!(!settings.enabled, "searxng must be opt-in (enabled=false)");
assert_eq!(settings.pool_target, 3);
assert_eq!(settings.max_failover_attempts, 2);
assert!(settings.seed_urls.is_empty());
assert!(!settings.user_agent.trim().is_empty());
}
#[test]
fn test_searxng_sub_settings_defaults() {
let settings = SearxngSearchSettings::default();
assert_eq!(settings.collector.interval_seconds, 21600);
assert_eq!(
settings.collector.data_url,
"https://searx.space/data/instances.json"
);
assert_eq!(settings.collector.request_timeout_seconds, 30);
assert_eq!(settings.collector.min_uptime_day, 95.0);
assert_eq!(settings.collector.retire_miss_cycles, 4);
assert_eq!(settings.health.interval_seconds, 300);
assert_eq!(settings.health.probe_timeout_seconds, 8);
assert_eq!(settings.health.probe_query, "crawlrs");
assert_eq!(settings.health.candidate_batch, 10);
assert_eq!(settings.health.failure_retire_threshold, 5);
assert_eq!(settings.breaker.failure_threshold, 3);
assert_eq!(settings.breaker.recovery_timeout_seconds, 120);
}
#[test]
fn test_searxng_settings_serde_roundtrip() {
let settings = SearxngSearchSettings {
enabled: true,
pool_target: 5,
max_failover_attempts: 4,
seed_urls: vec!["https://searx.example.org/".to_string()],
collector: SearxngCollectorSettings {
min_uptime_day: 90.0,
..SearxngCollectorSettings::default()
},
health: SearxngHealthSettings {
failure_retire_threshold: 7,
..SearxngHealthSettings::default()
},
breaker: SearxngBreakerSettings {
failure_threshold: 2,
..SearxngBreakerSettings::default()
},
..SearxngSearchSettings::default()
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: SearxngSearchSettings = serde_json::from_str(&json).expect("deserialize");
assert!(back.enabled);
assert_eq!(back.pool_target, 5);
assert_eq!(back.max_failover_attempts, 4);
assert_eq!(back.seed_urls, vec!["https://searx.example.org/"]);
assert_eq!(back.collector.min_uptime_day, 90.0);
assert_eq!(back.health.failure_retire_threshold, 7);
assert_eq!(back.breaker.failure_threshold, 2);
}
#[test]
fn test_searxng_settings_missing_section_equals_default() {
let back: SearxngSearchSettings = serde_json::from_str("{}").expect("deserialize");
let default = SearxngSearchSettings::default();
assert_eq!(back.enabled, default.enabled);
assert_eq!(back.pool_target, default.pool_target);
assert_eq!(back.max_failover_attempts, default.max_failover_attempts);
assert_eq!(back.collector.data_url, default.collector.data_url);
assert_eq!(
back.health.interval_seconds,
default.health.interval_seconds
);
assert_eq!(
back.breaker.failure_threshold,
default.breaker.failure_threshold
);
}
#[test]
fn test_search_settings_default_searxng_disabled() {
let settings = SearchSettings::default();
assert!(!settings.searxng.enabled);
assert_eq!(settings.searxng.pool_target, 3);
}
}