use sdforge::validator::Validate;
use serde::{Deserialize, Serialize};
pub use super::app::{
ConcurrencySettings, DatabaseSettings, RateLimitingSettings, RetentionSettings, ServerSettings,
WebSocketSettings,
};
pub use super::engines::{
EngineSettings, FlareSolverrCdpSettings, FlareSolverrSettings, FlareSolverrTlsSettings,
PlaywrightEngineSettings,
};
pub use super::fluxweir::FluxweirSettings;
pub use super::llm::{LLMSettings, LlmChunkingSettings};
pub use super::logging::{ConsoleLoggingSettings, FileLoggingSettings, LoggingSettings};
pub use super::rag::RagSettings;
pub use super::search::{
BingSearchSettings, ExaEngineSettings, FallbackEngineConfig, SearchFallbackConfig,
SearchSettings, TavilyEngineSettings,
};
#[derive(Debug, Clone, Deserialize, Serialize, Validate, confers::Config)]
#[validate(crate = "sdforge::validator")]
#[config(env_prefix = "CRAWLRS__", validate)]
pub struct Settings {
pub server: ServerSettings,
pub database: DatabaseSettings,
pub cors: CorsSettings,
pub rate_limiting: RateLimitingSettings,
pub concurrency: ConcurrencySettings,
pub retention: RetentionSettings,
pub webhook: WebhookSettings,
pub bing_search: BingSearchSettings,
pub search: SearchSettings,
pub llm: LLMSettings,
pub rag: RagSettings,
pub fluxweir: FluxweirSettings,
pub proxy: ProxySettings,
pub engines: EngineSettings,
pub markdown: MarkdownSettings,
#[validate(nested)]
pub robots: RobotsSettings,
#[validate(nested)]
pub politeness: PolitenessSettings,
#[validate(nested)]
pub session: SessionSettings,
pub logging: LoggingSettings,
pub workers: WorkerSettings,
#[serde(default)]
pub sitemap: SitemapSettings,
#[serde(default)]
pub crawl: CrawlSettings,
#[validate(nested)]
pub timeouts: TimeoutSettings,
pub cache: CacheSettings,
#[serde(default)]
pub websocket: WebSocketSettings,
pub trusted_proxies: TrustedProxySettings,
#[serde(default)]
pub observability: ObservabilitySettings,
pub auth: AuthSettings,
pub i18n: I18nSettings,
#[serde(default)]
pub content_extraction: ContentExtractionSettings,
#[serde(default)]
pub id: IdSettings,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__ID__")]
pub struct IdSettings {
#[config(default = "uuid_v8".to_string())]
pub algorithm: String,
#[config(default = "crawlrs".to_string())]
pub workspace: String,
#[config(default = "default".to_string())]
pub group: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, Validate, confers::Config)]
#[validate(crate = "sdforge::validator")]
#[config(env_prefix = "CRAWLRS__ROBOTS__")]
pub struct RobotsSettings {
#[config(default = "crawlrs-bot/1.0".to_string())]
#[validate(length(min = 1, max = 128))]
pub user_agent: String,
#[config(default = false)]
pub scrape_respect_robots: bool,
#[serde(default)]
#[config(default = false)]
pub fail_closed: bool,
#[config(default = 3600)]
pub cache_ttl_secs: u64,
#[config(default = 5)]
pub fetch_max_retries: u32,
#[config(default = 2)]
pub fetch_initial_backoff_secs: u64,
#[config(default = 10)]
pub fetch_max_backoff_secs: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum MarkdownConverterKind {
#[default]
Alchemize,
Htmd,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__MARKDOWN__")]
pub struct MarkdownSettings {
#[config(default = "alchemize".to_string())]
pub converter: String,
}
impl MarkdownSettings {
pub fn converter_kind(&self) -> Result<MarkdownConverterKind, String> {
match self.converter.to_ascii_lowercase().as_str() {
"alchemize" => Ok(MarkdownConverterKind::Alchemize),
"htmd" => Ok(MarkdownConverterKind::Htmd),
other => Err(format!(
"invalid [markdown].converter = {other:?}; expected \"alchemize\" or \"htmd\""
)),
}
}
}
#[derive(Debug, Clone, Deserialize, Serialize, Validate, confers::Config)]
#[validate(crate = "sdforge::validator")]
#[config(env_prefix = "CRAWLRS__POLITENESS__")]
pub struct PolitenessSettings {
#[config(default = 1000)]
#[validate(range(min = 0, max = 3_600_000))]
pub min_interval_ms: u64,
#[config(default = 60_000)]
#[validate(range(min = 1, max = 3_600_000))]
pub max_interval_ms: u64,
#[config(default = 60)]
#[validate(range(min = 0, max = 86_400))]
pub cooldown_default_secs: u64,
#[config(default = 3)]
#[validate(range(min = 1, max = 100))]
pub bare403_threshold: u32,
#[config(default = false)]
pub autothrottle_enabled: bool,
#[config(default = 8)]
#[validate(range(min = 1, max = 1024))]
pub target_concurrency: u32,
#[config(default = 2)]
#[validate(range(min = 0, max = 3600))]
pub backoff_base_secs: u64,
#[config(default = 60)]
#[validate(range(min = 1, max = 86_400))]
pub backoff_max_secs: u64,
#[config(default = 3)]
#[validate(range(min = 0, max = 100))]
pub stalled_threshold: u32,
#[config(default = 0.0)]
pub jitter_ratio: f64,
}
#[derive(Debug, Clone, Deserialize, Serialize, Validate, confers::Config)]
#[validate(crate = "sdforge::validator")]
#[config(env_prefix = "CRAWLRS__SESSION__")]
pub struct SessionSettings {
#[config(default = false)]
pub enabled: bool,
#[config(default = 16)]
#[validate(range(min = 1, max = 10_000))]
pub max_sessions: usize,
#[config(default = 3.0)]
#[validate(range(min = 0.0, max = 1_000_000.0))]
pub max_error_score: f64,
#[config(default = 100)]
#[validate(range(min = 1, max = 1_000_000))]
pub max_uses: u64,
#[config(default = 1800)]
#[validate(range(min = 0, max = 86_400))]
pub max_age_secs: u64,
#[config(default = 16384)]
#[validate(range(min = 64, max = 1_048_576))]
pub cookie_cache_max_bytes: usize,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CORS__")]
pub struct CorsSettings {
#[config(default = "*".to_string())]
pub allowed_origins: String,
}
impl CorsSettings {
pub fn allowed_origins_list(&self) -> Vec<String> {
self.allowed_origins
.split(',')
.map(str::trim)
.filter(|origin| !origin.is_empty())
.map(str::to_string)
.collect()
}
}
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__WEBHOOK__")]
pub struct WebhookSettings {
pub(crate) secret: String,
#[config(default = 5)]
pub max_retries: u32,
#[config(default = 1000)]
pub batch_size: usize,
}
impl std::fmt::Debug for WebhookSettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WebhookSettings")
.field("secret", &"[REDACTED]")
.field("max_retries", &self.max_retries)
.field("batch_size", &self.batch_size)
.finish()
}
}
impl WebhookSettings {
pub fn secret(&self) -> &str {
&self.secret
}
}
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__AUTH__")]
pub struct AuthSettings {
#[config(default = "".to_string())]
pub(crate) jwt_secret: String,
}
impl std::fmt::Debug for AuthSettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AuthSettings")
.field("jwt_secret", &"***REDACTED***")
.finish()
}
}
impl AuthSettings {
pub fn jwt_secret(&self) -> &str {
&self.jwt_secret
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum ProxyStrategy {
#[default]
RoundRobin,
Sticky,
}
#[derive(Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__PROXY__")]
pub struct ProxySettings {
#[config(default = Vec::<String>::new())]
pub urls: Vec<String>,
#[serde(default)]
pub strategy: ProxyStrategy,
#[config(default = false)]
pub enabled: bool,
#[config(default = 60)]
#[serde(default = "default_sticky_ttl_seconds")]
pub sticky_ttl_seconds: u64,
#[config(default = 30)]
#[serde(default = "default_cooldown_seconds")]
pub cooldown_seconds: u64,
}
fn default_sticky_ttl_seconds() -> u64 {
60
}
fn default_cooldown_seconds() -> u64 {
30
}
impl std::fmt::Debug for ProxySettings {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ProxySettings")
.field("urls", &"***REDACTED***")
.field("strategy", &self.strategy)
.field("enabled", &self.enabled)
.field("sticky_ttl_seconds", &self.sticky_ttl_seconds)
.field("cooldown_seconds", &self.cooldown_seconds)
.finish()
}
}
impl ProxySettings {
pub fn urls(&self) -> &[String] {
&self.urls
}
pub fn strategy(&self) -> ProxyStrategy {
self.strategy
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__WORKERS__")]
pub struct WorkerSettings {
pub count: WorkerCount,
#[config(default = 30)]
pub graceful_shutdown_seconds: u64,
#[serde(default)]
#[config(default = false)]
pub dedup_warmup_include_results: bool,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(untagged)]
pub enum WorkerCount {
Auto(String),
Fixed(usize),
}
impl Default for WorkerCount {
fn default() -> Self {
WorkerCount::Auto("auto".to_string())
}
}
impl WorkerCount {
pub fn resolve(&self) -> usize {
match self {
WorkerCount::Auto(s) if s.eq_ignore_ascii_case("auto") => {
let logical_cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4);
logical_cores * 2
}
WorkerCount::Fixed(n) => *n,
_ => 5,
}
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__SITEMAP__")]
pub struct SitemapSettings {
#[serde(default)]
#[config(default = false)]
pub enabled: bool,
#[serde(default = "default_sitemap_max_urls")]
#[config(default = 50_000)]
pub max_urls: usize,
#[serde(default = "default_sitemap_max_concurrent_sub_sitemaps")]
#[config(default = 4)]
pub max_concurrent_sub_sitemaps: usize,
#[serde(default = "default_sitemap_channel_capacity")]
#[config(default = 4096)]
pub channel_capacity: usize,
#[serde(default = "default_sitemap_fetch_timeout_secs")]
#[config(default = 30)]
pub fetch_timeout_secs: u64,
}
fn default_sitemap_max_urls() -> usize {
50_000
}
fn default_sitemap_max_concurrent_sub_sitemaps() -> usize {
4
}
fn default_sitemap_channel_capacity() -> usize {
4096
}
fn default_sitemap_fetch_timeout_secs() -> u64 {
30
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CRAWL__")]
pub struct CrawlSettings {
#[serde(default = "default_soft404_enabled")]
#[config(default = true)]
pub soft404_enabled: bool,
#[serde(default = "default_soft404_max_fingerprints")]
#[config(default = 10_000)]
pub soft404_max_fingerprints: usize,
}
fn default_soft404_enabled() -> bool {
true
}
fn default_soft404_max_fingerprints() -> usize {
10_000
}
#[derive(Debug, Clone, Deserialize, Serialize, Validate, confers::Config)]
#[validate(crate = "sdforge::validator")]
#[config(env_prefix = "CRAWLRS__TIMEOUTS__")]
pub struct TimeoutSettings {
pub workers: WorkerTimeoutSettings,
#[validate(nested)]
pub engines: EngineTimeoutSettings,
pub retry: RetryTimeoutSettings,
pub cache: CacheTimeoutSettings,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__TIMEOUTS__WORKERS__")]
pub struct WorkerTimeoutSettings {
#[config(default = 5)]
pub webhook_interval_seconds: u64,
#[config(default = 30)]
pub backlog_interval_seconds: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, Validate, confers::Config)]
#[validate(crate = "sdforge::validator")]
#[config(env_prefix = "CRAWLRS__TIMEOUTS__ENGINES__")]
pub struct EngineTimeoutSettings {
#[config(default = 30)]
#[validate(range(min = 1, max = 600))]
pub default_timeout_seconds: u64,
#[config(default = 30)]
#[validate(range(min = 1, max = 600))]
pub playwright_timeout_seconds: u64,
#[config(default = 30)]
#[validate(range(min = 1, max = 600))]
pub flaresolverr_timeout_seconds: u64,
#[config(default = 5)]
#[serde(default = "default_fetch_seconds")]
#[validate(range(min = 1, max = 600))]
pub fetch_seconds: u64,
#[config(default = 15)]
#[serde(default = "default_tls_seconds")]
#[validate(range(min = 1, max = 600))]
pub tls_seconds: u64,
#[config(default = 30)]
#[serde(default = "default_cdp_seconds")]
#[validate(range(min = 1, max = 600))]
pub cdp_seconds: u64,
}
fn default_fetch_seconds() -> u64 {
5
}
fn default_tls_seconds() -> u64 {
15
}
fn default_cdp_seconds() -> u64 {
30
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__TIMEOUTS__RETRY__")]
pub struct RetryTimeoutSettings {
#[config(default = 1)]
pub initial_backoff_seconds: u64,
#[config(default = 60)]
pub max_backoff_seconds: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__TIMEOUTS__CACHE__")]
pub struct CacheTimeoutSettings {
#[config(default = 600)]
pub default_ttl_seconds: u64,
#[config(default = 600)]
pub memory_ttl_seconds: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CACHE__TYPES__")]
pub struct CacheTypeSettings {
#[config(default = 300)]
pub ttl_seconds: u64,
#[config(default = 10000)]
pub max_size: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CACHE__")]
pub struct CacheSettings {
#[config(default = true)]
pub enabled: bool,
#[serde(default = "default_cache_backend")]
#[config(default = "memory".to_string())]
pub backend: String,
#[serde(default)]
#[config(default = "".to_string())]
pub redis_url: String,
pub memory: MemoryCacheSettings,
pub types: CacheTypeSpecificSettings,
}
fn default_cache_backend() -> String {
"memory".to_string()
}
impl CacheSettings {
pub fn validate_backend(&self) -> Result<(), sdforge::validator::ValidationError> {
match self.backend.as_str() {
"memory" => Ok(()),
"redis" => self.validate_redis_backend(),
_ => Err(sdforge::validator::ValidationError::new(
"cache_backend_unknown",
)),
}
}
#[cfg(not(feature = "cache-redis"))]
fn validate_redis_backend(&self) -> Result<(), sdforge::validator::ValidationError> {
Err(sdforge::validator::ValidationError::new(
"cache_backend_redis_feature_not_compiled",
))
}
#[cfg(feature = "cache-redis")]
fn validate_redis_backend(&self) -> Result<(), sdforge::validator::ValidationError> {
if self.redis_url.trim().is_empty() {
return Err(sdforge::validator::ValidationError::new(
"cache_backend_redis_url_missing",
));
}
Ok(())
}
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CACHE__MEMORY__")]
pub struct MemoryCacheSettings {
#[config(default = 10000)]
pub capacity: u64,
#[config(default = 300)]
pub ttl_seconds: u64,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CACHE__TYPES__")]
pub struct CacheTypeSpecificSettings {
pub search: CacheTypeSettings,
pub dns: CacheTypeSettings,
pub regex: CacheTypeSettings,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__TRUSTED_PROXIES__")]
pub struct TrustedProxySettings {
#[config(default = true)]
pub enabled: bool,
#[config(default = vec![
"10.0.0.0/8".to_string(),
"172.16.0.0/12".to_string(),
"192.168.0.0/16".to_string(),
"127.0.0.1".to_string(),
"::1".to_string(),
])]
pub proxies: Vec<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__OBSERVABILITY__")]
pub struct ObservabilitySettings {
#[serde(default)]
#[config(default = "".to_string())]
pub failure_snapshot_dir: String,
#[serde(default = "default_failure_snapshot_max_total")]
#[config(default = 200)]
pub failure_snapshot_max_total: usize,
}
fn default_failure_snapshot_max_total() -> usize {
200
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__I18N__")]
pub struct I18nSettings {
#[config(default = "en-US".to_string())]
pub default_locale: String,
#[config(default = vec!["en-US".to_string(), "zh-CN".to_string()])]
pub supported_locales: Vec<String>,
}
pub fn validate_i18n(settings: &I18nSettings) -> Result<(), sdforge::validator::ValidationError> {
use unic_langid::LanguageIdentifier;
settings
.default_locale
.parse::<LanguageIdentifier>()
.map_err(|_| sdforge::validator::ValidationError::new("i18n_invalid_default_locale"))?;
for locale in &settings.supported_locales {
locale.parse::<LanguageIdentifier>().map_err(|_| {
sdforge::validator::ValidationError::new("i18n_invalid_supported_locale")
})?;
}
if !settings
.supported_locales
.contains(&settings.default_locale)
{
return Err(sdforge::validator::ValidationError::new(
"i18n_default_locale_not_in_supported",
));
}
Ok(())
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__CONTENT_EXTRACTION__")]
pub struct ContentExtractionSettings {
#[serde(default = "default_content_extraction_min_word_count")]
#[config(default = 150)]
pub min_word_count: usize,
#[serde(default = "default_content_extraction_replacement_length_ratio")]
#[config(default = 1.5)]
pub replacement_length_ratio: f64,
#[serde(default = "default_content_extraction_prose_min_sentence_words")]
#[config(default = 7.0)]
pub prose_min_sentence_words: f64,
#[serde(default = "default_content_extraction_prose_max_link_density")]
#[config(default = 0.7)]
pub prose_max_link_density: f64,
}
fn default_content_extraction_min_word_count() -> usize {
150
}
fn default_content_extraction_replacement_length_ratio() -> f64 {
1.5
}
fn default_content_extraction_prose_min_sentence_words() -> f64 {
7.0
}
fn default_content_extraction_prose_max_link_density() -> f64 {
0.7
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
#[test]
fn test_settings_structure() {
let settings = Settings {
server: ServerSettings::default(),
database: DatabaseSettings::default(),
cors: CorsSettings::default(),
rate_limiting: RateLimitingSettings::default(),
concurrency: ConcurrencySettings::default(),
retention: RetentionSettings::default(),
webhook: WebhookSettings::default(),
bing_search: BingSearchSettings::default(),
search: SearchSettings::default(),
llm: LLMSettings::default(),
rag: RagSettings::default(),
fluxweir: FluxweirSettings::default(),
proxy: ProxySettings::default(),
engines: EngineSettings::default(),
markdown: MarkdownSettings::default(),
robots: RobotsSettings::default(),
logging: LoggingSettings::default(),
workers: WorkerSettings::default(),
sitemap: SitemapSettings::default(),
crawl: CrawlSettings::default(),
timeouts: TimeoutSettings::default(),
cache: CacheSettings::default(),
websocket: WebSocketSettings::default(),
trusted_proxies: TrustedProxySettings::default(),
observability: ObservabilitySettings::default(),
auth: AuthSettings::default(),
i18n: I18nSettings::default(),
politeness: PolitenessSettings::default(),
session: SessionSettings::default(),
content_extraction: ContentExtractionSettings::default(),
id: IdSettings::default(),
};
assert_eq!(settings.server.port, 8899);
assert!(settings.rate_limiting.enabled);
assert!(settings.trusted_proxies.enabled);
assert_eq!(settings.i18n.default_locale, "en-US");
assert_eq!(settings.i18n.supported_locales.len(), 2);
assert_eq!(settings.robots.user_agent, "crawlrs-bot/1.0");
assert!(!settings.robots.scrape_respect_robots);
assert!(!settings.robots.fail_closed);
assert_eq!(settings.politeness.min_interval_ms, 1000);
assert!(!settings.politeness.autothrottle_enabled);
assert!(!settings.session.enabled);
}
#[test]
fn test_session_settings_defaults_match_spec() {
let s = SessionSettings::default();
assert!(!s.enabled, "默认关:保持现状盲轮换,灰度开启");
assert_eq!(s.max_sessions, 16, "池容量上限(内存护栏)");
assert_eq!(s.max_error_score, 3.0, "退休分数阈值");
assert_eq!(s.max_uses, 100, "单身份使用上限");
assert_eq!(s.max_age_secs, 1800, "身份最长寿命");
assert_eq!(s.cookie_cache_max_bytes, 16384, "Cookie 缓存上限");
}
#[test]
fn test_session_settings_toml_parses_all_keys() {
let toml = r#"
enabled = true
max_sessions = 32
max_error_score = 5.0
max_uses = 200
max_age_secs = 3600
cookie_cache_max_bytes = 32768
"#;
let s: SessionSettings = toml::from_str(toml).expect("deserialize");
assert!(s.enabled);
assert_eq!(s.max_sessions, 32);
assert_eq!(s.max_error_score, 5.0);
assert_eq!(s.max_uses, 200);
assert_eq!(s.max_age_secs, 3600);
assert_eq!(s.cookie_cache_max_bytes, 32768);
}
#[test]
fn test_content_extraction_settings_defaults_match_spec() {
let s = ContentExtractionSettings::default();
assert_eq!(s.min_word_count, 150, "短结果升级阈值");
assert!(
(s.replacement_length_ratio - 1.5).abs() < f64::EPSILON,
"替换裁决长度比"
);
assert!(
(s.prose_min_sentence_words - 7.0).abs() < f64::EPSILON,
"句均词数下限"
);
assert!(
(s.prose_max_link_density - 0.7).abs() < f64::EPSILON,
"链接密度上限"
);
}
#[test]
fn test_content_extraction_settings_toml_parses_all_keys() {
let toml = r#"
min_word_count = 200
replacement_length_ratio = 1.8
prose_min_sentence_words = 9
prose_max_link_density = 0.5
"#;
let s: ContentExtractionSettings = toml::from_str(toml).expect("deserialize");
assert_eq!(s.min_word_count, 200);
assert!((s.replacement_length_ratio - 1.8).abs() < f64::EPSILON);
assert!((s.prose_min_sentence_words - 9.0).abs() < f64::EPSILON);
assert!((s.prose_max_link_density - 0.5).abs() < f64::EPSILON);
}
#[test]
fn test_politeness_settings_defaults_match_spec() {
let p = PolitenessSettings::default();
assert_eq!(p.min_interval_ms, 1000, "收敛下界对齐 DEFAULT_MIN_INTERVAL");
assert_eq!(p.max_interval_ms, 60_000, "收敛上界");
assert_eq!(p.cooldown_default_secs, 60, "对齐 DEFAULT_COOLDOWN");
assert_eq!(p.bare403_threshold, 3, "对齐 DEFAULT_BARE403_THRESHOLD");
assert!(!p.autothrottle_enabled, "默认关,保持现网行为不变");
assert_eq!(p.target_concurrency, 8);
assert_eq!(p.backoff_base_secs, 2, "429 指数退避底值");
assert_eq!(p.backoff_max_secs, 60, "指数封顶");
assert_eq!(p.stalled_threshold, 3);
assert_eq!(p.jitter_ratio, 0.0, "0=确定性时刻表(现状)");
}
#[test]
fn test_politeness_settings_toml_parses_all_keys() {
let toml = r#"
min_interval_ms = 500
max_interval_ms = 30000
cooldown_default_secs = 30
bare403_threshold = 5
autothrottle_enabled = true
target_concurrency = 4
backoff_base_secs = 1
backoff_max_secs = 120
stalled_threshold = 2
jitter_ratio = 0.25
"#;
let p: PolitenessSettings = toml::from_str(toml).expect("deserialize");
assert_eq!(p.min_interval_ms, 500);
assert_eq!(p.max_interval_ms, 30_000);
assert_eq!(p.cooldown_default_secs, 30);
assert_eq!(p.bare403_threshold, 5);
assert!(p.autothrottle_enabled);
assert_eq!(p.target_concurrency, 4);
assert_eq!(p.backoff_base_secs, 1);
assert_eq!(p.backoff_max_secs, 120);
assert_eq!(p.stalled_threshold, 2);
assert!((p.jitter_ratio - 0.25).abs() < f64::EPSILON);
}
#[tokio::test]
async fn test_politeness_settings_to_host_discipline() {
use crate::utils::host_discipline::HostDiscipline;
let p = PolitenessSettings {
min_interval_ms: 250,
max_interval_ms: 5000,
cooldown_default_secs: 45,
bare403_threshold: 2,
autothrottle_enabled: true,
target_concurrency: 4,
backoff_base_secs: 3,
backoff_max_secs: 90,
stalled_threshold: 4,
jitter_ratio: 0.0,
};
let d = HostDiscipline::from_politeness(&p);
d.gate("wire.example.com").await;
let start = std::time::Instant::now();
d.gate("wire.example.com").await;
let elapsed = start.elapsed();
assert!(
elapsed >= Duration::from_millis(230),
"from_politeness must honor min_interval_ms=250, took {elapsed:?}"
);
assert!(
elapsed <= Duration::from_millis(400),
"gate gap must stay at min_interval, took {elapsed:?}"
);
}
#[test]
fn test_robots_settings_serde_default_for_fail_closed() {
let old_toml = r#"
user_agent = "crawlrs-bot/1.0"
scrape_respect_robots = false
cache_ttl_secs = 3600
fetch_max_retries = 5
fetch_initial_backoff_secs = 2
fetch_max_backoff_secs = 10
"#;
let settings: RobotsSettings = toml::from_str(old_toml).expect("deserialize");
assert!(
!settings.fail_closed,
"缺键时应回退 serde 默认 false(fail-open)"
);
let enabled: RobotsSettings = toml::from_str(
r#"
user_agent = "crawlrs-bot/1.0"
scrape_respect_robots = false
cache_ttl_secs = 3600
fetch_max_retries = 5
fetch_initial_backoff_secs = 2
fetch_max_backoff_secs = 10
fail_closed = true
"#,
)
.expect("deserialize override");
assert!(enabled.fail_closed, "显式 fail_closed=true 应生效");
}
#[test]
fn test_auth_settings_debug_redacts_jwt_secret() {
let auth = AuthSettings {
jwt_secret: "super-secret-jwt-key-32-bytes-or-more!!".to_string(),
};
let debug_output = format!("{:?}", auth);
assert!(
!debug_output.contains("super-secret-jwt-key-32-bytes-or-more!!"),
"Debug output must not contain plaintext jwt_secret, got: {debug_output}"
);
assert!(
debug_output.contains("***REDACTED***"),
"Debug output should contain '***REDACTED***' placeholder, got: {debug_output}"
);
}
#[test]
fn test_webhook_settings_debug_redacts_secret() {
let settings = WebhookSettings {
secret: "fixture-webhook-value".to_string(), max_retries: 7,
batch_size: 64,
};
let debug = format!("{settings:?}");
assert!(debug.contains("WebhookSettings"), "debug: {debug}");
assert!(debug.contains("[REDACTED]"), "debug: {debug}");
assert!(
!debug.contains("fixture-webhook-value"),
"secret leaked: {debug}"
);
assert!(
debug.contains("7"),
"non-sensitive retries visible: {debug}"
);
assert!(debug.contains("64"), "non-sensitive batch visible: {debug}");
}
#[test]
fn test_observability_settings_defaults_match_spec() {
let o = ObservabilitySettings::default();
assert_eq!(o.failure_snapshot_dir, "", "默认空串 = 禁用,不影响现状");
assert_eq!(o.failure_snapshot_max_total, 200, "文件总数护栏默认 200");
}
#[test]
fn test_observability_settings_toml_parses_all_keys() {
let toml = r#"
failure_snapshot_dir = "data/failure_snapshots"
failure_snapshot_max_total = 500
"#;
let o: ObservabilitySettings = toml::from_str(toml).expect("deserialize");
assert_eq!(o.failure_snapshot_dir, "data/failure_snapshots");
assert_eq!(o.failure_snapshot_max_total, 500);
}
#[test]
fn test_observability_settings_missing_section_falls_back_to_defaults() {
let o: ObservabilitySettings = toml::from_str("").expect("empty toml with serde defaults");
assert_eq!(o.failure_snapshot_dir, "");
assert_eq!(o.failure_snapshot_max_total, 200);
}
#[test]
fn test_worker_count_fixed_resolve() {
let count = WorkerCount::Fixed(8);
assert_eq!(count.resolve(), 8);
}
#[test]
fn test_worker_count_auto_resolve_positive() {
let count = WorkerCount::Auto("auto".to_string());
let resolved = count.resolve();
assert!(
resolved > 0,
"Auto resolve should be positive, got {}",
resolved
);
}
#[test]
fn test_worker_count_auto_case_insensitive() {
let count = WorkerCount::Auto("AUTO".to_string());
let resolved = count.resolve();
assert!(resolved > 0, "AUTO should resolve to cpu-based value");
}
#[test]
fn test_worker_count_auto_non_auto_string_falls_to_default() {
let count = WorkerCount::Auto("not-auto".to_string());
assert_eq!(count.resolve(), 5);
}
#[test]
fn test_worker_count_default_is_auto() {
let count = WorkerCount::default();
assert!(matches!(count, WorkerCount::Auto(_)));
}
#[test]
fn test_worker_count_clone() {
let count = WorkerCount::Fixed(12);
let cloned = count.clone();
assert_eq!(cloned.resolve(), 12);
}
#[test]
fn test_worker_count_serde_fixed() {
let count = WorkerCount::Fixed(7);
let json = serde_json::to_string(&count).expect("serialize");
let back: WorkerCount = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.resolve(), 7);
}
#[test]
fn test_worker_count_serde_auto() {
let json = "\"auto\"";
let count: WorkerCount = serde_json::from_str(json).expect("deserialize");
assert!(matches!(count, WorkerCount::Auto(_)));
}
#[test]
fn test_cors_settings_default_allowed_origins() {
let settings = CorsSettings::default();
assert_eq!(settings.allowed_origins, "*");
}
#[test]
fn test_cors_settings_construction() {
let settings = CorsSettings {
allowed_origins: "https://example.com".to_string(),
};
assert_eq!(settings.allowed_origins, "https://example.com");
}
#[test]
fn test_cors_settings_clone() {
let settings = CorsSettings {
allowed_origins: "https://test.com,https://api.com".to_string(),
};
let cloned = settings.clone();
assert_eq!(cloned.allowed_origins, settings.allowed_origins);
}
#[test]
fn test_cors_settings_serde_roundtrip() {
let settings = CorsSettings {
allowed_origins: "https://roundtrip.com".to_string(),
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: CorsSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.allowed_origins, "https://roundtrip.com");
}
#[test]
fn test_proxy_settings_default_empty_urls_and_round_robin_strategy() {
let settings = ProxySettings::default();
assert!(settings.urls.is_empty(), "default urls should be empty");
assert_eq!(settings.strategy, ProxyStrategy::RoundRobin);
assert!(!settings.enabled);
assert_eq!(settings.sticky_ttl_seconds, 60);
assert_eq!(settings.cooldown_seconds, 30);
}
#[test]
fn test_proxy_settings_urls_and_strategy_accessor() {
let settings = ProxySettings {
urls: vec!["http://proxy.example.com:8080".to_string()],
strategy: ProxyStrategy::Sticky,
enabled: true,
sticky_ttl_seconds: 120,
cooldown_seconds: 45,
};
assert_eq!(settings.urls(), &["http://proxy.example.com:8080"]);
assert_eq!(settings.strategy(), ProxyStrategy::Sticky);
assert!(settings.enabled);
}
#[test]
fn test_proxy_settings_debug_redacts_urls() {
let settings = ProxySettings {
urls: vec!["http://secret:password@proxy:8080".to_string()],
strategy: ProxyStrategy::RoundRobin,
enabled: true,
sticky_ttl_seconds: 60,
cooldown_seconds: 30,
};
let debug_str = format!("{:?}", settings);
assert!(
debug_str.contains("***REDACTED***"),
"urls must be redacted in Debug"
);
assert!(
!debug_str.contains("secret:password"),
"credentials must not leak to Debug"
);
assert!(debug_str.contains("true"), "enabled should be visible");
assert!(
debug_str.contains("RoundRobin"),
"strategy should be visible (non-sensitive)"
);
}
#[test]
fn test_proxy_settings_clone_preserves_fields() {
let settings = ProxySettings {
urls: vec!["http://clone-proxy:9090".to_string()],
strategy: ProxyStrategy::Sticky,
enabled: true,
sticky_ttl_seconds: 90,
cooldown_seconds: 20,
};
let cloned = settings.clone();
assert_eq!(cloned.urls(), &["http://clone-proxy:9090"]);
assert_eq!(cloned.strategy(), ProxyStrategy::Sticky);
assert!(cloned.enabled);
assert_eq!(cloned.sticky_ttl_seconds, 90);
assert_eq!(cloned.cooldown_seconds, 20);
}
#[test]
fn test_proxy_settings_serde_roundtrip() {
let settings = ProxySettings {
urls: vec!["http://serde-proxy:7070".to_string()],
strategy: ProxyStrategy::Sticky,
enabled: false,
sticky_ttl_seconds: 100,
cooldown_seconds: 50,
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: ProxySettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.urls(), &["http://serde-proxy:7070"]);
assert_eq!(back.strategy(), ProxyStrategy::Sticky);
assert!(!back.enabled);
assert_eq!(back.sticky_ttl_seconds, 100);
assert_eq!(back.cooldown_seconds, 50);
}
#[test]
fn test_proxy_settings_strategy_serde_snake_case() {
let json = r#"{"urls":[],"strategy":"round_robin","enabled":false}"#;
let s: ProxySettings = serde_json::from_str(json).expect("deserialize round_robin");
assert_eq!(s.strategy, ProxyStrategy::RoundRobin);
let back = serde_json::to_string(&s).expect("serialize");
assert!(
back.contains("\"round_robin\""),
"serialized should be snake_case: {back}"
);
let json = r#"{"urls":[],"strategy":"sticky","enabled":false}"#;
let s: ProxySettings = serde_json::from_str(json).expect("deserialize sticky");
assert_eq!(s.strategy, ProxyStrategy::Sticky);
}
#[test]
fn test_proxy_settings_strategy_missing_falls_back_to_default() {
let json = r#"{"urls":["http://x:8080"],"enabled":true}"#;
let s: ProxySettings = serde_json::from_str(json).expect("deserialize missing strategy");
assert_eq!(s.strategy, ProxyStrategy::RoundRobin);
assert!(s.enabled);
}
#[test]
fn test_worker_settings_default_uses_auto() {
let settings = WorkerSettings::default();
assert!(matches!(settings.count, WorkerCount::Auto(_)));
}
#[test]
fn test_worker_settings_construction_fixed() {
let settings = WorkerSettings {
count: WorkerCount::Fixed(16),
graceful_shutdown_seconds: 30,
dedup_warmup_include_results: false,
};
assert_eq!(settings.count.resolve(), 16);
}
#[test]
fn test_worker_settings_dedup_warmup_include_results_default_false() {
assert!(!WorkerSettings::default().dedup_warmup_include_results);
}
#[test]
fn test_worker_settings_dedup_warmup_include_results_toml() {
let old_toml = r#"
count = 5
graceful_shutdown_seconds = 30
"#;
let s: WorkerSettings = toml::from_str(old_toml).expect("deserialize old config");
assert!(!s.dedup_warmup_include_results, "缺键回退 false");
let enabled: WorkerSettings = toml::from_str(
r#"
count = 5
graceful_shutdown_seconds = 30
dedup_warmup_include_results = true
"#,
)
.expect("deserialize override");
assert!(enabled.dedup_warmup_include_results, "显式 true 生效");
}
#[test]
fn test_sitemap_settings_defaults_match_spec() {
let s = SitemapSettings::default();
assert!(!s.enabled, "流式请求源默认关");
assert_eq!(s.max_urls, 50_000, "单次 map 消费上限");
assert_eq!(s.max_concurrent_sub_sitemaps, 4, "子 sitemap 并发");
assert_eq!(s.channel_capacity, 4096, "mpsc 背压边界");
assert_eq!(s.fetch_timeout_secs, 30, "单文档拉取超时");
}
#[test]
fn test_sitemap_settings_toml_parses_all_keys() {
let toml = r#"
enabled = true
max_urls = 1000
max_concurrent_sub_sitemaps = 2
channel_capacity = 128
fetch_timeout_secs = 10
"#;
let s: SitemapSettings = toml::from_str(toml).expect("deserialize");
assert!(s.enabled);
assert_eq!(s.max_urls, 1000);
assert_eq!(s.max_concurrent_sub_sitemaps, 2);
assert_eq!(s.channel_capacity, 128);
assert_eq!(s.fetch_timeout_secs, 10);
}
#[test]
fn test_sitemap_settings_missing_section_falls_back_to_defaults() {
let s: SitemapSettings = toml::from_str("").expect("empty toml with serde defaults");
assert!(!s.enabled);
assert_eq!(s.max_urls, 50_000);
}
#[test]
fn test_crawl_settings_defaults_match_spec() {
let c = CrawlSettings::default();
assert!(c.soft404_enabled, "软 404 默认开");
assert_eq!(c.soft404_max_fingerprints, 10_000, "指纹库容量");
}
#[test]
fn test_crawl_settings_toml_parses_and_falls_back() {
let c: CrawlSettings = toml::from_str(
r#"
soft404_enabled = false
soft404_max_fingerprints = 100
"#,
)
.expect("deserialize");
assert!(!c.soft404_enabled);
assert_eq!(c.soft404_max_fingerprints, 100);
let fallback: CrawlSettings = toml::from_str("").expect("empty toml with serde defaults");
assert!(fallback.soft404_enabled);
assert_eq!(fallback.soft404_max_fingerprints, 10_000);
}
#[test]
fn test_timeout_settings_default() {
let settings = TimeoutSettings::default();
assert_eq!(settings.workers.webhook_interval_seconds, 5);
assert_eq!(settings.workers.backlog_interval_seconds, 30);
assert_eq!(settings.engines.default_timeout_seconds, 30);
assert_eq!(settings.engines.playwright_timeout_seconds, 30);
assert_eq!(settings.engines.flaresolverr_timeout_seconds, 30);
assert_eq!(settings.retry.initial_backoff_seconds, 1);
assert_eq!(settings.retry.max_backoff_seconds, 60);
assert_eq!(settings.cache.default_ttl_seconds, 600);
assert_eq!(settings.cache.memory_ttl_seconds, 600);
}
#[test]
fn test_playwright_engine_settings_default_capture_off() {
let settings = PlaywrightEngineSettings::default();
assert!(!settings.capture_xhr);
assert!(settings.capture_xhr_url_patterns.is_empty());
assert_eq!(settings.capture_xhr_max_bytes, 262_144);
assert_eq!(settings.capture_xhr_max_responses, 50);
}
#[test]
fn test_playwright_engine_settings_toml_deserialize() {
let toml = r#"
capture_xhr = true
capture_xhr_url_patterns = ["*/api/*"]
capture_xhr_max_bytes = 1024
capture_xhr_max_responses = 5
"#;
let settings: PlaywrightEngineSettings =
toml::from_str(toml).expect("valid playwright engine toml");
assert!(settings.capture_xhr);
assert_eq!(
settings.capture_xhr_url_patterns,
vec!["*/api/*".to_string()]
);
assert_eq!(settings.capture_xhr_max_bytes, 1024);
assert_eq!(settings.capture_xhr_max_responses, 5);
}
#[test]
fn test_worker_timeout_settings_construction() {
let settings = WorkerTimeoutSettings {
webhook_interval_seconds: 10,
backlog_interval_seconds: 60,
};
assert_eq!(settings.webhook_interval_seconds, 10);
assert_eq!(settings.backlog_interval_seconds, 60);
}
#[test]
fn test_engine_timeout_settings_construction() {
let settings = EngineTimeoutSettings {
default_timeout_seconds: 45,
playwright_timeout_seconds: 50,
flaresolverr_timeout_seconds: 55,
fetch_seconds: 5,
tls_seconds: 15,
cdp_seconds: 30,
};
assert_eq!(settings.default_timeout_seconds, 45);
assert_eq!(settings.playwright_timeout_seconds, 50);
assert_eq!(settings.flaresolverr_timeout_seconds, 55);
assert_eq!(settings.fetch_seconds, 5);
assert_eq!(settings.tls_seconds, 15);
assert_eq!(settings.cdp_seconds, 30);
}
#[test]
fn test_engine_timeout_settings_serde_default_for_mrt_fields() {
let old_toml = r#"
default_timeout_seconds = 45
playwright_timeout_seconds = 50
flaresolverr_timeout_seconds = 55
"#;
let settings: EngineTimeoutSettings = toml::from_str(old_toml).expect("deserialize");
assert_eq!(settings.default_timeout_seconds, 45);
assert_eq!(settings.playwright_timeout_seconds, 50);
assert_eq!(settings.flaresolverr_timeout_seconds, 55);
assert_eq!(settings.fetch_seconds, 5, "fetch_seconds default");
assert_eq!(settings.tls_seconds, 15, "tls_seconds default");
assert_eq!(settings.cdp_seconds, 30, "cdp_seconds default");
}
#[test]
fn test_engine_timeout_settings_mrt_fields_override() {
let toml = r#"
default_timeout_seconds = 60
playwright_timeout_seconds = 45
flaresolverr_timeout_seconds = 60
fetch_seconds = 8
tls_seconds = 20
cdp_seconds = 45
"#;
let settings: EngineTimeoutSettings = toml::from_str(toml).expect("deserialize");
assert_eq!(settings.fetch_seconds, 8);
assert_eq!(settings.tls_seconds, 20);
assert_eq!(settings.cdp_seconds, 45);
}
#[test]
fn test_engine_timeout_settings_validate_rejects_zero() {
let settings = EngineTimeoutSettings {
default_timeout_seconds: 0,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 5,
tls_seconds: 15,
cdp_seconds: 30,
};
let result = settings.validate();
assert!(
result.is_err(),
"default_timeout_seconds=0 should be rejected (DoS risk)"
);
let err = result.unwrap_err().to_string();
assert!(
err.contains("default_timeout_seconds"),
"error should mention field name: {}",
err
);
}
#[test]
fn test_engine_timeout_settings_validate_rejects_zero_mrt_fields() {
let settings = EngineTimeoutSettings {
default_timeout_seconds: 30,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 0,
tls_seconds: 15,
cdp_seconds: 30,
};
assert!(
settings.validate().is_err(),
"fetch_seconds=0 should be rejected"
);
let settings = EngineTimeoutSettings {
default_timeout_seconds: 30,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 5,
tls_seconds: 0,
cdp_seconds: 30,
};
assert!(
settings.validate().is_err(),
"tls_seconds=0 should be rejected"
);
let settings = EngineTimeoutSettings {
default_timeout_seconds: 30,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 5,
tls_seconds: 15,
cdp_seconds: 0,
};
assert!(
settings.validate().is_err(),
"cdp_seconds=0 should be rejected"
);
}
#[test]
fn test_engine_timeout_settings_validate_rejects_exceeding_max() {
let settings = EngineTimeoutSettings {
default_timeout_seconds: 30,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 601,
tls_seconds: 15,
cdp_seconds: 30,
};
assert!(
settings.validate().is_err(),
"fetch_seconds=601 should be rejected (exceeding max)"
);
}
#[test]
fn test_engine_timeout_settings_validate_accepts_boundary_values() {
let settings = EngineTimeoutSettings {
default_timeout_seconds: 1,
playwright_timeout_seconds: 600,
flaresolverr_timeout_seconds: 1,
fetch_seconds: 1,
tls_seconds: 600,
cdp_seconds: 1,
};
assert!(
settings.validate().is_ok(),
"boundary values (1 and 600) should pass validation"
);
}
#[test]
fn test_engine_timeout_settings_validate_accepts_defaults() {
let settings = EngineTimeoutSettings {
default_timeout_seconds: 30,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 5,
tls_seconds: 15,
cdp_seconds: 30,
};
assert!(
settings.validate().is_ok(),
"default values should pass validation"
);
}
#[test]
fn test_retry_timeout_settings_construction() {
let settings = RetryTimeoutSettings {
initial_backoff_seconds: 2,
max_backoff_seconds: 120,
};
assert_eq!(settings.initial_backoff_seconds, 2);
assert_eq!(settings.max_backoff_seconds, 120);
}
#[test]
fn test_cache_timeout_settings_construction() {
let settings = CacheTimeoutSettings {
default_ttl_seconds: 300,
memory_ttl_seconds: 200,
};
assert_eq!(settings.default_ttl_seconds, 300);
assert_eq!(settings.memory_ttl_seconds, 200);
}
#[test]
fn test_cache_settings_default() {
let settings = CacheSettings::default();
assert!(settings.enabled);
assert_eq!(settings.backend, "memory");
assert_eq!(settings.redis_url, "");
assert_eq!(settings.memory.capacity, 10000);
assert_eq!(settings.memory.ttl_seconds, 300);
}
#[test]
fn test_memory_cache_settings_construction() {
let settings = MemoryCacheSettings {
capacity: 5000,
ttl_seconds: 120,
};
assert_eq!(settings.capacity, 5000);
assert_eq!(settings.ttl_seconds, 120);
}
#[test]
fn test_cache_type_settings_construction() {
let settings = CacheTypeSettings {
ttl_seconds: 100,
max_size: 500,
};
assert_eq!(settings.ttl_seconds, 100);
assert_eq!(settings.max_size, 500);
}
#[test]
fn test_cache_settings_serde_roundtrip() {
let settings = CacheSettings {
enabled: false,
backend: "memory".to_string(),
redis_url: String::new(),
memory: MemoryCacheSettings {
capacity: 100,
ttl_seconds: 50,
},
types: CacheTypeSpecificSettings {
search: CacheTypeSettings {
ttl_seconds: 10,
max_size: 20,
},
dns: CacheTypeSettings {
ttl_seconds: 30,
max_size: 40,
},
regex: CacheTypeSettings {
ttl_seconds: 60,
max_size: 70,
},
},
};
let json = serde_json::to_string(&settings).expect("serialize");
let back: CacheSettings = serde_json::from_str(&json).expect("deserialize");
assert!(!back.enabled);
assert_eq!(back.backend, "memory");
assert_eq!(back.memory.capacity, 100);
assert_eq!(back.types.search.ttl_seconds, 10);
assert_eq!(back.types.dns.max_size, 40);
assert_eq!(back.types.regex.ttl_seconds, 60);
}
#[test]
fn test_cache_settings_missing_keys_fall_back_to_memory() {
let toml = r#"
enabled = true
[memory]
capacity = 100
ttl_seconds = 60
[types.search]
ttl_seconds = 60
max_size = 100
[types.dns]
ttl_seconds = 300
max_size = 100
[types.regex]
ttl_seconds = 600
max_size = 50
"#;
let s: CacheSettings = toml::from_str(toml).expect("deserialize legacy config");
assert_eq!(s.backend, "memory");
assert_eq!(s.redis_url, "");
assert!(s.validate_backend().is_ok());
}
#[test]
fn test_validate_backend_memory_and_unknown() {
let mut s = CacheSettings::default();
assert!(s.validate_backend().is_ok());
s.backend = "memcached".to_string();
let err = s.validate_backend().expect_err("unknown backend");
assert_eq!(err.code.to_string(), "cache_backend_unknown");
}
#[cfg(not(feature = "cache-redis"))]
#[test]
fn test_validate_backend_redis_rejected_without_feature() {
let mut s = CacheSettings {
backend: "redis".to_string(),
..CacheSettings::default()
};
let err = s.validate_backend().expect_err("feature not compiled");
assert_eq!(
err.code.to_string(),
"cache_backend_redis_feature_not_compiled"
);
s.redis_url = "rediss://127.0.0.1:6379".to_string();
let err = s.validate_backend().expect_err("url cannot bypass feature");
assert_eq!(
err.code.to_string(),
"cache_backend_redis_feature_not_compiled"
);
}
#[cfg(feature = "cache-redis")]
#[test]
fn test_validate_backend_redis_requires_url() {
let mut s = CacheSettings {
backend: "redis".to_string(),
..CacheSettings::default()
};
let err = s.validate_backend().expect_err("url missing");
assert_eq!(err.code.to_string(), "cache_backend_redis_url_missing");
s.redis_url = "rediss://127.0.0.1:6379".to_string();
assert!(s.validate_backend().is_ok());
}
}