use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__LLM__CHUNKING__")]
pub struct LlmChunkingSettings {
#[serde(default)]
#[config(default = false)]
pub enabled: bool,
#[serde(default = "default_chunking_model_context_tokens")]
#[config(default = 8192)]
pub model_context_tokens: u32,
#[serde(default = "default_chunking_max_output_tokens")]
#[config(default = 2048)]
pub max_output_tokens: u32,
#[serde(default = "default_chunking_prompt_overhead_tokens")]
#[config(default = 1024)]
pub prompt_overhead_tokens: u32,
#[serde(default = "default_chunking_chars_per_token")]
#[config(default = 4.0)]
pub chars_per_token: f64,
#[serde(default = "default_chunking_max_parallel")]
#[config(default = 4)]
pub max_parallel: usize,
#[serde(default = "default_chunking_merge_enabled")]
#[config(default = true)]
pub merge_enabled: bool,
#[serde(default = "default_chunking_term_precheck_enabled")]
#[config(default = true)]
pub term_precheck_enabled: bool,
}
fn default_chunking_model_context_tokens() -> u32 {
8192
}
fn default_chunking_max_output_tokens() -> u32 {
2048
}
fn default_chunking_prompt_overhead_tokens() -> u32 {
1024
}
fn default_chunking_chars_per_token() -> f64 {
4.0
}
fn default_chunking_max_parallel() -> usize {
4
}
fn default_chunking_merge_enabled() -> bool {
true
}
fn default_chunking_term_precheck_enabled() -> bool {
true
}
fn default_provenance_min_coverage() -> f64 {
0.75
}
fn default_request_timeout_secs() -> u64 {
120
}
#[derive(Debug, Clone, Deserialize, Serialize, confers::Config)]
#[config(env_prefix = "CRAWLRS__LLM__")]
pub struct LLMSettings {
#[config(default = Some("gpt-3.5-turbo".to_string()))]
pub model: Option<String>,
#[serde(default)]
pub chunking: LlmChunkingSettings,
#[serde(default = "default_request_timeout_secs")]
#[config(default = 120)]
pub request_timeout_secs: u64,
#[serde(default)]
#[config(default = false)]
pub provenance_enabled: bool,
#[serde(default = "default_provenance_min_coverage")]
#[config(default = 0.75)]
pub provenance_min_coverage: f64,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_llm_chunking_settings_defaults_match_spec() {
let s = LlmChunkingSettings::default();
assert!(!s.enabled, "默认关:长文单 prompt 直发(现状)");
assert_eq!(s.model_context_tokens, 8192, "模型窗口兜底值");
assert_eq!(s.max_output_tokens, 2048, "回答预留");
assert_eq!(s.prompt_overhead_tokens, 1024, "模板注入开销预留");
assert!((s.chars_per_token - 4.0).abs() < f64::EPSILON, "保守换算");
assert_eq!(s.max_parallel, 4, "JoinSet 并行上限");
assert!(s.merge_enabled, "单次 merge prompt 综合");
assert!(s.term_precheck_enabled, "零成本术语预检短路");
}
#[test]
fn test_llm_chunking_settings_toml_parses_all_keys() {
let toml = r#"
enabled = true
model_context_tokens = 32768
max_output_tokens = 4096
prompt_overhead_tokens = 512
chars_per_token = 1.5
max_parallel = 8
merge_enabled = false
term_precheck_enabled = false
"#;
let s: LlmChunkingSettings = toml::from_str(toml).expect("deserialize");
assert!(s.enabled);
assert_eq!(s.model_context_tokens, 32768);
assert_eq!(s.max_output_tokens, 4096);
assert_eq!(s.prompt_overhead_tokens, 512);
assert!((s.chars_per_token - 1.5).abs() < f64::EPSILON);
assert_eq!(s.max_parallel, 8);
assert!(!s.merge_enabled);
assert!(!s.term_precheck_enabled);
}
#[test]
fn test_llm_settings_without_chunking_section_defaults() {
let toml = r#"
model = "qwen3:1.7b"
"#;
let s: LLMSettings = toml::from_str(toml).expect("deserialize");
assert!(!s.chunking.enabled);
assert_eq!(s.chunking.model_context_tokens, 8192);
}
#[test]
fn test_llm_provenance_settings_defaults_match_spec() {
let s = LLMSettings::default();
assert!(!s.provenance_enabled, "默认关:结果不挂 _provenance");
assert!(
(s.provenance_min_coverage - 0.75).abs() < f64::EPSILON,
"langextract 同款覆盖率门 0.75"
);
}
#[test]
fn test_llm_provenance_settings_toml_parses() {
let toml = r#"
provenance_enabled = true
provenance_min_coverage = 0.8
"#;
let s: LLMSettings = toml::from_str(toml).expect("deserialize");
assert!(s.provenance_enabled);
assert!((s.provenance_min_coverage - 0.8).abs() < f64::EPSILON);
}
#[test]
fn test_llm_settings_without_provenance_keys_defaults() {
let toml = r#"
model = "qwen3:1.7b"
"#;
let s: LLMSettings = toml::from_str(toml).expect("deserialize");
assert!(!s.provenance_enabled);
assert!((s.provenance_min_coverage - 0.75).abs() < f64::EPSILON);
}
#[test]
fn test_llm_default_model() {
let settings = LLMSettings::default();
assert_eq!(
settings.model.as_deref(),
Some("gpt-3.5-turbo"),
"default model should be gpt-3.5-turbo"
);
}
#[test]
fn test_llm_retired_provider_keys_are_ignored() {
let toml = r#"
model = "host-carrier-model"
provider = "ollama"
api_key = "legacy-secret"
api_base_url = "http://localhost:11434/v1"
"#;
let settings: LLMSettings = toml::from_str(toml).expect("deserialize");
assert_eq!(settings.model.as_deref(), Some("host-carrier-model"));
assert!(!settings.provenance_enabled);
}
#[test]
fn test_llm_serde_roundtrip_default() {
let settings = LLMSettings::default();
let json = serde_json::to_string(&settings).expect("serialize");
let back: LLMSettings = serde_json::from_str(&json).expect("deserialize");
assert_eq!(back.model.as_deref(), settings.model.as_deref());
assert_eq!(
back.chunking.model_context_tokens,
settings.chunking.model_context_tokens
);
assert_eq!(back.provenance_enabled, settings.provenance_enabled);
}
#[test]
fn test_llm_clone_preserves_fields() {
let settings = LLMSettings {
model: Some("llama2".to_string()),
chunking: LlmChunkingSettings::default(),
provenance_enabled: true,
request_timeout_secs: 90,
provenance_min_coverage: 0.8,
};
let cloned = settings.clone();
assert_eq!(cloned.model, settings.model);
assert_eq!(cloned.provenance_enabled, settings.provenance_enabled);
assert_eq!(cloned.request_timeout_secs, 90);
}
#[test]
fn test_llm_debug_contains_model() {
let settings = LLMSettings {
model: Some("gpt-4".to_string()),
..LLMSettings::default()
};
let debug = format!("{:?}", settings);
assert!(debug.contains("LLMSettings"));
assert!(debug.contains("gpt-4"));
}
}