assay_core/
config.rs

1use crate::errors::ConfigError;
2use crate::model::EvalConfig;
3use std::path::Path;
4
5pub mod path_resolver;
6pub mod resolve;
7
8pub const SUPPORTED_CONFIG_VERSION: u32 = 1;
9
10pub fn load_config(
11    path: &Path,
12    legacy_mode: bool,
13    strict: bool,
14) -> Result<EvalConfig, ConfigError> {
15    let raw = std::fs::read_to_string(path)
16        .map_err(|e| ConfigError(format!("failed to read config {}: {}", path.display(), e)))?;
17
18    let mut ignored_keys = std::collections::HashSet::new();
19    let deserializer = serde_yaml::Deserializer::from_str(&raw);
20
21    // serde_ignored wrapper to capture unknown fields
22    let mut cfg: EvalConfig = serde_ignored::deserialize(deserializer, |path| {
23        ignored_keys.insert(path.to_string());
24    })
25    .map_err(|e| ConfigError(format!("failed to parse YAML: {}", e)))?;
26
27    // Check strictness / significant unknown fields
28    if strict && !ignored_keys.is_empty() {
29        // Whitelist common YAML anchor keys
30        let meaningful_unknowns: Vec<_> = ignored_keys
31            .iter()
32            .filter(|k| *k != "definitions" && !k.starts_with("_") && !k.starts_with("x-"))
33            .collect();
34
35        if meaningful_unknowns.is_empty() {
36            // All unknowns are whitelisted (e.g. anchors). PASS.
37        } else {
38            // Special helpful error for v0 'policies'
39            if ignored_keys.contains("policies") {
40                return Err(ConfigError(format!(
41                    "Top-level 'policies' is not valid in configVersion: {}. Did you mean to run assay migrate on a v0 config, or remove legacy keys? (file: {})",
42                    cfg.version,
43                    path.display()
44                )));
45            }
46
47            // Generic strict error
48            return Err(ConfigError(format!(
49                "Unknown fields detected in strict mode: {:?} (file: {})",
50                meaningful_unknowns,
51                path.display()
52            )));
53        }
54    } else if !ignored_keys.is_empty() {
55        // In non-strict mode, we ideally WARN, but standard logging might not be initialized here.
56        // For now, we proceed as 'careful ignore' but validated at least.
57        // The user specifically asked for migrate FAIL (strict=true) and run WARN.
58        eprintln!("WARN: Ignored unknown config fields: {:?}", ignored_keys);
59    }
60
61    // Legacy override
62    if legacy_mode {
63        cfg.version = 0;
64    }
65
66    // Allow 0 or 1
67    if cfg.version != 0 && cfg.version != SUPPORTED_CONFIG_VERSION {
68        return Err(ConfigError(format!(
69            "unsupported config version {} (supported: 0, {})",
70            cfg.version, SUPPORTED_CONFIG_VERSION
71        )));
72    }
73
74    if cfg.tests.is_empty() {
75        return Err(ConfigError("config has no tests".into()));
76    }
77
78    normalize_paths(&mut cfg, path)
79        .map_err(|e| ConfigError(format!("failed to normalize config paths: {}", e)))?;
80
81    Ok(cfg)
82}
83
84fn normalize_paths(cfg: &mut EvalConfig, config_path: &Path) -> anyhow::Result<()> {
85    let r = path_resolver::PathResolver::new(config_path);
86
87    for tc in &mut cfg.tests {
88        if let crate::model::Expected::JsonSchema { schema_file, .. } = &mut tc.expected {
89            if let Some(orig) = schema_file.clone() {
90                let before = orig.clone();
91                r.resolve_opt_str(schema_file);
92
93                if let Some(resolved) = schema_file.as_ref() {
94                    if *resolved != before {
95                        let meta = tc.metadata.get_or_insert_with(|| serde_json::json!({}));
96                        if !meta.get("assay").is_some_and(|v| v.is_object()) {
97                            meta["assay"] = serde_json::json!({});
98                        }
99
100                        meta["assay"]["schema_file_original"] = serde_json::json!(before);
101                        meta["assay"]["schema_file_resolved"] = serde_json::json!(resolved);
102                        meta["assay"]["config_dir"] = serde_json::json!(config_path
103                            .parent()
104                            .unwrap_or(Path::new("."))
105                            .to_string_lossy());
106                    }
107                }
108            }
109        }
110    }
111    Ok(())
112}
113
114pub fn write_sample_config(path: &Path) -> Result<(), ConfigError> {
115    std::fs::write(
116        path,
117        r#"version: 1
118suite: demo
119model: dummy
120settings:
121  parallel: 4
122  timeout_seconds: 30
123  cache: true
124tests:
125  - id: t1_must_contain
126    tags: ["smoke"]
127    input:
128      prompt: "Say hello and mention Amsterdam."
129    expected:
130      type: must_contain
131      must_contain: ["hello", "Amsterdam"]
132  - id: t2_must_not_contain
133    tags: ["smoke"]
134    input:
135      prompt: "Write a sentence without the word banana."
136    expected:
137      type: must_not_contain
138      must_not_contain: ["banana"]
139"#,
140    )
141    .map_err(|e| ConfigError(format!("failed to write sample config: {}", e)))?;
142    Ok(())
143}
assay_core/config.rs

assay_core/
config.rs
