assay_core/
config.rs

1use crate::errors::ConfigError;
2use crate::model::EvalConfig;
3use std::path::Path;
4
5pub mod otel;
6pub mod path_resolver;
7pub mod resolve;
8
9pub const SUPPORTED_CONFIG_VERSION: u32 = 1;
10
11/// What a caller wants the loader to refuse, rather than merely parse.
12///
13/// Each field is a separate axis on purpose. `strict_unknown_fields` and
14/// `deny_ineffective_assertions` refuse different things — a key the schema does not know versus an
15/// assertion that no trace could ever fail — and folding them into one flag would mean a caller who
16/// wanted one silently acquired the other.
17#[derive(Debug, Clone, Copy, Default)]
18pub struct LoadOptions {
19    /// Treat a v0 config as v0 rather than trusting its declared version.
20    pub legacy_mode: bool,
21    /// Refuse a config carrying keys this version does not understand.
22    pub strict_unknown_fields: bool,
23    /// Refuse a config whose `assertions:` include one that cannot fail.
24    ///
25    /// Opt-in, and deliberately not on by default: `assay validate` has reported these as a
26    /// warning since #1983, and turning that into a load-time error for every caller at once would
27    /// break suites that are running today. The phased route in #1949 is warning, then opt-in here,
28    /// then default at a major after an announced window.
29    pub deny_ineffective_assertions: bool,
30}
31
32/// Backwards-compatible loader. Every existing caller keeps its behaviour, and the new refusal is
33/// reachable only through [`load_config_with`], which is what makes it opt-in.
34pub fn load_config(
35    path: &Path,
36    legacy_mode: bool,
37    strict: bool,
38) -> Result<EvalConfig, ConfigError> {
39    load_config_with(
40        path,
41        LoadOptions {
42            legacy_mode,
43            strict_unknown_fields: strict,
44            ..Default::default()
45        },
46    )
47}
48
49pub fn load_config_with(path: &Path, opts: LoadOptions) -> Result<EvalConfig, ConfigError> {
50    let LoadOptions {
51        legacy_mode,
52        strict_unknown_fields: strict,
53        deny_ineffective_assertions,
54    } = opts;
55    let raw = std::fs::read_to_string(path)
56        .map_err(|e| ConfigError(format!("failed to read config {}: {}", path.display(), e)))?;
57
58    let mut ignored_keys = std::collections::HashSet::new();
59    let deserializer = serde_yaml::Deserializer::from_str(&raw);
60
61    // serde_ignored wrapper to capture unknown fields
62    let mut cfg: EvalConfig = serde_ignored::deserialize(deserializer, |path| {
63        ignored_keys.insert(path.to_string());
64    })
65    .map_err(|e| ConfigError(format!("failed to parse YAML: {}", e)))?;
66
67    // Check strictness / significant unknown fields
68    if strict && !ignored_keys.is_empty() {
69        // Whitelist common YAML anchor keys
70        let meaningful_unknowns: Vec<_> = ignored_keys
71            .iter()
72            .filter(|k| *k != "definitions" && !k.starts_with("_") && !k.starts_with("x-"))
73            .collect();
74
75        if meaningful_unknowns.is_empty() {
76            // All unknowns are whitelisted (e.g. anchors). PASS.
77        } else {
78            // Special helpful error for v0 'policies'
79            if ignored_keys.contains("policies") {
80                return Err(ConfigError(format!(
81                    "Top-level 'policies' is not valid in configVersion: {}. Did you mean to run assay migrate on a v0 config, or remove legacy keys? (file: {})",
82                    cfg.version,
83                    path.display()
84                )));
85            }
86
87            // Generic strict error
88            return Err(ConfigError(format!(
89                "Unknown fields detected in strict mode: {:?} (file: {})",
90                meaningful_unknowns,
91                path.display()
92            )));
93        }
94    } else if !ignored_keys.is_empty() {
95        // In non-strict mode, we ideally WARN, but standard logging might not be initialized here.
96        // For now, we proceed as 'careful ignore' but validated at least.
97        // The user specifically asked for migrate FAIL (strict=true) and run WARN.
98        eprintln!("WARN: Ignored unknown config fields: {:?}", ignored_keys);
99    }
100
101    // Legacy override
102    if legacy_mode {
103        cfg.version = 0;
104    }
105
106    // Allow 0 or 1
107    if cfg.version != 0 && cfg.version != SUPPORTED_CONFIG_VERSION {
108        return Err(ConfigError(format!(
109            "unsupported config version {} (supported: 0, {})",
110            cfg.version, SUPPORTED_CONFIG_VERSION
111        )));
112    }
113
114    if cfg.tests.is_empty() {
115        return Err(ConfigError("config has no tests".into()));
116    }
117
118    // Fail closed before execution rather than after. An assertion that cannot fail reports a pass
119    // carrying no information, and a run that reaches it has already spent the time and money to
120    // produce that non-answer. The decision itself is `validate::ineffective_assertions`, which is
121    // the same code `assay validate` sweeps with, so this cannot drift away from what the warning
122    // says. Diagnostics stay value-free: they name the test, the index, the variant and the
123    // responsible field, never the configured value.
124    if deny_ineffective_assertions {
125        let ineffective = crate::validate::ineffective_assertions(&cfg);
126        if !ineffective.is_empty() {
127            let detail = ineffective
128                .iter()
129                .map(|d| {
130                    let test = d
131                        .context
132                        .get("test_id")
133                        .and_then(|v| v.as_str())
134                        .unwrap_or("?");
135                    let index = d
136                        .context
137                        .get("assertion_index")
138                        .and_then(|v| v.as_u64())
139                        .map(|i| i.to_string())
140                        .unwrap_or_else(|| "?".into());
141                    format!("test '{}' assertion {}: {}", test, index, d.message)
142                })
143                .collect::<Vec<_>>()
144                .join("; ");
145            return Err(ConfigError(format!(
146                "{} assertion(s) cannot fail and were refused because --deny-ineffective-assertions is set ({}): {}",
147                ineffective.len(),
148                path.display(),
149                detail
150            )));
151        }
152    }
153
154    normalize_paths(&mut cfg, path)
155        .map_err(|e| ConfigError(format!("failed to normalize config paths: {}", e)))?;
156
157    Ok(cfg)
158}
159
160fn normalize_paths(cfg: &mut EvalConfig, config_path: &Path) -> anyhow::Result<()> {
161    let r = path_resolver::PathResolver::new(config_path);
162
163    for tc in &mut cfg.tests {
164        if let crate::model::Expected::JsonSchema { schema_file, .. } = &mut tc.expected {
165            if let Some(orig) = schema_file.clone() {
166                let before = orig.clone();
167                r.resolve_opt_str(schema_file);
168
169                if let Some(resolved) = schema_file.as_ref() {
170                    if *resolved != before {
171                        let meta = tc.metadata.get_or_insert_with(|| serde_json::json!({}));
172                        if !meta.get("assay").is_some_and(|v| v.is_object()) {
173                            meta["assay"] = serde_json::json!({});
174                        }
175
176                        meta["assay"]["schema_file_original"] = serde_json::json!(before);
177                        meta["assay"]["schema_file_resolved"] = serde_json::json!(resolved);
178                        meta["assay"]["config_dir"] = serde_json::json!(config_path
179                            .parent()
180                            .unwrap_or(Path::new("."))
181                            .to_string_lossy());
182                    }
183                }
184            }
185        }
186    }
187    Ok(())
188}
189
190pub fn write_sample_config(path: &Path) -> Result<(), ConfigError> {
191    std::fs::write(
192        path,
193        r#"version: 1
194suite: demo
195model: dummy
196settings:
197  parallel: 4
198  timeout_seconds: 30
199  cache: true
200tests:
201  - id: t1_must_contain
202    tags: ["smoke"]
203    input:
204      prompt: "Say hello and mention Amsterdam."
205    expected:
206      type: must_contain
207      must_contain: ["hello", "Amsterdam"]
208  - id: t2_must_not_contain
209    tags: ["smoke"]
210    input:
211      prompt: "Write a sentence without the word banana."
212    expected:
213      type: must_not_contain
214      must_not_contain: ["banana"]
215"#,
216    )
217    .map_err(|e| ConfigError(format!("failed to write sample config: {}", e)))?;
218    Ok(())
219}
assay_core/config.rs

assay_core/
config.rs
