assay_core/
config.rs

1use crate::errors::ConfigError;
2use crate::model::EvalConfig;
3use std::path::Path;
4
5pub mod otel;
6pub mod path_resolver;
7pub mod resolve;
8
9pub const SUPPORTED_CONFIG_VERSION: u32 = 1;
10
11/// What a caller wants the loader to refuse, rather than merely parse.
12///
13/// Each field is a separate axis on purpose. `strict_unknown_fields` and
14/// `allow_ineffective_assertions` decide different things — a key the schema does not know versus
15/// an assertion that no trace could ever fail — and folding them into one flag would mean a caller
16/// who wanted one silently acquired the other.
17#[derive(Debug, Clone, Copy, Default)]
18pub struct LoadOptions {
19    /// Treat a v0 config as v0 rather than trusting its declared version.
20    pub legacy_mode: bool,
21    /// Refuse a config carrying keys this version does not understand.
22    pub strict_unknown_fields: bool,
23    /// Accept a config whose `assertions:` include one that cannot fail.
24    ///
25    /// **Refusing is the default, and this is the escape hatch.** The phased route in #1949 was
26    /// warning, then opt-in, then default at a major: `assay validate` has warned since #1983, the
27    /// opt-in landed as `--deny-ineffective-assertions`, and 5.0.0 is the major that carries the
28    /// flip (#1949).
29    ///
30    /// The polarity is inverted rather than the default being overridden, so that
31    /// `#[derive(Default)]` still produces the intended behaviour. A `deny_*` field defaulting to
32    /// `true` needs a hand-written `Default`, and every `..Default::default()` in the tree would
33    /// then depend on that impl being right. `false` meaning "do not allow" is the same fact with
34    /// nothing to keep in sync.
35    pub allow_ineffective_assertions: bool,
36}
37
38/// Convenience loader for the two older axes.
39///
40/// It no longer preserves the pre-5.0.0 behaviour and is not meant to: `..Default::default()` now
41/// carries the ineffective-assertion refusal, so a caller on this path gets it too. That is the
42/// point of the flip in #1949 rather than an oversight, and a caller who needs the old behaviour
43/// asks for it through [`load_config_with`] with `allow_ineffective_assertions: true`.
44pub fn load_config(
45    path: &Path,
46    legacy_mode: bool,
47    strict: bool,
48) -> Result<EvalConfig, ConfigError> {
49    load_config_with(
50        path,
51        LoadOptions {
52            legacy_mode,
53            strict_unknown_fields: strict,
54            ..Default::default()
55        },
56    )
57}
58
59pub fn load_config_with(path: &Path, opts: LoadOptions) -> Result<EvalConfig, ConfigError> {
60    let LoadOptions {
61        legacy_mode,
62        strict_unknown_fields: strict,
63        allow_ineffective_assertions,
64    } = opts;
65    let raw = std::fs::read_to_string(path)
66        .map_err(|e| ConfigError(format!("failed to read config {}: {}", path.display(), e)))?;
67
68    let mut ignored_keys = std::collections::HashSet::new();
69    let deserializer = serde_yaml::Deserializer::from_str(&raw);
70
71    // serde_ignored wrapper to capture unknown fields
72    let mut cfg: EvalConfig = serde_ignored::deserialize(deserializer, |path| {
73        ignored_keys.insert(path.to_string());
74    })
75    .map_err(|e| ConfigError(format!("failed to parse YAML: {}", e)))?;
76
77    // Check strictness / significant unknown fields
78    if strict && !ignored_keys.is_empty() {
79        // Whitelist common YAML anchor keys
80        let meaningful_unknowns: Vec<_> = ignored_keys
81            .iter()
82            .filter(|k| *k != "definitions" && !k.starts_with("_") && !k.starts_with("x-"))
83            .collect();
84
85        if meaningful_unknowns.is_empty() {
86            // All unknowns are whitelisted (e.g. anchors). PASS.
87        } else {
88            // Special helpful error for v0 'policies'
89            if ignored_keys.contains("policies") {
90                return Err(ConfigError(format!(
91                    "Top-level 'policies' is not valid in configVersion: {}. Did you mean to run assay migrate on a v0 config, or remove legacy keys? (file: {})",
92                    cfg.version,
93                    path.display()
94                )));
95            }
96
97            // Generic strict error
98            return Err(ConfigError(format!(
99                "Unknown fields detected in strict mode: {:?} (file: {})",
100                meaningful_unknowns,
101                path.display()
102            )));
103        }
104    } else if !ignored_keys.is_empty() {
105        // In non-strict mode, we ideally WARN, but standard logging might not be initialized here.
106        // For now, we proceed as 'careful ignore' but validated at least.
107        // The user specifically asked for migrate FAIL (strict=true) and run WARN.
108        eprintln!("WARN: Ignored unknown config fields: {:?}", ignored_keys);
109    }
110
111    // Legacy override
112    if legacy_mode {
113        cfg.version = 0;
114    }
115
116    // Allow 0 or 1
117    if cfg.version != 0 && cfg.version != SUPPORTED_CONFIG_VERSION {
118        return Err(ConfigError(format!(
119            "unsupported config version {} (supported: 0, {})",
120            cfg.version, SUPPORTED_CONFIG_VERSION
121        )));
122    }
123
124    if cfg.tests.is_empty() {
125        return Err(ConfigError("config has no tests".into()));
126    }
127
128    // Fail closed before execution rather than after. An assertion that cannot fail reports a pass
129    // carrying no information, and a run that reaches it has already spent the time and money to
130    // produce that non-answer. The decision itself is `validate::ineffective_assertions`, which is
131    // the same code `assay validate` sweeps with, so this cannot drift away from what the warning
132    // says. Diagnostics stay value-free: they name the test, the index, the variant and the
133    // responsible field, never the configured value.
134    if !allow_ineffective_assertions {
135        let ineffective = crate::validate::ineffective_assertions(&cfg);
136        if !ineffective.is_empty() {
137            let detail = ineffective
138                .iter()
139                .map(|d| {
140                    let test = d
141                        .context
142                        .get("test_id")
143                        .and_then(|v| v.as_str())
144                        .unwrap_or("?");
145                    let index = d
146                        .context
147                        .get("assertion_index")
148                        .and_then(|v| v.as_u64())
149                        .map(|i| i.to_string())
150                        .unwrap_or_else(|| "?".into());
151                    format!("test '{}' assertion {}: {}", test, index, d.message)
152                })
153                .collect::<Vec<_>>()
154                .join("; ");
155            return Err(ConfigError(format!(
156                "{} assertion(s) cannot fail and were refused ({}): {} \
157                 An assertion that cannot fail reports a pass carrying no information. \
158                 Fix the assertion, or pass --allow-ineffective-assertions to run anyway.",
159                ineffective.len(),
160                path.display(),
161                detail
162            )));
163        }
164    }
165
166    normalize_paths(&mut cfg, path)
167        .map_err(|e| ConfigError(format!("failed to normalize config paths: {}", e)))?;
168
169    Ok(cfg)
170}
171
172fn normalize_paths(cfg: &mut EvalConfig, config_path: &Path) -> anyhow::Result<()> {
173    let r = path_resolver::PathResolver::new(config_path);
174
175    for tc in &mut cfg.tests {
176        if let crate::model::Expected::JsonSchema { schema_file, .. } = &mut tc.expected {
177            if let Some(orig) = schema_file.clone() {
178                let before = orig.clone();
179                r.resolve_opt_str(schema_file);
180
181                if let Some(resolved) = schema_file.as_ref() {
182                    if *resolved != before {
183                        let meta = tc.metadata.get_or_insert_with(|| serde_json::json!({}));
184                        if !meta.get("assay").is_some_and(|v| v.is_object()) {
185                            meta["assay"] = serde_json::json!({});
186                        }
187
188                        meta["assay"]["schema_file_original"] = serde_json::json!(before);
189                        meta["assay"]["schema_file_resolved"] = serde_json::json!(resolved);
190                        meta["assay"]["config_dir"] = serde_json::json!(config_path
191                            .parent()
192                            .unwrap_or(Path::new("."))
193                            .to_string_lossy());
194                    }
195                }
196            }
197        }
198    }
199    Ok(())
200}
201
202pub fn write_sample_config(path: &Path) -> Result<(), ConfigError> {
203    std::fs::write(
204        path,
205        r#"version: 1
206suite: demo
207model: dummy
208settings:
209  parallel: 4
210  timeout_seconds: 30
211  cache: true
212tests:
213  - id: t1_must_contain
214    tags: ["smoke"]
215    input:
216      prompt: "Say hello and mention Amsterdam."
217    expected:
218      type: must_contain
219      must_contain: ["hello", "Amsterdam"]
220  - id: t2_must_not_contain
221    tags: ["smoke"]
222    input:
223      prompt: "Write a sentence without the word banana."
224    expected:
225      type: must_not_contain
226      must_not_contain: ["banana"]
227"#,
228    )
229    .map_err(|e| ConfigError(format!("failed to write sample config: {}", e)))?;
230    Ok(())
231}
assay_core/config.rs

assay_core/
config.rs
