1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
use crate::errors::ConfigError;
use crate::model::EvalConfig;
use std::path::Path;
pub mod otel;
pub mod path_resolver;
pub mod resolve;
pub const SUPPORTED_CONFIG_VERSION: u32 = 1;
/// What a caller wants the loader to refuse, rather than merely parse.
///
/// Each field is a separate axis on purpose. `strict_unknown_fields` and
/// `allow_ineffective_assertions` decide different things — a key the schema does not know versus
/// an assertion that no trace could ever fail — and folding them into one flag would mean a caller
/// who wanted one silently acquired the other.
#[derive(Debug, Clone, Copy, Default)]
pub struct LoadOptions {
/// Treat a v0 config as v0 rather than trusting its declared version.
pub legacy_mode: bool,
/// Refuse a config carrying keys this version does not understand.
pub strict_unknown_fields: bool,
/// Accept a config whose `assertions:` include one that cannot fail.
///
/// **Refusing is the default, and this is the escape hatch.** The phased route in #1949 was
/// warning, then opt-in, then default at a major: `assay validate` has warned since #1983, the
/// opt-in landed as `--deny-ineffective-assertions`, and 5.0.0 is the major that carries the
/// flip (#1949).
///
/// The polarity is inverted rather than the default being overridden, so that
/// `#[derive(Default)]` still produces the intended behaviour. A `deny_*` field defaulting to
/// `true` needs a hand-written `Default`, and every `..Default::default()` in the tree would
/// then depend on that impl being right. `false` meaning "do not allow" is the same fact with
/// nothing to keep in sync.
pub allow_ineffective_assertions: bool,
}
/// Convenience loader for the two older axes.
///
/// It no longer preserves the pre-5.0.0 behaviour and is not meant to: `..Default::default()` now
/// carries the ineffective-assertion refusal, so a caller on this path gets it too. That is the
/// point of the flip in #1949 rather than an oversight, and a caller who needs the old behaviour
/// asks for it through [`load_config_with`] with `allow_ineffective_assertions: true`.
pub fn load_config(
path: &Path,
legacy_mode: bool,
strict: bool,
) -> Result<EvalConfig, ConfigError> {
load_config_with(
path,
LoadOptions {
legacy_mode,
strict_unknown_fields: strict,
..Default::default()
},
)
}
pub fn load_config_with(path: &Path, opts: LoadOptions) -> Result<EvalConfig, ConfigError> {
let LoadOptions {
legacy_mode,
strict_unknown_fields: strict,
allow_ineffective_assertions,
} = opts;
let raw = std::fs::read_to_string(path)
.map_err(|e| ConfigError(format!("failed to read config {}: {}", path.display(), e)))?;
let mut ignored_keys = std::collections::HashSet::new();
let deserializer = serde_yaml::Deserializer::from_str(&raw);
// serde_ignored wrapper to capture unknown fields
let mut cfg: EvalConfig = serde_ignored::deserialize(deserializer, |path| {
ignored_keys.insert(path.to_string());
})
.map_err(|e| ConfigError(format!("failed to parse YAML: {}", e)))?;
// Check strictness / significant unknown fields
if strict && !ignored_keys.is_empty() {
// Whitelist common YAML anchor keys
let meaningful_unknowns: Vec<_> = ignored_keys
.iter()
.filter(|k| *k != "definitions" && !k.starts_with("_") && !k.starts_with("x-"))
.collect();
if meaningful_unknowns.is_empty() {
// All unknowns are whitelisted (e.g. anchors). PASS.
} else {
// Special helpful error for v0 'policies'
if ignored_keys.contains("policies") {
return Err(ConfigError(format!(
"Top-level 'policies' is not valid in configVersion: {}. Did you mean to run assay migrate on a v0 config, or remove legacy keys? (file: {})",
cfg.version,
path.display()
)));
}
// Generic strict error
return Err(ConfigError(format!(
"Unknown fields detected in strict mode: {:?} (file: {})",
meaningful_unknowns,
path.display()
)));
}
} else if !ignored_keys.is_empty() {
// In non-strict mode, we ideally WARN, but standard logging might not be initialized here.
// For now, we proceed as 'careful ignore' but validated at least.
// The user specifically asked for migrate FAIL (strict=true) and run WARN.
eprintln!("WARN: Ignored unknown config fields: {:?}", ignored_keys);
}
// Legacy override
if legacy_mode {
cfg.version = 0;
}
// Allow 0 or 1
if cfg.version != 0 && cfg.version != SUPPORTED_CONFIG_VERSION {
return Err(ConfigError(format!(
"unsupported config version {} (supported: 0, {})",
cfg.version, SUPPORTED_CONFIG_VERSION
)));
}
if cfg.tests.is_empty() {
return Err(ConfigError("config has no tests".into()));
}
// Fail closed before execution rather than after. An assertion that cannot fail reports a pass
// carrying no information, and a run that reaches it has already spent the time and money to
// produce that non-answer. The decision itself is `validate::ineffective_assertions`, which is
// the same code `assay validate` sweeps with, so this cannot drift away from what the warning
// says. Diagnostics stay value-free: they name the test, the index, the variant and the
// responsible field, never the configured value.
if !allow_ineffective_assertions {
let ineffective = crate::validate::ineffective_assertions(&cfg);
if !ineffective.is_empty() {
let detail = ineffective
.iter()
.map(|d| {
let test = d
.context
.get("test_id")
.and_then(|v| v.as_str())
.unwrap_or("?");
let index = d
.context
.get("assertion_index")
.and_then(|v| v.as_u64())
.map(|i| i.to_string())
.unwrap_or_else(|| "?".into());
format!("test '{}' assertion {}: {}", test, index, d.message)
})
.collect::<Vec<_>>()
.join("; ");
return Err(ConfigError(format!(
"{} assertion(s) cannot fail and were refused ({}): {} \
An assertion that cannot fail reports a pass carrying no information. \
Fix the assertion, or pass --allow-ineffective-assertions to run anyway.",
ineffective.len(),
path.display(),
detail
)));
}
}
normalize_paths(&mut cfg, path)
.map_err(|e| ConfigError(format!("failed to normalize config paths: {}", e)))?;
Ok(cfg)
}
fn normalize_paths(cfg: &mut EvalConfig, config_path: &Path) -> anyhow::Result<()> {
let r = path_resolver::PathResolver::new(config_path);
for tc in &mut cfg.tests {
if let crate::model::Expected::JsonSchema { schema_file, .. } = &mut tc.expected {
if let Some(orig) = schema_file.clone() {
let before = orig.clone();
r.resolve_opt_str(schema_file);
if let Some(resolved) = schema_file.as_ref() {
if *resolved != before {
let meta = tc.metadata.get_or_insert_with(|| serde_json::json!({}));
if !meta.get("assay").is_some_and(|v| v.is_object()) {
meta["assay"] = serde_json::json!({});
}
meta["assay"]["schema_file_original"] = serde_json::json!(before);
meta["assay"]["schema_file_resolved"] = serde_json::json!(resolved);
meta["assay"]["config_dir"] = serde_json::json!(config_path
.parent()
.unwrap_or(Path::new("."))
.to_string_lossy());
}
}
}
}
}
Ok(())
}
pub fn write_sample_config(path: &Path) -> Result<(), ConfigError> {
std::fs::write(
path,
r#"version: 1
suite: demo
model: dummy
settings:
parallel: 4
timeout_seconds: 30
cache: true
tests:
- id: t1_must_contain
tags: ["smoke"]
input:
prompt: "Say hello and mention Amsterdam."
expected:
type: must_contain
must_contain: ["hello", "Amsterdam"]
- id: t2_must_not_contain
tags: ["smoke"]
input:
prompt: "Write a sentence without the word banana."
expected:
type: must_not_contain
must_not_contain: ["banana"]
"#,
)
.map_err(|e| ConfigError(format!("failed to write sample config: {}", e)))?;
Ok(())
}