1use crate::cache::vcr::VcrCache;
2use crate::metrics_api::Metric;
3use crate::model::{EvalConfig, LlmResponse, TestCase, TestResultRow, TestStatus};
4use crate::providers::llm::LlmClient;
5use crate::quarantine::QuarantineMode;
6use crate::report::progress::ProgressSink;
7use crate::report::RunArtifacts;
8use crate::storage::store::Store;
9use std::sync::Arc;
10
11#[path = "runner_next/mod.rs"]
12mod runner_next;
13
14#[derive(Debug, Clone)]
15pub struct RunPolicy {
16 pub rerun_failures: u32,
17 pub quarantine_mode: QuarantineMode,
18 pub replay_strict: bool,
19}
20
21impl Default for RunPolicy {
22 fn default() -> Self {
23 Self {
24 rerun_failures: 1,
25 quarantine_mode: QuarantineMode::Warn,
26 replay_strict: false,
27 }
28 }
29}
30
31pub struct Runner {
32 pub store: Store,
33 pub cache: VcrCache,
34 pub client: Arc<dyn LlmClient>,
35 pub metrics: Vec<Arc<dyn Metric>>,
36 pub policy: RunPolicy,
37 pub _network_guard: Option<crate::providers::network::NetworkPolicyGuard>,
38 pub embedder: Option<Arc<dyn crate::providers::embedder::Embedder>>,
39 pub refresh_embeddings: bool,
40 pub incremental: bool,
41 pub refresh_cache: bool,
42 pub judge: Option<crate::judge::JudgeService>,
43 pub baseline: Option<crate::baseline::Baseline>,
44}
45
46impl Runner {
47 pub async fn run_suite(
51 &self,
52 cfg: &EvalConfig,
53 progress: Option<ProgressSink>,
54 ) -> anyhow::Result<RunArtifacts> {
55 runner_next::execute::run_suite_impl(self, cfg, progress).await
56 }
57
58 fn apply_agent_assertions(
59 &self,
60 run_id: i64,
61 tc: &TestCase,
62 resp: &crate::model::LlmResponse,
63 final_row: &mut TestResultRow,
64 ) -> anyhow::Result<()> {
65 runner_next::assertions::apply_agent_assertions_impl(self, run_id, tc, resp, final_row)
66 }
67
68 async fn run_test_once(
69 &self,
70 cfg: &EvalConfig,
71 tc: &TestCase,
72 ) -> anyhow::Result<(TestResultRow, LlmResponse)> {
73 runner_next::single::run_test_once_impl(self, cfg, tc).await
74 }
75
76 async fn call_llm(&self, cfg: &EvalConfig, tc: &TestCase) -> anyhow::Result<LlmResponse> {
77 runner_next::execute::call_llm_impl(self, cfg, tc).await
78 }
79
80 fn check_baseline_regressions(
81 &self,
82 tc: &TestCase,
83 cfg: &EvalConfig,
84 details: &serde_json::Value,
85 metrics: &[Arc<dyn Metric>],
86 baseline: &crate::baseline::Baseline,
87 ) -> Option<(TestStatus, String)> {
88 runner_next::baseline::check_baseline_regressions_impl(
89 self, tc, cfg, details, metrics, baseline,
90 )
91 }
92
93 async fn enrich_semantic(
95 &self,
96 _cfg: &EvalConfig,
97 tc: &TestCase,
98 resp: &mut LlmResponse,
99 ) -> anyhow::Result<()> {
100 runner_next::scoring::enrich_semantic_impl(self, _cfg, tc, resp).await
101 }
102
103 pub async fn embed_text(
104 &self,
105 model_id: &str,
106 embedder: &dyn crate::providers::embedder::Embedder,
107 text: &str,
108 ) -> anyhow::Result<(Vec<f32>, &'static str)> {
109 runner_next::cache::embed_text_impl(self, model_id, embedder, text).await
110 }
111
112 async fn enrich_judge(
113 &self,
114 cfg: &EvalConfig,
115 tc: &TestCase,
116 resp: &mut LlmResponse,
117 ) -> anyhow::Result<()> {
118 runner_next::scoring::enrich_judge_impl(self, cfg, tc, resp).await
119 }
120}
121
122#[cfg(test)]
123mod tests {
124 use super::*;
125 use crate::metrics_api::{Metric, MetricResult};
126 use crate::model::{Expected, Settings, TestInput};
127 use crate::on_error::ErrorPolicy;
128 use crate::providers::llm::fake::FakeClient;
129 use crate::providers::llm::LlmClient;
130 use async_trait::async_trait;
131 use std::sync::atomic::{AtomicUsize, Ordering};
132
133 #[derive(Clone, Copy)]
134 enum MetricMode {
135 FailThenPass,
136 AlwaysFail,
137 AlwaysPass,
138 }
139
140 struct ScriptedMetric {
141 mode: MetricMode,
142 calls: AtomicUsize,
143 }
144
145 impl ScriptedMetric {
146 fn fail_then_pass() -> Self {
147 Self {
148 mode: MetricMode::FailThenPass,
149 calls: AtomicUsize::new(0),
150 }
151 }
152
153 fn always_fail() -> Self {
154 Self {
155 mode: MetricMode::AlwaysFail,
156 calls: AtomicUsize::new(0),
157 }
158 }
159
160 fn always_pass() -> Self {
161 Self {
162 mode: MetricMode::AlwaysPass,
163 calls: AtomicUsize::new(0),
164 }
165 }
166 }
167
168 #[async_trait]
169 impl Metric for ScriptedMetric {
170 fn name(&self) -> &'static str {
171 "scripted"
172 }
173
174 async fn evaluate(
175 &self,
176 _tc: &TestCase,
177 _expected: &Expected,
178 _resp: &LlmResponse,
179 ) -> anyhow::Result<MetricResult> {
180 let n = self.calls.fetch_add(1, Ordering::SeqCst);
181 match self.mode {
182 MetricMode::FailThenPass => {
183 if n == 0 {
184 Ok(MetricResult::fail(0.0, "scripted_fail_once"))
185 } else {
186 Ok(MetricResult::pass(1.0))
187 }
188 }
189 MetricMode::AlwaysFail => Ok(MetricResult::fail(0.0, "scripted_fail")),
190 MetricMode::AlwaysPass => Ok(MetricResult::pass(1.0)),
191 }
192 }
193 }
194
195 struct ErrorClient;
196
197 #[async_trait]
198 impl LlmClient for ErrorClient {
199 async fn complete(
200 &self,
201 _prompt: &str,
202 _context: Option<&[String]>,
203 ) -> anyhow::Result<LlmResponse> {
204 Err(anyhow::anyhow!("scripted provider error"))
205 }
206
207 fn provider_name(&self) -> &'static str {
208 "error_client"
209 }
210 }
211
212 struct CountingClient {
213 calls: AtomicUsize,
214 }
215
216 #[async_trait]
217 impl LlmClient for CountingClient {
218 async fn complete(
219 &self,
220 _prompt: &str,
221 _context: Option<&[String]>,
222 ) -> anyhow::Result<LlmResponse> {
223 self.calls.fetch_add(1, Ordering::SeqCst);
224 Ok(LlmResponse {
225 text: "ok".into(),
226 provider: "counting".into(),
227 model: "fake-model".into(),
228 cached: false,
229 meta: serde_json::json!({}),
230 })
231 }
232
233 fn provider_name(&self) -> &'static str {
234 "counting"
235 }
236 }
237
238 struct DeletingClient {
239 path: std::path::PathBuf,
240 }
241
242 #[async_trait]
243 impl LlmClient for DeletingClient {
244 async fn complete(
245 &self,
246 _prompt: &str,
247 _context: Option<&[String]>,
248 ) -> anyhow::Result<LlmResponse> {
249 std::fs::remove_file(&self.path)?;
250 Ok(LlmResponse {
251 text: r#"{"ok":true}"#.into(),
252 provider: "deleting".into(),
253 model: "fake-model".into(),
254 cached: false,
255 meta: serde_json::json!({}),
256 })
257 }
258
259 fn provider_name(&self) -> &'static str {
260 "deleting"
261 }
262 }
263
264 struct BoundSchemaMetric;
265
266 #[async_trait]
267 impl Metric for BoundSchemaMetric {
268 fn name(&self) -> &'static str {
269 "bound_schema"
270 }
271
272 async fn evaluate(
273 &self,
274 _tc: &TestCase,
275 expected: &Expected,
276 _resp: &LlmResponse,
277 ) -> anyhow::Result<MetricResult> {
278 let Expected::JsonSchema {
279 json_schema,
280 schema_file,
281 } = expected
282 else {
283 anyhow::bail!("expected json_schema contract");
284 };
285 anyhow::ensure!(
286 schema_file.is_none(),
287 "schema_file was reread after preflight"
288 );
289 anyhow::ensure!(
290 json_schema.contains("required"),
291 "schema bytes were not bound"
292 );
293 Ok(MetricResult::pass(1.0))
294 }
295 }
296
297 struct BoundSequenceMetric;
298
299 #[async_trait]
300 impl Metric for BoundSequenceMetric {
301 fn name(&self) -> &'static str {
302 "bound_sequence"
303 }
304
305 async fn evaluate(
306 &self,
307 _tc: &TestCase,
308 expected: &Expected,
309 _resp: &LlmResponse,
310 ) -> anyhow::Result<MetricResult> {
311 let Expected::SequenceValid {
312 policy,
313 sequence,
314 rules,
315 } = expected
316 else {
317 anyhow::bail!("expected sequence_valid contract");
318 };
319 anyhow::ensure!(policy.is_none(), "policy path survived binding");
320 anyhow::ensure!(sequence.as_deref() == Some(&[]), "inline sequence changed");
321 anyhow::ensure!(
322 matches!(rules.as_deref(), Some([crate::model::SequenceRule::Require { tool }]) if tool.tool() == "Search"),
323 "file-backed rules were not merged into the snapshot"
324 );
325 Ok(MetricResult::pass(1.0))
326 }
327 }
328
329 fn runner_for_contract_tests(
330 client: Arc<dyn LlmClient>,
331 metrics: Vec<Arc<dyn Metric>>,
332 rerun_failures: u32,
333 ) -> Runner {
334 let store = Store::memory().expect("in-memory store");
335 store.init_schema().expect("schema init");
336 Runner {
337 store: store.clone(),
338 cache: VcrCache::new(store),
339 client,
340 metrics,
341 policy: RunPolicy {
342 rerun_failures,
343 quarantine_mode: QuarantineMode::Off,
344 replay_strict: false,
345 },
346 _network_guard: None,
347 embedder: None,
348 refresh_embeddings: false,
349 incremental: false,
350 refresh_cache: false,
351 judge: None,
352 baseline: None,
353 }
354 }
355
356 fn single_test_config(on_error: ErrorPolicy) -> EvalConfig {
357 EvalConfig {
358 version: 1,
359 suite: "runner-contract".to_string(),
360 model: "fake-model".to_string(),
361 settings: Settings {
362 parallel: Some(1),
363 cache: Some(false),
364 seed: Some(1234),
365 on_error,
366 ..Default::default()
367 },
368 thresholds: Default::default(),
369 otel: Default::default(),
370 tests: vec![TestCase {
371 id: "t1".to_string(),
372 input: TestInput {
373 prompt: "contract prompt".to_string(),
374 context: None,
375 },
376 expected: Expected::MustContain {
378 must_contain: vec!["ok".to_string()],
379 },
380 assertions: None,
381 on_error: None,
382 tags: vec![],
383 metadata: None,
384 }],
385 }
386 }
387
388 fn config_with_test_ids(ids: &[&str], on_error: ErrorPolicy) -> EvalConfig {
389 EvalConfig {
390 version: 1,
391 suite: "runner-contract".to_string(),
392 model: "fake-model".to_string(),
393 settings: Settings {
394 parallel: Some(1),
395 cache: Some(false),
396 seed: Some(1234),
397 on_error,
398 ..Default::default()
399 },
400 thresholds: Default::default(),
401 otel: Default::default(),
402 tests: ids
403 .iter()
404 .map(|id| TestCase {
405 id: (*id).to_string(),
406 input: TestInput {
407 prompt: format!("prompt-{id}"),
408 context: None,
409 },
410 expected: Expected::MustContain {
411 must_contain: vec!["ok".to_string()],
412 },
413 assertions: None,
414 on_error: None,
415 tags: vec![],
416 metadata: None,
417 })
418 .collect(),
419 }
420 }
421
422 #[tokio::test]
423 async fn runner_contract_flake_fail_then_pass_classified_flaky() -> anyhow::Result<()> {
424 let cfg = single_test_config(ErrorPolicy::Block);
425 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
426 let metric = Arc::new(ScriptedMetric::fail_then_pass());
427 let runner = runner_for_contract_tests(client, vec![metric], 1);
428
429 let artifacts = runner.run_suite(&cfg, None).await?;
430 let row = artifacts
431 .results
432 .iter()
433 .find(|r| r.test_id == "t1")
434 .expect("result for t1");
435
436 assert_eq!(row.status, TestStatus::Flaky);
437 assert_eq!(row.message, "flake detected (rerun passed)");
438 let attempts = row.attempts.as_ref().expect("attempts");
439 assert_eq!(attempts.len(), 2);
440 assert_eq!(attempts[0].status, TestStatus::Fail);
441 assert_eq!(attempts[1].status, TestStatus::Pass);
442 Ok(())
443 }
444
445 #[tokio::test]
446 async fn runner_contract_fail_after_retries_stays_fail() -> anyhow::Result<()> {
447 let cfg = single_test_config(ErrorPolicy::Block);
448 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
449 let metric = Arc::new(ScriptedMetric::always_fail());
450 let runner = runner_for_contract_tests(client, vec![metric], 1);
451
452 let artifacts = runner.run_suite(&cfg, None).await?;
453 let row = artifacts
454 .results
455 .iter()
456 .find(|r| r.test_id == "t1")
457 .expect("result for t1");
458
459 assert_eq!(row.status, TestStatus::Fail);
460 assert!(
461 row.message.contains("failed: scripted"),
462 "expected stable failure reason, got: {}",
463 row.message
464 );
465 let attempts = row.attempts.as_ref().expect("attempts");
466 assert_eq!(attempts.len(), 2);
467 assert_eq!(attempts[0].status, TestStatus::Fail);
468 assert_eq!(attempts[1].status, TestStatus::Fail);
469 Ok(())
470 }
471
472 #[tokio::test]
473 async fn runner_contract_on_error_allow_marks_allowed_and_policy_applied() -> anyhow::Result<()>
474 {
475 let cfg = single_test_config(ErrorPolicy::Allow);
476 let client = Arc::new(ErrorClient);
477 let runner = runner_for_contract_tests(client, vec![], 2);
478
479 let artifacts = runner.run_suite(&cfg, None).await?;
480 let row = artifacts
481 .results
482 .iter()
483 .find(|r| r.test_id == "t1")
484 .expect("result for t1");
485
486 assert_eq!(row.status, TestStatus::AllowedOnError);
487 assert_eq!(row.error_policy_applied, Some(ErrorPolicy::Allow));
488 assert_eq!(row.details["policy_applied"], serde_json::json!("allow"));
489 let attempts = row.attempts.as_ref().expect("attempts");
490 assert_eq!(attempts.len(), 1);
491 assert_eq!(attempts[0].status, TestStatus::AllowedOnError);
492 Ok(())
493 }
494
495 #[tokio::test]
496 async fn runner_contract_results_sorted_by_test_id() -> anyhow::Result<()> {
497 let mut cfg = config_with_test_ids(&["t3", "t1", "t2"], ErrorPolicy::Block);
498 cfg.settings.parallel = Some(3);
499 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
500 let metric = Arc::new(ScriptedMetric::always_pass());
501 let runner = runner_for_contract_tests(client, vec![metric], 0);
502
503 let artifacts = runner.run_suite(&cfg, None).await?;
504 let ids: Vec<_> = artifacts
505 .results
506 .iter()
507 .map(|r| r.test_id.as_str())
508 .collect();
509 assert_eq!(ids, vec!["t1", "t2", "t3"]);
510 Ok(())
511 }
512
513 #[tokio::test]
514 async fn runner_rejects_an_unresolved_expected_reference_before_execution() {
515 let mut cfg = single_test_config(ErrorPolicy::Block);
516 cfg.tests[0].expected = Expected::Reference {
517 path: "checks/expected.yaml".to_string(),
518 };
519 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
520 let runner =
521 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
522
523 let err = runner
524 .run_test_once(&cfg, &cfg.tests[0])
525 .await
526 .expect_err("an unresolved migration reference must not reach metric dispatch");
527 assert!(err.to_string().contains("unresolved `$ref`"), "{err:#}");
528 }
529
530 #[tokio::test]
531 async fn on_error_allow_cannot_turn_invalid_configuration_into_a_pass() {
532 let mut cfg = single_test_config(ErrorPolicy::Allow);
533 cfg.tests[0].expected = Expected::Reference {
534 path: "checks/expected.yaml".to_string(),
535 };
536 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
537 let runner =
538 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
539
540 let err = runner
541 .run_suite(&cfg, None)
542 .await
543 .expect_err("configuration errors must precede the on_error policy");
544 assert!(err.to_string().contains("unresolved `$ref`"), "{err:#}");
545 }
546
547 #[tokio::test]
548 async fn static_expected_inputs_are_validated_before_provider_dispatch() {
549 let mut cfg = single_test_config(ErrorPolicy::Allow);
550 cfg.tests[0].expected = Expected::RegexMatch {
551 pattern: "(".into(),
552 flags: Vec::new(),
553 };
554 let client = Arc::new(CountingClient {
555 calls: AtomicUsize::new(0),
556 });
557 let runner = runner_for_contract_tests(client.clone(), vec![], 0);
558
559 let err = runner
560 .run_suite(&cfg, None)
561 .await
562 .expect_err("invalid regex must fail before provider dispatch");
563 assert!(err.to_string().contains("invalid regex"), "{err:#}");
564 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
565
566 cfg.tests[0].expected = Expected::ArgsValid {
567 policy: None,
568 schema: Some(serde_json::json!({
569 "Search": {"type": "definitely-not-a-json-schema-type"}
570 })),
571 };
572 let err = runner
573 .run_suite(&cfg, None)
574 .await
575 .expect_err("invalid schema must fail before provider dispatch");
576 assert!(err.to_string().contains("failed to compile"), "{err:#}");
577 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
578
579 let dir = tempfile::tempdir().expect("temporary policy directory");
580 let policy_path = dir.path().join("invalid-policy.yaml");
581 std::fs::write(
582 &policy_path,
583 "version: '2.0'\nschemas: definitely-not-a-map\n",
584 )
585 .expect("write invalid policy");
586 cfg.tests[0].expected = Expected::ArgsValid {
587 policy: Some(policy_path.to_string_lossy().into_owned()),
588 schema: None,
589 };
590 let err = runner
591 .run_suite(&cfg, None)
592 .await
593 .expect_err("invalid policy must fail before provider dispatch");
594 assert!(
595 err.to_string().contains("schemas must be a mapping"),
596 "{err:#}"
597 );
598 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
599 }
600
601 #[tokio::test]
602 async fn schema_file_bytes_are_bound_before_provider_dispatch() {
603 let dir = tempfile::tempdir().expect("temporary schema directory");
604 let schema_path = dir.path().join("response.schema.json");
605 std::fs::write(&schema_path, r#"{"type":"object","required":["ok"]}"#)
606 .expect("write schema");
607 let mut cfg = single_test_config(ErrorPolicy::Allow);
608 cfg.tests[0].expected = Expected::JsonSchema {
609 json_schema: String::new(),
610 schema_file: Some(schema_path.to_string_lossy().into_owned()),
611 };
612 let runner = runner_for_contract_tests(
613 Arc::new(DeletingClient {
614 path: schema_path.clone(),
615 }),
616 vec![Arc::new(BoundSchemaMetric)],
617 0,
618 );
619
620 let artifacts = runner
621 .run_suite(&cfg, None)
622 .await
623 .expect("validated bytes must survive deletion of the source file");
624 assert_eq!(artifacts.results[0].status, TestStatus::Pass);
625 assert!(!schema_path.exists());
626 }
627
628 #[tokio::test]
629 async fn schema_file_content_participates_in_incremental_fingerprint() {
630 let dir = tempfile::tempdir().expect("temporary schema directory");
631 let schema_path = dir.path().join("response.schema.json");
632 std::fs::write(&schema_path, r#"{"type":"string"}"#).expect("write first schema");
633 let mut cfg = single_test_config(ErrorPolicy::Block);
634 cfg.tests[0].expected = Expected::JsonSchema {
635 json_schema: String::new(),
636 schema_file: Some(schema_path.to_string_lossy().into_owned()),
637 };
638 let client = Arc::new(CountingClient {
639 calls: AtomicUsize::new(0),
640 });
641 let mut runner = runner_for_contract_tests(
642 client.clone(),
643 vec![Arc::new(ScriptedMetric::always_pass())],
644 0,
645 );
646 runner.incremental = true;
647
648 runner.run_suite(&cfg, None).await.expect("first run");
649 std::fs::write(&schema_path, r#"{"type":"number"}"#).expect("write changed schema");
650 runner
651 .run_suite(&cfg, None)
652 .await
653 .expect("changed schema must run again");
654
655 assert_eq!(client.calls.load(Ordering::SeqCst), 2);
656 }
657
658 #[tokio::test]
659 async fn vacuous_file_policy_is_rejected_before_provider_dispatch() {
660 let dir = tempfile::tempdir().expect("temporary policy directory");
661 let policy_path = dir.path().join("vacuous-policy.yaml");
662 let mut cfg = single_test_config(ErrorPolicy::Allow);
663 cfg.tests[0].expected = Expected::ArgsValid {
664 policy: Some(policy_path.to_string_lossy().into_owned()),
665 schema: None,
666 };
667 let client = Arc::new(CountingClient {
668 calls: AtomicUsize::new(0),
669 });
670 let runner = runner_for_contract_tests(client.clone(), vec![], 0);
671
672 for (source, expected_error) in [
673 ("version: '2.0'\n", "asserts nothing"),
674 ("version: '2.0'\nconstraints: {}\n", "not enforced"),
675 ("version: '2.0'\nallow: ['*']\n", "asserts nothing"),
676 (
677 "version: '2.0'\ndeny: ['exec']\nenforcement:\n unconstrained_tools: typo\n",
678 "must be one of",
679 ),
680 ("Search: true\n", "asserts nothing"),
681 ("Search: {}\n", "asserts nothing"),
682 ] {
683 std::fs::write(&policy_path, source).expect("write policy");
684 let err = runner
685 .run_suite(&cfg, None)
686 .await
687 .expect_err("file policy without an enforced constraint must be rejected");
688 assert!(err.to_string().contains(expected_error), "{err:#}");
689 }
690 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
691 }
692
693 #[tokio::test]
694 async fn mixed_sequence_policy_bytes_are_bound_before_provider_dispatch() {
695 let dir = tempfile::tempdir().expect("temporary policy directory");
696 let policy_path = dir.path().join("rules.yaml");
697 std::fs::write(&policy_path, "- type: require\n tool: Search\n")
698 .expect("write sequence policy");
699 let mut cfg = single_test_config(ErrorPolicy::Allow);
700 cfg.tests[0].expected = Expected::SequenceValid {
701 policy: Some(policy_path.to_string_lossy().into_owned()),
702 sequence: Some(Vec::new()),
703 rules: None,
704 };
705 let runner = runner_for_contract_tests(
706 Arc::new(DeletingClient {
707 path: policy_path.clone(),
708 }),
709 vec![Arc::new(BoundSequenceMetric)],
710 0,
711 );
712
713 let artifacts = runner
714 .run_suite(&cfg, None)
715 .await
716 .expect("mixed policy bytes must survive deletion after dispatch");
717 assert_eq!(artifacts.results[0].status, TestStatus::Pass);
718 assert!(!policy_path.exists());
719 }
720
721 #[tokio::test]
722 async fn runner_allows_omitted_expected_when_trace_assertions_carry_the_test() {
723 let mut cfg = single_test_config(ErrorPolicy::Block);
724 cfg.tests[0].expected = Expected::default();
725 cfg.tests[0].assertions = Some(vec![
726 crate::agent_assertions::model::TraceAssertion::TraceMaxSteps { max: 1 },
727 ]);
728 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
729 let runner =
730 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
731
732 runner
733 .run_test_once(&cfg, &cfg.tests[0])
734 .await
735 .expect("effective trace assertions may carry a test with no expected block");
736 }
737
738 #[tokio::test]
739 async fn runner_preserves_warning_only_compatibility_for_an_omitted_expected_block() {
740 let mut cfg = single_test_config(ErrorPolicy::Block);
741 cfg.tests[0].expected = Expected::default();
742 cfg.tests[0].assertions = None;
743 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
744 let runner =
745 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
746
747 runner
748 .run_test_once(&cfg, &cfg.tests[0])
749 .await
750 .expect("omitting both checks remains a validate warning, not a runtime error");
751 }
752
753 #[tokio::test]
754 async fn runner_contract_progress_sink_reports_done_total() -> anyhow::Result<()> {
755 let cfg = config_with_test_ids(&["p1", "p2", "p3"], ErrorPolicy::Block);
756 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
757 let metric = Arc::new(ScriptedMetric::always_pass());
758 let runner = runner_for_contract_tests(client, vec![metric], 0);
759
760 let events = Arc::new(std::sync::Mutex::new(Vec::<(usize, usize)>::new()));
761 let sink = {
762 let events = Arc::clone(&events);
763 Arc::new(move |ev: crate::report::progress::ProgressEvent| {
764 events
765 .lock()
766 .expect("progress lock")
767 .push((ev.done, ev.total));
768 }) as crate::report::progress::ProgressSink
769 };
770
771 let artifacts = runner.run_suite(&cfg, Some(sink)).await?;
772 assert_eq!(artifacts.results.len(), 3);
773
774 let observed = events.lock().expect("progress lock");
775 assert_eq!(observed.len(), 3);
776 assert_eq!(observed.last(), Some(&(3, 3)));
777 assert!(observed.windows(2).all(|w| w[0].0 < w[1].0));
778 Ok(())
779 }
780
781 #[tokio::test]
782 async fn runner_contract_relative_baseline_missing_warns_in_helper() -> anyhow::Result<()> {
783 let mut cfg = single_test_config(ErrorPolicy::Block);
784 cfg.settings.thresholding = Some(crate::model::ThresholdingSettings {
785 mode: Some("relative".to_string()),
786 max_drop: Some(0.05),
787 min_floor: None,
788 });
789
790 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
791 let metric = Arc::new(ScriptedMetric::always_pass());
792 let runner = runner_for_contract_tests(client, vec![], 0);
793 let baseline = crate::baseline::Baseline {
794 schema_version: 1,
795 suite: "runner-contract".to_string(),
796 assay_version: env!("CARGO_PKG_VERSION").to_string(),
797 created_at: "2026-01-01T00:00:00Z".to_string(),
798 config_fingerprint: "md5:test".to_string(),
799 git_info: None,
800 entries: vec![],
801 };
802 let tc = cfg.tests.first().cloned().expect("single test case");
803 let details = serde_json::json!({
804 "metrics": {
805 "scripted": {
806 "score": 1.0,
807 "passed": true,
808 "unstable": false,
809 "details": {}
810 }
811 }
812 });
813
814 let verdict = runner.check_baseline_regressions(&tc, &cfg, &details, &[metric], &baseline);
815 let (status, message) = verdict.expect("relative baseline decision");
816 assert_eq!(status, TestStatus::Warn);
817 assert_eq!(message, "missing baseline for t1/scripted");
818 Ok(())
819 }
820}