1use crate::cache::vcr::VcrCache;
2use crate::metrics_api::Metric;
3use crate::model::{EvalConfig, LlmResponse, TestCase, TestResultRow, TestStatus};
4use crate::providers::llm::LlmClient;
5use crate::quarantine::QuarantineMode;
6use crate::report::progress::ProgressSink;
7use crate::report::RunArtifacts;
8use crate::storage::store::Store;
9use std::sync::Arc;
10
11#[path = "runner_next/mod.rs"]
12mod runner_next;
13
14#[derive(Debug, Clone)]
15pub struct RunPolicy {
16 pub rerun_failures: u32,
17 pub quarantine_mode: QuarantineMode,
18 pub replay_strict: bool,
19}
20
21impl Default for RunPolicy {
22 fn default() -> Self {
23 Self {
24 rerun_failures: 1,
25 quarantine_mode: QuarantineMode::Warn,
26 replay_strict: false,
27 }
28 }
29}
30
31pub struct Runner {
32 pub store: Store,
33 pub cache: VcrCache,
34 pub client: Arc<dyn LlmClient>,
35 pub metrics: Vec<Arc<dyn Metric>>,
36 pub policy: RunPolicy,
37 pub _network_guard: Option<crate::providers::network::NetworkPolicyGuard>,
38 pub embedder: Option<Arc<dyn crate::providers::embedder::Embedder>>,
39 pub refresh_embeddings: bool,
40 pub incremental: bool,
41 pub refresh_cache: bool,
42 pub judge: Option<crate::judge::JudgeService>,
43 pub baseline: Option<crate::baseline::Baseline>,
44}
45
46impl Runner {
47 pub async fn run_suite(
51 &self,
52 cfg: &EvalConfig,
53 progress: Option<ProgressSink>,
54 ) -> anyhow::Result<RunArtifacts> {
55 runner_next::execute::run_suite_impl(self, cfg, progress).await
56 }
57
58 fn apply_agent_assertions(
59 &self,
60 run_id: i64,
61 tc: &TestCase,
62 final_row: &mut TestResultRow,
63 ) -> anyhow::Result<()> {
64 runner_next::assertions::apply_agent_assertions_impl(self, run_id, tc, final_row)
65 }
66
67 async fn run_test_once(
68 &self,
69 cfg: &EvalConfig,
70 tc: &TestCase,
71 ) -> anyhow::Result<(TestResultRow, LlmResponse)> {
72 runner_next::single::run_test_once_impl(self, cfg, tc).await
73 }
74
75 async fn call_llm(&self, cfg: &EvalConfig, tc: &TestCase) -> anyhow::Result<LlmResponse> {
76 runner_next::execute::call_llm_impl(self, cfg, tc).await
77 }
78
79 fn check_baseline_regressions(
80 &self,
81 tc: &TestCase,
82 cfg: &EvalConfig,
83 details: &serde_json::Value,
84 metrics: &[Arc<dyn Metric>],
85 baseline: &crate::baseline::Baseline,
86 ) -> Option<(TestStatus, String)> {
87 runner_next::baseline::check_baseline_regressions_impl(
88 self, tc, cfg, details, metrics, baseline,
89 )
90 }
91
92 async fn enrich_semantic(
94 &self,
95 _cfg: &EvalConfig,
96 tc: &TestCase,
97 resp: &mut LlmResponse,
98 ) -> anyhow::Result<()> {
99 runner_next::scoring::enrich_semantic_impl(self, _cfg, tc, resp).await
100 }
101
102 pub async fn embed_text(
103 &self,
104 model_id: &str,
105 embedder: &dyn crate::providers::embedder::Embedder,
106 text: &str,
107 ) -> anyhow::Result<(Vec<f32>, &'static str)> {
108 runner_next::cache::embed_text_impl(self, model_id, embedder, text).await
109 }
110
111 async fn enrich_judge(
112 &self,
113 cfg: &EvalConfig,
114 tc: &TestCase,
115 resp: &mut LlmResponse,
116 ) -> anyhow::Result<()> {
117 runner_next::scoring::enrich_judge_impl(self, cfg, tc, resp).await
118 }
119}
120
121#[cfg(test)]
122mod tests {
123 use super::*;
124 use crate::metrics_api::{Metric, MetricResult};
125 use crate::model::{Expected, Settings, TestInput};
126 use crate::on_error::ErrorPolicy;
127 use crate::providers::llm::fake::FakeClient;
128 use crate::providers::llm::LlmClient;
129 use async_trait::async_trait;
130 use std::sync::atomic::{AtomicUsize, Ordering};
131
132 #[derive(Clone, Copy)]
133 enum MetricMode {
134 FailThenPass,
135 AlwaysFail,
136 AlwaysPass,
137 }
138
139 struct ScriptedMetric {
140 mode: MetricMode,
141 calls: AtomicUsize,
142 }
143
144 impl ScriptedMetric {
145 fn fail_then_pass() -> Self {
146 Self {
147 mode: MetricMode::FailThenPass,
148 calls: AtomicUsize::new(0),
149 }
150 }
151
152 fn always_fail() -> Self {
153 Self {
154 mode: MetricMode::AlwaysFail,
155 calls: AtomicUsize::new(0),
156 }
157 }
158
159 fn always_pass() -> Self {
160 Self {
161 mode: MetricMode::AlwaysPass,
162 calls: AtomicUsize::new(0),
163 }
164 }
165 }
166
167 #[async_trait]
168 impl Metric for ScriptedMetric {
169 fn name(&self) -> &'static str {
170 "scripted"
171 }
172
173 async fn evaluate(
174 &self,
175 _tc: &TestCase,
176 _expected: &Expected,
177 _resp: &LlmResponse,
178 ) -> anyhow::Result<MetricResult> {
179 let n = self.calls.fetch_add(1, Ordering::SeqCst);
180 match self.mode {
181 MetricMode::FailThenPass => {
182 if n == 0 {
183 Ok(MetricResult::fail(0.0, "scripted_fail_once"))
184 } else {
185 Ok(MetricResult::pass(1.0))
186 }
187 }
188 MetricMode::AlwaysFail => Ok(MetricResult::fail(0.0, "scripted_fail")),
189 MetricMode::AlwaysPass => Ok(MetricResult::pass(1.0)),
190 }
191 }
192 }
193
194 struct ErrorClient;
195
196 #[async_trait]
197 impl LlmClient for ErrorClient {
198 async fn complete(
199 &self,
200 _prompt: &str,
201 _context: Option<&[String]>,
202 ) -> anyhow::Result<LlmResponse> {
203 Err(anyhow::anyhow!("scripted provider error"))
204 }
205
206 fn provider_name(&self) -> &'static str {
207 "error_client"
208 }
209 }
210
211 struct CountingClient {
212 calls: AtomicUsize,
213 }
214
215 #[async_trait]
216 impl LlmClient for CountingClient {
217 async fn complete(
218 &self,
219 _prompt: &str,
220 _context: Option<&[String]>,
221 ) -> anyhow::Result<LlmResponse> {
222 self.calls.fetch_add(1, Ordering::SeqCst);
223 Ok(LlmResponse {
224 text: "ok".into(),
225 provider: "counting".into(),
226 model: "fake-model".into(),
227 cached: false,
228 meta: serde_json::json!({}),
229 })
230 }
231
232 fn provider_name(&self) -> &'static str {
233 "counting"
234 }
235 }
236
237 struct DeletingClient {
238 path: std::path::PathBuf,
239 }
240
241 #[async_trait]
242 impl LlmClient for DeletingClient {
243 async fn complete(
244 &self,
245 _prompt: &str,
246 _context: Option<&[String]>,
247 ) -> anyhow::Result<LlmResponse> {
248 std::fs::remove_file(&self.path)?;
249 Ok(LlmResponse {
250 text: r#"{"ok":true}"#.into(),
251 provider: "deleting".into(),
252 model: "fake-model".into(),
253 cached: false,
254 meta: serde_json::json!({}),
255 })
256 }
257
258 fn provider_name(&self) -> &'static str {
259 "deleting"
260 }
261 }
262
263 struct BoundSchemaMetric;
264
265 #[async_trait]
266 impl Metric for BoundSchemaMetric {
267 fn name(&self) -> &'static str {
268 "bound_schema"
269 }
270
271 async fn evaluate(
272 &self,
273 _tc: &TestCase,
274 expected: &Expected,
275 _resp: &LlmResponse,
276 ) -> anyhow::Result<MetricResult> {
277 let Expected::JsonSchema {
278 json_schema,
279 schema_file,
280 } = expected
281 else {
282 anyhow::bail!("expected json_schema contract");
283 };
284 anyhow::ensure!(
285 schema_file.is_none(),
286 "schema_file was reread after preflight"
287 );
288 anyhow::ensure!(
289 json_schema.contains("required"),
290 "schema bytes were not bound"
291 );
292 Ok(MetricResult::pass(1.0))
293 }
294 }
295
296 struct BoundSequenceMetric;
297
298 #[async_trait]
299 impl Metric for BoundSequenceMetric {
300 fn name(&self) -> &'static str {
301 "bound_sequence"
302 }
303
304 async fn evaluate(
305 &self,
306 _tc: &TestCase,
307 expected: &Expected,
308 _resp: &LlmResponse,
309 ) -> anyhow::Result<MetricResult> {
310 let Expected::SequenceValid {
311 policy,
312 sequence,
313 rules,
314 } = expected
315 else {
316 anyhow::bail!("expected sequence_valid contract");
317 };
318 anyhow::ensure!(policy.is_none(), "policy path survived binding");
319 anyhow::ensure!(sequence.as_deref() == Some(&[]), "inline sequence changed");
320 anyhow::ensure!(
321 matches!(rules.as_deref(), Some([crate::model::SequenceRule::Require { tool }]) if tool == "Search"),
322 "file-backed rules were not merged into the snapshot"
323 );
324 Ok(MetricResult::pass(1.0))
325 }
326 }
327
328 fn runner_for_contract_tests(
329 client: Arc<dyn LlmClient>,
330 metrics: Vec<Arc<dyn Metric>>,
331 rerun_failures: u32,
332 ) -> Runner {
333 let store = Store::memory().expect("in-memory store");
334 store.init_schema().expect("schema init");
335 Runner {
336 store: store.clone(),
337 cache: VcrCache::new(store),
338 client,
339 metrics,
340 policy: RunPolicy {
341 rerun_failures,
342 quarantine_mode: QuarantineMode::Off,
343 replay_strict: false,
344 },
345 _network_guard: None,
346 embedder: None,
347 refresh_embeddings: false,
348 incremental: false,
349 refresh_cache: false,
350 judge: None,
351 baseline: None,
352 }
353 }
354
355 fn single_test_config(on_error: ErrorPolicy) -> EvalConfig {
356 EvalConfig {
357 version: 1,
358 suite: "runner-contract".to_string(),
359 model: "fake-model".to_string(),
360 settings: Settings {
361 parallel: Some(1),
362 cache: Some(false),
363 seed: Some(1234),
364 on_error,
365 ..Default::default()
366 },
367 thresholds: Default::default(),
368 otel: Default::default(),
369 tests: vec![TestCase {
370 id: "t1".to_string(),
371 input: TestInput {
372 prompt: "contract prompt".to_string(),
373 context: None,
374 },
375 expected: Expected::MustContain {
377 must_contain: vec!["ok".to_string()],
378 },
379 assertions: None,
380 on_error: None,
381 tags: vec![],
382 metadata: None,
383 }],
384 }
385 }
386
387 fn config_with_test_ids(ids: &[&str], on_error: ErrorPolicy) -> EvalConfig {
388 EvalConfig {
389 version: 1,
390 suite: "runner-contract".to_string(),
391 model: "fake-model".to_string(),
392 settings: Settings {
393 parallel: Some(1),
394 cache: Some(false),
395 seed: Some(1234),
396 on_error,
397 ..Default::default()
398 },
399 thresholds: Default::default(),
400 otel: Default::default(),
401 tests: ids
402 .iter()
403 .map(|id| TestCase {
404 id: (*id).to_string(),
405 input: TestInput {
406 prompt: format!("prompt-{id}"),
407 context: None,
408 },
409 expected: Expected::MustContain {
410 must_contain: vec!["ok".to_string()],
411 },
412 assertions: None,
413 on_error: None,
414 tags: vec![],
415 metadata: None,
416 })
417 .collect(),
418 }
419 }
420
421 #[tokio::test]
422 async fn runner_contract_flake_fail_then_pass_classified_flaky() -> anyhow::Result<()> {
423 let cfg = single_test_config(ErrorPolicy::Block);
424 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
425 let metric = Arc::new(ScriptedMetric::fail_then_pass());
426 let runner = runner_for_contract_tests(client, vec![metric], 1);
427
428 let artifacts = runner.run_suite(&cfg, None).await?;
429 let row = artifacts
430 .results
431 .iter()
432 .find(|r| r.test_id == "t1")
433 .expect("result for t1");
434
435 assert_eq!(row.status, TestStatus::Flaky);
436 assert_eq!(row.message, "flake detected (rerun passed)");
437 let attempts = row.attempts.as_ref().expect("attempts");
438 assert_eq!(attempts.len(), 2);
439 assert_eq!(attempts[0].status, TestStatus::Fail);
440 assert_eq!(attempts[1].status, TestStatus::Pass);
441 Ok(())
442 }
443
444 #[tokio::test]
445 async fn runner_contract_fail_after_retries_stays_fail() -> anyhow::Result<()> {
446 let cfg = single_test_config(ErrorPolicy::Block);
447 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
448 let metric = Arc::new(ScriptedMetric::always_fail());
449 let runner = runner_for_contract_tests(client, vec![metric], 1);
450
451 let artifacts = runner.run_suite(&cfg, None).await?;
452 let row = artifacts
453 .results
454 .iter()
455 .find(|r| r.test_id == "t1")
456 .expect("result for t1");
457
458 assert_eq!(row.status, TestStatus::Fail);
459 assert!(
460 row.message.contains("failed: scripted"),
461 "expected stable failure reason, got: {}",
462 row.message
463 );
464 let attempts = row.attempts.as_ref().expect("attempts");
465 assert_eq!(attempts.len(), 2);
466 assert_eq!(attempts[0].status, TestStatus::Fail);
467 assert_eq!(attempts[1].status, TestStatus::Fail);
468 Ok(())
469 }
470
471 #[tokio::test]
472 async fn runner_contract_on_error_allow_marks_allowed_and_policy_applied() -> anyhow::Result<()>
473 {
474 let cfg = single_test_config(ErrorPolicy::Allow);
475 let client = Arc::new(ErrorClient);
476 let runner = runner_for_contract_tests(client, vec![], 2);
477
478 let artifacts = runner.run_suite(&cfg, None).await?;
479 let row = artifacts
480 .results
481 .iter()
482 .find(|r| r.test_id == "t1")
483 .expect("result for t1");
484
485 assert_eq!(row.status, TestStatus::AllowedOnError);
486 assert_eq!(row.error_policy_applied, Some(ErrorPolicy::Allow));
487 assert_eq!(row.details["policy_applied"], serde_json::json!("allow"));
488 let attempts = row.attempts.as_ref().expect("attempts");
489 assert_eq!(attempts.len(), 1);
490 assert_eq!(attempts[0].status, TestStatus::AllowedOnError);
491 Ok(())
492 }
493
494 #[tokio::test]
495 async fn runner_contract_results_sorted_by_test_id() -> anyhow::Result<()> {
496 let mut cfg = config_with_test_ids(&["t3", "t1", "t2"], ErrorPolicy::Block);
497 cfg.settings.parallel = Some(3);
498 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
499 let metric = Arc::new(ScriptedMetric::always_pass());
500 let runner = runner_for_contract_tests(client, vec![metric], 0);
501
502 let artifacts = runner.run_suite(&cfg, None).await?;
503 let ids: Vec<_> = artifacts
504 .results
505 .iter()
506 .map(|r| r.test_id.as_str())
507 .collect();
508 assert_eq!(ids, vec!["t1", "t2", "t3"]);
509 Ok(())
510 }
511
512 #[tokio::test]
513 async fn runner_rejects_an_unresolved_expected_reference_before_execution() {
514 let mut cfg = single_test_config(ErrorPolicy::Block);
515 cfg.tests[0].expected = Expected::Reference {
516 path: "checks/expected.yaml".to_string(),
517 };
518 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
519 let runner =
520 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
521
522 let err = runner
523 .run_test_once(&cfg, &cfg.tests[0])
524 .await
525 .expect_err("an unresolved migration reference must not reach metric dispatch");
526 assert!(err.to_string().contains("unresolved `$ref`"), "{err:#}");
527 }
528
529 #[tokio::test]
530 async fn on_error_allow_cannot_turn_invalid_configuration_into_a_pass() {
531 let mut cfg = single_test_config(ErrorPolicy::Allow);
532 cfg.tests[0].expected = Expected::Reference {
533 path: "checks/expected.yaml".to_string(),
534 };
535 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
536 let runner =
537 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
538
539 let err = runner
540 .run_suite(&cfg, None)
541 .await
542 .expect_err("configuration errors must precede the on_error policy");
543 assert!(err.to_string().contains("unresolved `$ref`"), "{err:#}");
544 }
545
546 #[tokio::test]
547 async fn static_expected_inputs_are_validated_before_provider_dispatch() {
548 let mut cfg = single_test_config(ErrorPolicy::Allow);
549 cfg.tests[0].expected = Expected::RegexMatch {
550 pattern: "(".into(),
551 flags: Vec::new(),
552 };
553 let client = Arc::new(CountingClient {
554 calls: AtomicUsize::new(0),
555 });
556 let runner = runner_for_contract_tests(client.clone(), vec![], 0);
557
558 let err = runner
559 .run_suite(&cfg, None)
560 .await
561 .expect_err("invalid regex must fail before provider dispatch");
562 assert!(err.to_string().contains("invalid regex"), "{err:#}");
563 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
564
565 cfg.tests[0].expected = Expected::ArgsValid {
566 policy: None,
567 schema: Some(serde_json::json!({
568 "Search": {"type": "definitely-not-a-json-schema-type"}
569 })),
570 };
571 let err = runner
572 .run_suite(&cfg, None)
573 .await
574 .expect_err("invalid schema must fail before provider dispatch");
575 assert!(err.to_string().contains("failed to compile"), "{err:#}");
576 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
577
578 let dir = tempfile::tempdir().expect("temporary policy directory");
579 let policy_path = dir.path().join("invalid-policy.yaml");
580 std::fs::write(
581 &policy_path,
582 "version: '2.0'\nschemas: definitely-not-a-map\n",
583 )
584 .expect("write invalid policy");
585 cfg.tests[0].expected = Expected::ArgsValid {
586 policy: Some(policy_path.to_string_lossy().into_owned()),
587 schema: None,
588 };
589 let err = runner
590 .run_suite(&cfg, None)
591 .await
592 .expect_err("invalid policy must fail before provider dispatch");
593 assert!(
594 err.to_string().contains("schemas must be a mapping"),
595 "{err:#}"
596 );
597 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
598 }
599
600 #[tokio::test]
601 async fn schema_file_bytes_are_bound_before_provider_dispatch() {
602 let dir = tempfile::tempdir().expect("temporary schema directory");
603 let schema_path = dir.path().join("response.schema.json");
604 std::fs::write(&schema_path, r#"{"type":"object","required":["ok"]}"#)
605 .expect("write schema");
606 let mut cfg = single_test_config(ErrorPolicy::Allow);
607 cfg.tests[0].expected = Expected::JsonSchema {
608 json_schema: String::new(),
609 schema_file: Some(schema_path.to_string_lossy().into_owned()),
610 };
611 let runner = runner_for_contract_tests(
612 Arc::new(DeletingClient {
613 path: schema_path.clone(),
614 }),
615 vec![Arc::new(BoundSchemaMetric)],
616 0,
617 );
618
619 let artifacts = runner
620 .run_suite(&cfg, None)
621 .await
622 .expect("validated bytes must survive deletion of the source file");
623 assert_eq!(artifacts.results[0].status, TestStatus::Pass);
624 assert!(!schema_path.exists());
625 }
626
627 #[tokio::test]
628 async fn schema_file_content_participates_in_incremental_fingerprint() {
629 let dir = tempfile::tempdir().expect("temporary schema directory");
630 let schema_path = dir.path().join("response.schema.json");
631 std::fs::write(&schema_path, r#"{"type":"string"}"#).expect("write first schema");
632 let mut cfg = single_test_config(ErrorPolicy::Block);
633 cfg.tests[0].expected = Expected::JsonSchema {
634 json_schema: String::new(),
635 schema_file: Some(schema_path.to_string_lossy().into_owned()),
636 };
637 let client = Arc::new(CountingClient {
638 calls: AtomicUsize::new(0),
639 });
640 let mut runner = runner_for_contract_tests(
641 client.clone(),
642 vec![Arc::new(ScriptedMetric::always_pass())],
643 0,
644 );
645 runner.incremental = true;
646
647 runner.run_suite(&cfg, None).await.expect("first run");
648 std::fs::write(&schema_path, r#"{"type":"number"}"#).expect("write changed schema");
649 runner
650 .run_suite(&cfg, None)
651 .await
652 .expect("changed schema must run again");
653
654 assert_eq!(client.calls.load(Ordering::SeqCst), 2);
655 }
656
657 #[tokio::test]
658 async fn vacuous_file_policy_is_rejected_before_provider_dispatch() {
659 let dir = tempfile::tempdir().expect("temporary policy directory");
660 let policy_path = dir.path().join("vacuous-policy.yaml");
661 let mut cfg = single_test_config(ErrorPolicy::Allow);
662 cfg.tests[0].expected = Expected::ArgsValid {
663 policy: Some(policy_path.to_string_lossy().into_owned()),
664 schema: None,
665 };
666 let client = Arc::new(CountingClient {
667 calls: AtomicUsize::new(0),
668 });
669 let runner = runner_for_contract_tests(client.clone(), vec![], 0);
670
671 for (source, expected_error) in [
672 ("version: '2.0'\n", "asserts nothing"),
673 ("version: '2.0'\nconstraints: {}\n", "not enforced"),
674 ("version: '2.0'\nallow: ['*']\n", "asserts nothing"),
675 (
676 "version: '2.0'\ndeny: ['exec']\nenforcement:\n unconstrained_tools: typo\n",
677 "must be one of",
678 ),
679 ("Search: true\n", "asserts nothing"),
680 ("Search: {}\n", "asserts nothing"),
681 ] {
682 std::fs::write(&policy_path, source).expect("write policy");
683 let err = runner
684 .run_suite(&cfg, None)
685 .await
686 .expect_err("file policy without an enforced constraint must be rejected");
687 assert!(err.to_string().contains(expected_error), "{err:#}");
688 }
689 assert_eq!(client.calls.load(Ordering::SeqCst), 0);
690 }
691
692 #[tokio::test]
693 async fn mixed_sequence_policy_bytes_are_bound_before_provider_dispatch() {
694 let dir = tempfile::tempdir().expect("temporary policy directory");
695 let policy_path = dir.path().join("rules.yaml");
696 std::fs::write(&policy_path, "- type: require\n tool: Search\n")
697 .expect("write sequence policy");
698 let mut cfg = single_test_config(ErrorPolicy::Allow);
699 cfg.tests[0].expected = Expected::SequenceValid {
700 policy: Some(policy_path.to_string_lossy().into_owned()),
701 sequence: Some(Vec::new()),
702 rules: None,
703 };
704 let runner = runner_for_contract_tests(
705 Arc::new(DeletingClient {
706 path: policy_path.clone(),
707 }),
708 vec![Arc::new(BoundSequenceMetric)],
709 0,
710 );
711
712 let artifacts = runner
713 .run_suite(&cfg, None)
714 .await
715 .expect("mixed policy bytes must survive deletion after dispatch");
716 assert_eq!(artifacts.results[0].status, TestStatus::Pass);
717 assert!(!policy_path.exists());
718 }
719
720 #[tokio::test]
721 async fn runner_allows_omitted_expected_when_trace_assertions_carry_the_test() {
722 let mut cfg = single_test_config(ErrorPolicy::Block);
723 cfg.tests[0].expected = Expected::default();
724 cfg.tests[0].assertions = Some(vec![
725 crate::agent_assertions::model::TraceAssertion::TraceMaxSteps { max: 1 },
726 ]);
727 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
728 let runner =
729 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
730
731 runner
732 .run_test_once(&cfg, &cfg.tests[0])
733 .await
734 .expect("effective trace assertions may carry a test with no expected block");
735 }
736
737 #[tokio::test]
738 async fn runner_preserves_warning_only_compatibility_for_an_omitted_expected_block() {
739 let mut cfg = single_test_config(ErrorPolicy::Block);
740 cfg.tests[0].expected = Expected::default();
741 cfg.tests[0].assertions = None;
742 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
743 let runner =
744 runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
745
746 runner
747 .run_test_once(&cfg, &cfg.tests[0])
748 .await
749 .expect("omitting both checks remains a validate warning, not a runtime error");
750 }
751
752 #[tokio::test]
753 async fn runner_contract_progress_sink_reports_done_total() -> anyhow::Result<()> {
754 let cfg = config_with_test_ids(&["p1", "p2", "p3"], ErrorPolicy::Block);
755 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
756 let metric = Arc::new(ScriptedMetric::always_pass());
757 let runner = runner_for_contract_tests(client, vec![metric], 0);
758
759 let events = Arc::new(std::sync::Mutex::new(Vec::<(usize, usize)>::new()));
760 let sink = {
761 let events = Arc::clone(&events);
762 Arc::new(move |ev: crate::report::progress::ProgressEvent| {
763 events
764 .lock()
765 .expect("progress lock")
766 .push((ev.done, ev.total));
767 }) as crate::report::progress::ProgressSink
768 };
769
770 let artifacts = runner.run_suite(&cfg, Some(sink)).await?;
771 assert_eq!(artifacts.results.len(), 3);
772
773 let observed = events.lock().expect("progress lock");
774 assert_eq!(observed.len(), 3);
775 assert_eq!(observed.last(), Some(&(3, 3)));
776 assert!(observed.windows(2).all(|w| w[0].0 < w[1].0));
777 Ok(())
778 }
779
780 #[tokio::test]
781 async fn runner_contract_relative_baseline_missing_warns_in_helper() -> anyhow::Result<()> {
782 let mut cfg = single_test_config(ErrorPolicy::Block);
783 cfg.settings.thresholding = Some(crate::model::ThresholdingSettings {
784 mode: Some("relative".to_string()),
785 max_drop: Some(0.05),
786 min_floor: None,
787 });
788
789 let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
790 let metric = Arc::new(ScriptedMetric::always_pass());
791 let runner = runner_for_contract_tests(client, vec![], 0);
792 let baseline = crate::baseline::Baseline {
793 schema_version: 1,
794 suite: "runner-contract".to_string(),
795 assay_version: env!("CARGO_PKG_VERSION").to_string(),
796 created_at: "2026-01-01T00:00:00Z".to_string(),
797 config_fingerprint: "md5:test".to_string(),
798 git_info: None,
799 entries: vec![],
800 };
801 let tc = cfg.tests.first().cloned().expect("single test case");
802 let details = serde_json::json!({
803 "metrics": {
804 "scripted": {
805 "score": 1.0,
806 "passed": true,
807 "unstable": false,
808 "details": {}
809 }
810 }
811 });
812
813 let verdict = runner.check_baseline_regressions(&tc, &cfg, &details, &[metric], &baseline);
814 let (status, message) = verdict.expect("relative baseline decision");
815 assert_eq!(status, TestStatus::Warn);
816 assert_eq!(message, "missing baseline for t1/scripted");
817 Ok(())
818 }
819}