assay_core/engine/
runner.rs

1use crate::cache::vcr::VcrCache;
2use crate::metrics_api::Metric;
3use crate::model::{EvalConfig, LlmResponse, TestCase, TestResultRow, TestStatus};
4use crate::providers::llm::LlmClient;
5use crate::quarantine::QuarantineMode;
6use crate::report::progress::ProgressSink;
7use crate::report::RunArtifacts;
8use crate::storage::store::Store;
9use std::sync::Arc;
10
11#[path = "runner_next/mod.rs"]
12mod runner_next;
13
14#[derive(Debug, Clone)]
15pub struct RunPolicy {
16    pub rerun_failures: u32,
17    pub quarantine_mode: QuarantineMode,
18    pub replay_strict: bool,
19}
20
21impl Default for RunPolicy {
22    fn default() -> Self {
23        Self {
24            rerun_failures: 1,
25            quarantine_mode: QuarantineMode::Warn,
26            replay_strict: false,
27        }
28    }
29}
30
31pub struct Runner {
32    pub store: Store,
33    pub cache: VcrCache,
34    pub client: Arc<dyn LlmClient>,
35    pub metrics: Vec<Arc<dyn Metric>>,
36    pub policy: RunPolicy,
37    pub _network_guard: Option<crate::providers::network::NetworkPolicyGuard>,
38    pub embedder: Option<Arc<dyn crate::providers::embedder::Embedder>>,
39    pub refresh_embeddings: bool,
40    pub incremental: bool,
41    pub refresh_cache: bool,
42    pub judge: Option<crate::judge::JudgeService>,
43    pub baseline: Option<crate::baseline::Baseline>,
44}
45
46impl Runner {
47    /// Run the suite; results are collected in completion order internally but returned
48    /// sorted by test_id for deterministic output. If `progress` is set, it is called
49    /// after each test completes (E4.3 realtime progress).
50    pub async fn run_suite(
51        &self,
52        cfg: &EvalConfig,
53        progress: Option<ProgressSink>,
54    ) -> anyhow::Result<RunArtifacts> {
55        runner_next::execute::run_suite_impl(self, cfg, progress).await
56    }
57
58    fn apply_agent_assertions(
59        &self,
60        run_id: i64,
61        tc: &TestCase,
62        resp: &crate::model::LlmResponse,
63        final_row: &mut TestResultRow,
64    ) -> anyhow::Result<()> {
65        runner_next::assertions::apply_agent_assertions_impl(self, run_id, tc, resp, final_row)
66    }
67
68    async fn run_test_once(
69        &self,
70        cfg: &EvalConfig,
71        tc: &TestCase,
72    ) -> anyhow::Result<(TestResultRow, LlmResponse)> {
73        runner_next::single::run_test_once_impl(self, cfg, tc).await
74    }
75
76    async fn call_llm(&self, cfg: &EvalConfig, tc: &TestCase) -> anyhow::Result<LlmResponse> {
77        runner_next::execute::call_llm_impl(self, cfg, tc).await
78    }
79
80    fn check_baseline_regressions(
81        &self,
82        tc: &TestCase,
83        cfg: &EvalConfig,
84        details: &serde_json::Value,
85        metrics: &[Arc<dyn Metric>],
86        baseline: &crate::baseline::Baseline,
87    ) -> Option<(TestStatus, String)> {
88        runner_next::baseline::check_baseline_regressions_impl(
89            self, tc, cfg, details, metrics, baseline,
90        )
91    }
92
93    // Embeddings logic
94    async fn enrich_semantic(
95        &self,
96        _cfg: &EvalConfig,
97        tc: &TestCase,
98        resp: &mut LlmResponse,
99    ) -> anyhow::Result<()> {
100        runner_next::scoring::enrich_semantic_impl(self, _cfg, tc, resp).await
101    }
102
103    pub async fn embed_text(
104        &self,
105        model_id: &str,
106        embedder: &dyn crate::providers::embedder::Embedder,
107        text: &str,
108    ) -> anyhow::Result<(Vec<f32>, &'static str)> {
109        runner_next::cache::embed_text_impl(self, model_id, embedder, text).await
110    }
111
112    async fn enrich_judge(
113        &self,
114        cfg: &EvalConfig,
115        tc: &TestCase,
116        resp: &mut LlmResponse,
117    ) -> anyhow::Result<()> {
118        runner_next::scoring::enrich_judge_impl(self, cfg, tc, resp).await
119    }
120}
121
122#[cfg(test)]
123mod tests {
124    use super::*;
125    use crate::metrics_api::{Metric, MetricResult};
126    use crate::model::{Expected, Settings, TestInput};
127    use crate::on_error::ErrorPolicy;
128    use crate::providers::llm::fake::FakeClient;
129    use crate::providers::llm::LlmClient;
130    use async_trait::async_trait;
131    use std::sync::atomic::{AtomicUsize, Ordering};
132
133    #[derive(Clone, Copy)]
134    enum MetricMode {
135        FailThenPass,
136        AlwaysFail,
137        AlwaysPass,
138    }
139
140    struct ScriptedMetric {
141        mode: MetricMode,
142        calls: AtomicUsize,
143    }
144
145    impl ScriptedMetric {
146        fn fail_then_pass() -> Self {
147            Self {
148                mode: MetricMode::FailThenPass,
149                calls: AtomicUsize::new(0),
150            }
151        }
152
153        fn always_fail() -> Self {
154            Self {
155                mode: MetricMode::AlwaysFail,
156                calls: AtomicUsize::new(0),
157            }
158        }
159
160        fn always_pass() -> Self {
161            Self {
162                mode: MetricMode::AlwaysPass,
163                calls: AtomicUsize::new(0),
164            }
165        }
166    }
167
168    #[async_trait]
169    impl Metric for ScriptedMetric {
170        fn name(&self) -> &'static str {
171            "scripted"
172        }
173
174        async fn evaluate(
175            &self,
176            _tc: &TestCase,
177            _expected: &Expected,
178            _resp: &LlmResponse,
179        ) -> anyhow::Result<MetricResult> {
180            let n = self.calls.fetch_add(1, Ordering::SeqCst);
181            match self.mode {
182                MetricMode::FailThenPass => {
183                    if n == 0 {
184                        Ok(MetricResult::fail(0.0, "scripted_fail_once"))
185                    } else {
186                        Ok(MetricResult::pass(1.0))
187                    }
188                }
189                MetricMode::AlwaysFail => Ok(MetricResult::fail(0.0, "scripted_fail")),
190                MetricMode::AlwaysPass => Ok(MetricResult::pass(1.0)),
191            }
192        }
193    }
194
195    struct ErrorClient;
196
197    #[async_trait]
198    impl LlmClient for ErrorClient {
199        async fn complete(
200            &self,
201            _prompt: &str,
202            _context: Option<&[String]>,
203        ) -> anyhow::Result<LlmResponse> {
204            Err(anyhow::anyhow!("scripted provider error"))
205        }
206
207        fn provider_name(&self) -> &'static str {
208            "error_client"
209        }
210    }
211
212    struct CountingClient {
213        calls: AtomicUsize,
214    }
215
216    #[async_trait]
217    impl LlmClient for CountingClient {
218        async fn complete(
219            &self,
220            _prompt: &str,
221            _context: Option<&[String]>,
222        ) -> anyhow::Result<LlmResponse> {
223            self.calls.fetch_add(1, Ordering::SeqCst);
224            Ok(LlmResponse {
225                text: "ok".into(),
226                provider: "counting".into(),
227                model: "fake-model".into(),
228                cached: false,
229                meta: serde_json::json!({}),
230            })
231        }
232
233        fn provider_name(&self) -> &'static str {
234            "counting"
235        }
236    }
237
238    struct DeletingClient {
239        path: std::path::PathBuf,
240    }
241
242    #[async_trait]
243    impl LlmClient for DeletingClient {
244        async fn complete(
245            &self,
246            _prompt: &str,
247            _context: Option<&[String]>,
248        ) -> anyhow::Result<LlmResponse> {
249            std::fs::remove_file(&self.path)?;
250            Ok(LlmResponse {
251                text: r#"{"ok":true}"#.into(),
252                provider: "deleting".into(),
253                model: "fake-model".into(),
254                cached: false,
255                meta: serde_json::json!({}),
256            })
257        }
258
259        fn provider_name(&self) -> &'static str {
260            "deleting"
261        }
262    }
263
264    struct BoundSchemaMetric;
265
266    #[async_trait]
267    impl Metric for BoundSchemaMetric {
268        fn name(&self) -> &'static str {
269            "bound_schema"
270        }
271
272        async fn evaluate(
273            &self,
274            _tc: &TestCase,
275            expected: &Expected,
276            _resp: &LlmResponse,
277        ) -> anyhow::Result<MetricResult> {
278            let Expected::JsonSchema {
279                json_schema,
280                schema_file,
281            } = expected
282            else {
283                anyhow::bail!("expected json_schema contract");
284            };
285            anyhow::ensure!(
286                schema_file.is_none(),
287                "schema_file was reread after preflight"
288            );
289            anyhow::ensure!(
290                json_schema.contains("required"),
291                "schema bytes were not bound"
292            );
293            Ok(MetricResult::pass(1.0))
294        }
295    }
296
297    struct BoundSequenceMetric;
298
299    #[async_trait]
300    impl Metric for BoundSequenceMetric {
301        fn name(&self) -> &'static str {
302            "bound_sequence"
303        }
304
305        async fn evaluate(
306            &self,
307            _tc: &TestCase,
308            expected: &Expected,
309            _resp: &LlmResponse,
310        ) -> anyhow::Result<MetricResult> {
311            let Expected::SequenceValid {
312                policy,
313                sequence,
314                rules,
315            } = expected
316            else {
317                anyhow::bail!("expected sequence_valid contract");
318            };
319            anyhow::ensure!(policy.is_none(), "policy path survived binding");
320            anyhow::ensure!(sequence.as_deref() == Some(&[]), "inline sequence changed");
321            anyhow::ensure!(
322                matches!(rules.as_deref(), Some([crate::model::SequenceRule::Require { tool }]) if tool == "Search"),
323                "file-backed rules were not merged into the snapshot"
324            );
325            Ok(MetricResult::pass(1.0))
326        }
327    }
328
329    fn runner_for_contract_tests(
330        client: Arc<dyn LlmClient>,
331        metrics: Vec<Arc<dyn Metric>>,
332        rerun_failures: u32,
333    ) -> Runner {
334        let store = Store::memory().expect("in-memory store");
335        store.init_schema().expect("schema init");
336        Runner {
337            store: store.clone(),
338            cache: VcrCache::new(store),
339            client,
340            metrics,
341            policy: RunPolicy {
342                rerun_failures,
343                quarantine_mode: QuarantineMode::Off,
344                replay_strict: false,
345            },
346            _network_guard: None,
347            embedder: None,
348            refresh_embeddings: false,
349            incremental: false,
350            refresh_cache: false,
351            judge: None,
352            baseline: None,
353        }
354    }
355
356    fn single_test_config(on_error: ErrorPolicy) -> EvalConfig {
357        EvalConfig {
358            version: 1,
359            suite: "runner-contract".to_string(),
360            model: "fake-model".to_string(),
361            settings: Settings {
362                parallel: Some(1),
363                cache: Some(false),
364                seed: Some(1234),
365                on_error,
366                ..Default::default()
367            },
368            thresholds: Default::default(),
369            otel: Default::default(),
370            tests: vec![TestCase {
371                id: "t1".to_string(),
372                input: TestInput {
373                    prompt: "contract prompt".to_string(),
374                    context: None,
375                },
376                // Expected payload is not used by scripted metrics, but keeps test case valid.
377                expected: Expected::MustContain {
378                    must_contain: vec!["ok".to_string()],
379                },
380                assertions: None,
381                on_error: None,
382                tags: vec![],
383                metadata: None,
384            }],
385        }
386    }
387
388    fn config_with_test_ids(ids: &[&str], on_error: ErrorPolicy) -> EvalConfig {
389        EvalConfig {
390            version: 1,
391            suite: "runner-contract".to_string(),
392            model: "fake-model".to_string(),
393            settings: Settings {
394                parallel: Some(1),
395                cache: Some(false),
396                seed: Some(1234),
397                on_error,
398                ..Default::default()
399            },
400            thresholds: Default::default(),
401            otel: Default::default(),
402            tests: ids
403                .iter()
404                .map(|id| TestCase {
405                    id: (*id).to_string(),
406                    input: TestInput {
407                        prompt: format!("prompt-{id}"),
408                        context: None,
409                    },
410                    expected: Expected::MustContain {
411                        must_contain: vec!["ok".to_string()],
412                    },
413                    assertions: None,
414                    on_error: None,
415                    tags: vec![],
416                    metadata: None,
417                })
418                .collect(),
419        }
420    }
421
422    #[tokio::test]
423    async fn runner_contract_flake_fail_then_pass_classified_flaky() -> anyhow::Result<()> {
424        let cfg = single_test_config(ErrorPolicy::Block);
425        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
426        let metric = Arc::new(ScriptedMetric::fail_then_pass());
427        let runner = runner_for_contract_tests(client, vec![metric], 1);
428
429        let artifacts = runner.run_suite(&cfg, None).await?;
430        let row = artifacts
431            .results
432            .iter()
433            .find(|r| r.test_id == "t1")
434            .expect("result for t1");
435
436        assert_eq!(row.status, TestStatus::Flaky);
437        assert_eq!(row.message, "flake detected (rerun passed)");
438        let attempts = row.attempts.as_ref().expect("attempts");
439        assert_eq!(attempts.len(), 2);
440        assert_eq!(attempts[0].status, TestStatus::Fail);
441        assert_eq!(attempts[1].status, TestStatus::Pass);
442        Ok(())
443    }
444
445    #[tokio::test]
446    async fn runner_contract_fail_after_retries_stays_fail() -> anyhow::Result<()> {
447        let cfg = single_test_config(ErrorPolicy::Block);
448        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
449        let metric = Arc::new(ScriptedMetric::always_fail());
450        let runner = runner_for_contract_tests(client, vec![metric], 1);
451
452        let artifacts = runner.run_suite(&cfg, None).await?;
453        let row = artifacts
454            .results
455            .iter()
456            .find(|r| r.test_id == "t1")
457            .expect("result for t1");
458
459        assert_eq!(row.status, TestStatus::Fail);
460        assert!(
461            row.message.contains("failed: scripted"),
462            "expected stable failure reason, got: {}",
463            row.message
464        );
465        let attempts = row.attempts.as_ref().expect("attempts");
466        assert_eq!(attempts.len(), 2);
467        assert_eq!(attempts[0].status, TestStatus::Fail);
468        assert_eq!(attempts[1].status, TestStatus::Fail);
469        Ok(())
470    }
471
472    #[tokio::test]
473    async fn runner_contract_on_error_allow_marks_allowed_and_policy_applied() -> anyhow::Result<()>
474    {
475        let cfg = single_test_config(ErrorPolicy::Allow);
476        let client = Arc::new(ErrorClient);
477        let runner = runner_for_contract_tests(client, vec![], 2);
478
479        let artifacts = runner.run_suite(&cfg, None).await?;
480        let row = artifacts
481            .results
482            .iter()
483            .find(|r| r.test_id == "t1")
484            .expect("result for t1");
485
486        assert_eq!(row.status, TestStatus::AllowedOnError);
487        assert_eq!(row.error_policy_applied, Some(ErrorPolicy::Allow));
488        assert_eq!(row.details["policy_applied"], serde_json::json!("allow"));
489        let attempts = row.attempts.as_ref().expect("attempts");
490        assert_eq!(attempts.len(), 1);
491        assert_eq!(attempts[0].status, TestStatus::AllowedOnError);
492        Ok(())
493    }
494
495    #[tokio::test]
496    async fn runner_contract_results_sorted_by_test_id() -> anyhow::Result<()> {
497        let mut cfg = config_with_test_ids(&["t3", "t1", "t2"], ErrorPolicy::Block);
498        cfg.settings.parallel = Some(3);
499        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
500        let metric = Arc::new(ScriptedMetric::always_pass());
501        let runner = runner_for_contract_tests(client, vec![metric], 0);
502
503        let artifacts = runner.run_suite(&cfg, None).await?;
504        let ids: Vec<_> = artifacts
505            .results
506            .iter()
507            .map(|r| r.test_id.as_str())
508            .collect();
509        assert_eq!(ids, vec!["t1", "t2", "t3"]);
510        Ok(())
511    }
512
513    #[tokio::test]
514    async fn runner_rejects_an_unresolved_expected_reference_before_execution() {
515        let mut cfg = single_test_config(ErrorPolicy::Block);
516        cfg.tests[0].expected = Expected::Reference {
517            path: "checks/expected.yaml".to_string(),
518        };
519        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
520        let runner =
521            runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
522
523        let err = runner
524            .run_test_once(&cfg, &cfg.tests[0])
525            .await
526            .expect_err("an unresolved migration reference must not reach metric dispatch");
527        assert!(err.to_string().contains("unresolved `$ref`"), "{err:#}");
528    }
529
530    #[tokio::test]
531    async fn on_error_allow_cannot_turn_invalid_configuration_into_a_pass() {
532        let mut cfg = single_test_config(ErrorPolicy::Allow);
533        cfg.tests[0].expected = Expected::Reference {
534            path: "checks/expected.yaml".to_string(),
535        };
536        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
537        let runner =
538            runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
539
540        let err = runner
541            .run_suite(&cfg, None)
542            .await
543            .expect_err("configuration errors must precede the on_error policy");
544        assert!(err.to_string().contains("unresolved `$ref`"), "{err:#}");
545    }
546
547    #[tokio::test]
548    async fn static_expected_inputs_are_validated_before_provider_dispatch() {
549        let mut cfg = single_test_config(ErrorPolicy::Allow);
550        cfg.tests[0].expected = Expected::RegexMatch {
551            pattern: "(".into(),
552            flags: Vec::new(),
553        };
554        let client = Arc::new(CountingClient {
555            calls: AtomicUsize::new(0),
556        });
557        let runner = runner_for_contract_tests(client.clone(), vec![], 0);
558
559        let err = runner
560            .run_suite(&cfg, None)
561            .await
562            .expect_err("invalid regex must fail before provider dispatch");
563        assert!(err.to_string().contains("invalid regex"), "{err:#}");
564        assert_eq!(client.calls.load(Ordering::SeqCst), 0);
565
566        cfg.tests[0].expected = Expected::ArgsValid {
567            policy: None,
568            schema: Some(serde_json::json!({
569                "Search": {"type": "definitely-not-a-json-schema-type"}
570            })),
571        };
572        let err = runner
573            .run_suite(&cfg, None)
574            .await
575            .expect_err("invalid schema must fail before provider dispatch");
576        assert!(err.to_string().contains("failed to compile"), "{err:#}");
577        assert_eq!(client.calls.load(Ordering::SeqCst), 0);
578
579        let dir = tempfile::tempdir().expect("temporary policy directory");
580        let policy_path = dir.path().join("invalid-policy.yaml");
581        std::fs::write(
582            &policy_path,
583            "version: '2.0'\nschemas: definitely-not-a-map\n",
584        )
585        .expect("write invalid policy");
586        cfg.tests[0].expected = Expected::ArgsValid {
587            policy: Some(policy_path.to_string_lossy().into_owned()),
588            schema: None,
589        };
590        let err = runner
591            .run_suite(&cfg, None)
592            .await
593            .expect_err("invalid policy must fail before provider dispatch");
594        assert!(
595            err.to_string().contains("schemas must be a mapping"),
596            "{err:#}"
597        );
598        assert_eq!(client.calls.load(Ordering::SeqCst), 0);
599    }
600
601    #[tokio::test]
602    async fn schema_file_bytes_are_bound_before_provider_dispatch() {
603        let dir = tempfile::tempdir().expect("temporary schema directory");
604        let schema_path = dir.path().join("response.schema.json");
605        std::fs::write(&schema_path, r#"{"type":"object","required":["ok"]}"#)
606            .expect("write schema");
607        let mut cfg = single_test_config(ErrorPolicy::Allow);
608        cfg.tests[0].expected = Expected::JsonSchema {
609            json_schema: String::new(),
610            schema_file: Some(schema_path.to_string_lossy().into_owned()),
611        };
612        let runner = runner_for_contract_tests(
613            Arc::new(DeletingClient {
614                path: schema_path.clone(),
615            }),
616            vec![Arc::new(BoundSchemaMetric)],
617            0,
618        );
619
620        let artifacts = runner
621            .run_suite(&cfg, None)
622            .await
623            .expect("validated bytes must survive deletion of the source file");
624        assert_eq!(artifacts.results[0].status, TestStatus::Pass);
625        assert!(!schema_path.exists());
626    }
627
628    #[tokio::test]
629    async fn schema_file_content_participates_in_incremental_fingerprint() {
630        let dir = tempfile::tempdir().expect("temporary schema directory");
631        let schema_path = dir.path().join("response.schema.json");
632        std::fs::write(&schema_path, r#"{"type":"string"}"#).expect("write first schema");
633        let mut cfg = single_test_config(ErrorPolicy::Block);
634        cfg.tests[0].expected = Expected::JsonSchema {
635            json_schema: String::new(),
636            schema_file: Some(schema_path.to_string_lossy().into_owned()),
637        };
638        let client = Arc::new(CountingClient {
639            calls: AtomicUsize::new(0),
640        });
641        let mut runner = runner_for_contract_tests(
642            client.clone(),
643            vec![Arc::new(ScriptedMetric::always_pass())],
644            0,
645        );
646        runner.incremental = true;
647
648        runner.run_suite(&cfg, None).await.expect("first run");
649        std::fs::write(&schema_path, r#"{"type":"number"}"#).expect("write changed schema");
650        runner
651            .run_suite(&cfg, None)
652            .await
653            .expect("changed schema must run again");
654
655        assert_eq!(client.calls.load(Ordering::SeqCst), 2);
656    }
657
658    #[tokio::test]
659    async fn vacuous_file_policy_is_rejected_before_provider_dispatch() {
660        let dir = tempfile::tempdir().expect("temporary policy directory");
661        let policy_path = dir.path().join("vacuous-policy.yaml");
662        let mut cfg = single_test_config(ErrorPolicy::Allow);
663        cfg.tests[0].expected = Expected::ArgsValid {
664            policy: Some(policy_path.to_string_lossy().into_owned()),
665            schema: None,
666        };
667        let client = Arc::new(CountingClient {
668            calls: AtomicUsize::new(0),
669        });
670        let runner = runner_for_contract_tests(client.clone(), vec![], 0);
671
672        for (source, expected_error) in [
673            ("version: '2.0'\n", "asserts nothing"),
674            ("version: '2.0'\nconstraints: {}\n", "not enforced"),
675            ("version: '2.0'\nallow: ['*']\n", "asserts nothing"),
676            (
677                "version: '2.0'\ndeny: ['exec']\nenforcement:\n  unconstrained_tools: typo\n",
678                "must be one of",
679            ),
680            ("Search: true\n", "asserts nothing"),
681            ("Search: {}\n", "asserts nothing"),
682        ] {
683            std::fs::write(&policy_path, source).expect("write policy");
684            let err = runner
685                .run_suite(&cfg, None)
686                .await
687                .expect_err("file policy without an enforced constraint must be rejected");
688            assert!(err.to_string().contains(expected_error), "{err:#}");
689        }
690        assert_eq!(client.calls.load(Ordering::SeqCst), 0);
691    }
692
693    #[tokio::test]
694    async fn mixed_sequence_policy_bytes_are_bound_before_provider_dispatch() {
695        let dir = tempfile::tempdir().expect("temporary policy directory");
696        let policy_path = dir.path().join("rules.yaml");
697        std::fs::write(&policy_path, "- type: require\n  tool: Search\n")
698            .expect("write sequence policy");
699        let mut cfg = single_test_config(ErrorPolicy::Allow);
700        cfg.tests[0].expected = Expected::SequenceValid {
701            policy: Some(policy_path.to_string_lossy().into_owned()),
702            sequence: Some(Vec::new()),
703            rules: None,
704        };
705        let runner = runner_for_contract_tests(
706            Arc::new(DeletingClient {
707                path: policy_path.clone(),
708            }),
709            vec![Arc::new(BoundSequenceMetric)],
710            0,
711        );
712
713        let artifacts = runner
714            .run_suite(&cfg, None)
715            .await
716            .expect("mixed policy bytes must survive deletion after dispatch");
717        assert_eq!(artifacts.results[0].status, TestStatus::Pass);
718        assert!(!policy_path.exists());
719    }
720
721    #[tokio::test]
722    async fn runner_allows_omitted_expected_when_trace_assertions_carry_the_test() {
723        let mut cfg = single_test_config(ErrorPolicy::Block);
724        cfg.tests[0].expected = Expected::default();
725        cfg.tests[0].assertions = Some(vec![
726            crate::agent_assertions::model::TraceAssertion::TraceMaxSteps { max: 1 },
727        ]);
728        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
729        let runner =
730            runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
731
732        runner
733            .run_test_once(&cfg, &cfg.tests[0])
734            .await
735            .expect("effective trace assertions may carry a test with no expected block");
736    }
737
738    #[tokio::test]
739    async fn runner_preserves_warning_only_compatibility_for_an_omitted_expected_block() {
740        let mut cfg = single_test_config(ErrorPolicy::Block);
741        cfg.tests[0].expected = Expected::default();
742        cfg.tests[0].assertions = None;
743        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
744        let runner =
745            runner_for_contract_tests(client, vec![Arc::new(ScriptedMetric::always_pass())], 0);
746
747        runner
748            .run_test_once(&cfg, &cfg.tests[0])
749            .await
750            .expect("omitting both checks remains a validate warning, not a runtime error");
751    }
752
753    #[tokio::test]
754    async fn runner_contract_progress_sink_reports_done_total() -> anyhow::Result<()> {
755        let cfg = config_with_test_ids(&["p1", "p2", "p3"], ErrorPolicy::Block);
756        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
757        let metric = Arc::new(ScriptedMetric::always_pass());
758        let runner = runner_for_contract_tests(client, vec![metric], 0);
759
760        let events = Arc::new(std::sync::Mutex::new(Vec::<(usize, usize)>::new()));
761        let sink = {
762            let events = Arc::clone(&events);
763            Arc::new(move |ev: crate::report::progress::ProgressEvent| {
764                events
765                    .lock()
766                    .expect("progress lock")
767                    .push((ev.done, ev.total));
768            }) as crate::report::progress::ProgressSink
769        };
770
771        let artifacts = runner.run_suite(&cfg, Some(sink)).await?;
772        assert_eq!(artifacts.results.len(), 3);
773
774        let observed = events.lock().expect("progress lock");
775        assert_eq!(observed.len(), 3);
776        assert_eq!(observed.last(), Some(&(3, 3)));
777        assert!(observed.windows(2).all(|w| w[0].0 < w[1].0));
778        Ok(())
779    }
780
781    #[tokio::test]
782    async fn runner_contract_relative_baseline_missing_warns_in_helper() -> anyhow::Result<()> {
783        let mut cfg = single_test_config(ErrorPolicy::Block);
784        cfg.settings.thresholding = Some(crate::model::ThresholdingSettings {
785            mode: Some("relative".to_string()),
786            max_drop: Some(0.05),
787            min_floor: None,
788        });
789
790        let client = Arc::new(FakeClient::new("fake-model".to_string()).with_response("ok".into()));
791        let metric = Arc::new(ScriptedMetric::always_pass());
792        let runner = runner_for_contract_tests(client, vec![], 0);
793        let baseline = crate::baseline::Baseline {
794            schema_version: 1,
795            suite: "runner-contract".to_string(),
796            assay_version: env!("CARGO_PKG_VERSION").to_string(),
797            created_at: "2026-01-01T00:00:00Z".to_string(),
798            config_fingerprint: "md5:test".to_string(),
799            git_info: None,
800            entries: vec![],
801        };
802        let tc = cfg.tests.first().cloned().expect("single test case");
803        let details = serde_json::json!({
804            "metrics": {
805                "scripted": {
806                    "score": 1.0,
807                    "passed": true,
808                    "unstable": false,
809                    "details": {}
810                }
811            }
812        });
813
814        let verdict = runner.check_baseline_regressions(&tc, &cfg, &details, &[metric], &baseline);
815        let (status, message) = verdict.expect("relative baseline decision");
816        assert_eq!(status, TestStatus::Warn);
817        assert_eq!(message, "missing baseline for t1/scripted");
818        Ok(())
819    }
820}
assay_core/engine/runner.rs

assay_core/engine/
runner.rs
