assay_core/model/
validation.rs

1use crate::on_error::ErrorPolicy;
2
3use super::types::{EvalConfig, Expected, Settings, TestCase, TestStatus, ThresholdingConfig};
4
5pub(crate) fn is_default_otel(o: &crate::config::otel::OtelConfig) -> bool {
6    o == &crate::config::otel::OtelConfig::default()
7}
8
9pub(crate) fn is_default_thresholds(t: &crate::thresholds::ThresholdConfig) -> bool {
10    t == &crate::thresholds::ThresholdConfig::default()
11}
12
13pub(crate) fn is_default_error_policy(p: &ErrorPolicy) -> bool {
14    *p == ErrorPolicy::default()
15}
16
17pub(crate) fn is_default_settings(s: &Settings) -> bool {
18    s == &Settings::default()
19}
20
21pub(crate) fn default_one() -> u32 {
22    1
23}
24
25pub(crate) fn default_min_score() -> f64 {
26    0.80
27}
28
29impl EvalConfig {
30    pub fn is_legacy(&self) -> bool {
31        self.version == 0
32    }
33
34    pub fn has_legacy_usage(&self) -> bool {
35        self.tests
36            .iter()
37            .any(|t: &TestCase| t.expected.get_policy_path().is_some())
38    }
39
40    pub fn validate(&self) -> anyhow::Result<()> {
41        if self.version >= 1 {
42            for test in &self.tests {
43                if matches!(test.expected, Expected::Reference { .. }) {
44                    anyhow::bail!("$ref in expected block is not allowed in configVersion >= 1. Run `assay migrate` to inline policies.");
45                }
46            }
47        }
48        Ok(())
49    }
50
51    /// Get the effective error policy for a test.
52    /// Test-level on_error overrides suite-level settings.
53    pub fn effective_error_policy(&self, test: &TestCase) -> ErrorPolicy {
54        test.on_error.unwrap_or(self.settings.on_error)
55    }
56}
57
58impl Expected {
59    pub fn get_policy_path(&self) -> Option<&str> {
60        match self {
61            Expected::ArgsValid { policy, .. } => policy.as_deref(),
62            Expected::SequenceValid { policy, .. } => policy.as_deref(),
63            _ => None,
64        }
65    }
66
67    /// Per-test thresholding for baseline regression (mode/max_drop) when this Expected variant matches the metric.
68    pub fn thresholding_for_metric(&self, metric_name: &str) -> Option<&ThresholdingConfig> {
69        match (metric_name, self) {
70            ("semantic_similarity_to", Expected::SemanticSimilarityTo { thresholding, .. }) => {
71                thresholding.as_ref()
72            }
73            ("faithfulness", Expected::Faithfulness { thresholding, .. }) => thresholding.as_ref(),
74            ("relevance", Expected::Relevance { thresholding, .. }) => thresholding.as_ref(),
75            _ => None,
76        }
77    }
78}
79
80impl TestStatus {
81    pub fn parse(s: &str) -> Self {
82        match s {
83            "pass" => TestStatus::Pass,
84            "fail" => TestStatus::Fail,
85            "flaky" => TestStatus::Flaky,
86            "warn" => TestStatus::Warn,
87            "error" => TestStatus::Error,
88            "skipped" => TestStatus::Skipped,
89            "unstable" => TestStatus::Unstable,
90            "allowed_on_error" => TestStatus::AllowedOnError,
91            _ => TestStatus::Error,
92        }
93    }
94
95    /// Returns true if this status should be treated as passing for CI purposes
96    pub fn is_passing(&self) -> bool {
97        matches!(
98            self,
99            TestStatus::Pass | TestStatus::AllowedOnError | TestStatus::Warn
100        )
101    }
102
103    /// Returns true if this status should block CI
104    pub fn is_blocking(&self) -> bool {
105        matches!(self, TestStatus::Fail | TestStatus::Error)
106    }
107}
assay_core/model/validation.rs

assay_core/model/
validation.rs
