workshop_rs_cli/
conformance.rs

1//! Public contracts for Workshop test results.
2
3use serde::{Deserialize, Serialize};
4use std::collections::HashSet;
5
6use workshop_rs::catalog::{Catalog, CatalogIdentity, Kind, Locale};
7
8pub const CONFORMANCE_SCHEMA_VERSION: u32 = 2;
9
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
11#[serde(rename_all = "kebab-case")]
12pub enum FeatureNamespace {
13    Catalog,
14    Wir,
15    Settings,
16    Localization,
17}
18
19impl FeatureNamespace {
20    pub const fn as_str(self) -> &'static str {
21        match self {
22            Self::Catalog => "catalog",
23            Self::Wir => "wir",
24            Self::Settings => "settings",
25            Self::Localization => "localization",
26        }
27    }
28}
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
31#[serde(rename_all = "kebab-case")]
32pub enum FeatureKind {
33    Event,
34    Action,
35    Value,
36    Operator,
37    Enum,
38    EnumMember,
39    Setting,
40    Variable,
41    Subroutine,
42    ControlFlow,
43    String,
44    Localization,
45    ContentId,
46    Structural,
47}
48
49impl FeatureKind {
50    pub const fn as_str(self) -> &'static str {
51        match self {
52            Self::Event => "event",
53            Self::Action => "action",
54            Self::Value => "value",
55            Self::Operator => "operator",
56            Self::Enum => "enum",
57            Self::EnumMember => "enum-member",
58            Self::Setting => "setting",
59            Self::Variable => "variable",
60            Self::Subroutine => "subroutine",
61            Self::ControlFlow => "control-flow",
62            Self::String => "string",
63            Self::Localization => "localization",
64            Self::ContentId => "content-id",
65            Self::Structural => "structural",
66        }
67    }
68}
69
70impl From<Kind> for FeatureKind {
71    fn from(kind: Kind) -> Self {
72        match kind {
73            Kind::Structural => Self::Structural,
74            Kind::Action => Self::Action,
75            Kind::Value => Self::Value,
76            Kind::Event => Self::Event,
77            Kind::Operator => Self::Operator,
78            Kind::Enum => Self::Enum,
79            Kind::Setting => Self::Setting,
80        }
81    }
82}
83
84#[derive(Debug, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
85pub struct FeatureId {
86    pub namespace: FeatureNamespace,
87    pub kind: FeatureKind,
88    pub name: String,
89}
90
91impl FeatureId {
92    pub fn new(
93        namespace: FeatureNamespace,
94        kind: FeatureKind,
95        name: impl Into<String>,
96    ) -> Result<Self, ConformanceError> {
97        let name = name.into();
98        if name.is_empty() {
99            return Err(ConformanceError::invalid(
100                "feature.name",
101                "must not be empty",
102            ));
103        }
104        if name
105            .chars()
106            .any(|character| character.is_whitespace() || character.is_control())
107        {
108            return Err(ConformanceError::invalid(
109                "feature.name",
110                "must not contain whitespace or control characters",
111            ));
112        }
113        Ok(Self {
114            namespace,
115            kind,
116            name,
117        })
118    }
119
120    pub fn from_catalog(kind: Kind, id: impl Into<String>) -> Result<Self, ConformanceError> {
121        Self::new(FeatureNamespace::Catalog, kind.into(), id)
122    }
123
124    pub fn from_enum_member(
125        domain: impl Into<String>,
126        member: impl Into<String>,
127    ) -> Result<Self, ConformanceError> {
128        let domain = domain.into();
129        let member = member.into();
130        if domain.is_empty() || member.is_empty() {
131            return Err(ConformanceError::invalid(
132                "feature.name",
133                "enum member identities require a domain and member",
134            ));
135        }
136        Self::new(
137            FeatureNamespace::Catalog,
138            FeatureKind::EnumMember,
139            format!("{domain}/{member}"),
140        )
141    }
142
143    pub fn owned(
144        namespace: FeatureNamespace,
145        kind: FeatureKind,
146        name: impl Into<String>,
147    ) -> Result<Self, ConformanceError> {
148        Self::new(namespace, kind, name)
149    }
150}
151
152/// An immutable identity for a test input, expected result, or observed result.
153#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
154pub struct TestArtifact {
155    pub name: String,
156    pub revision: Option<String>,
157    pub path: Option<String>,
158    #[serde(rename = "sha256")]
159    pub sha256: Option<String>,
160    pub license: Option<String>,
161}
162
163impl TestArtifact {
164    pub fn new(name: impl Into<String>) -> Self {
165        Self {
166            name: name.into(),
167            revision: None,
168            path: None,
169            sha256: None,
170            license: None,
171        }
172    }
173}
174
175#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
176#[serde(rename_all = "kebab-case")]
177pub enum Equivalence {
178    Semantic,
179    Normalized,
180    ExactText,
181    NotComparable,
182}
183
184#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
185#[serde(rename_all = "camelCase")]
186pub struct Comparison {
187    pub mode: Equivalence,
188    pub expected: Option<TestArtifact>,
189    pub observed: Option<TestArtifact>,
190    pub normalizer: Option<String>,
191}
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
194#[serde(rename_all = "kebab-case")]
195pub enum ReasonCode {
196    Unsupported,
197    KnownGap,
198    UnexpectedRegression,
199    Inconclusive,
200}
201
202#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
203#[serde(rename_all = "camelCase")]
204pub struct ConformanceReason {
205    pub code: ReasonCode,
206    pub detail: String,
207}
208
209#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
210#[serde(rename_all = "kebab-case")]
211pub enum ConformanceStatus {
212    Matched,
213    Unsupported,
214    KnownGap,
215    UnexpectedRegression,
216    Inconclusive,
217}
218
219impl ConformanceStatus {
220    pub const fn is_match(self) -> bool {
221        matches!(self, Self::Matched)
222    }
223}
224
225/// One machine-readable result from a Workshop test or reference comparison.
226#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
227#[serde(rename_all = "camelCase")]
228pub struct ConformanceResult {
229    pub schema_version: u32,
230    pub case_id: String,
231    pub features: Vec<FeatureId>,
232    pub status: ConformanceStatus,
233    pub comparison: Comparison,
234    pub source: TestArtifact,
235    pub catalog: CatalogIdentity,
236    pub locale: Option<Locale>,
237    pub reason: Option<ConformanceReason>,
238}
239
240impl ConformanceResult {
241    pub fn validate(&self) -> Result<(), ConformanceError> {
242        if self.schema_version != CONFORMANCE_SCHEMA_VERSION {
243            return Err(ConformanceError::invalid(
244                "schemaVersion",
245                format!(
246                    "unsupported schema version {}; expected {}",
247                    self.schema_version, CONFORMANCE_SCHEMA_VERSION
248                ),
249            ));
250        }
251        validate_non_empty("caseId", &self.case_id)?;
252        let mut seen_features: HashSet<&FeatureId> = HashSet::with_capacity(self.features.len());
253        if self.features.is_empty() {
254            return Err(ConformanceError::invalid(
255                "features",
256                "must contain at least one feature",
257            ));
258        }
259        for (index, feature) in self.features.iter().enumerate() {
260            FeatureId::new(feature.namespace, feature.kind, feature.name.clone())
261                .map_err(|error| error.at(format!("features[{index}]")))?;
262            if !seen_features.insert(feature) {
263                return Err(ConformanceError::invalid(
264                    format!("features[{index}]"),
265                    "must not contain duplicate feature identities",
266                ));
267            }
268        }
269        validate_artifact("source", &self.source, false, false)?;
270        validate_catalog_identity(&self.catalog)?;
271        validate_comparison(&self.comparison, &self.source)?;
272        if self.status.is_match() {
273            if self.comparison.mode == Equivalence::NotComparable {
274                return Err(ConformanceError::invalid(
275                    "comparison.mode",
276                    "matched results must declare semantic, normalized, or exact-text equivalence",
277                ));
278            }
279            if self.comparison.expected.is_none() || self.comparison.observed.is_none() {
280                return Err(ConformanceError::invalid(
281                    "comparison",
282                    "matched results require expected and observed artifacts",
283                ));
284            }
285            if self.reason.is_some() {
286                return Err(ConformanceError::invalid(
287                    "reason",
288                    "matched results must not carry a reason",
289                ));
290            }
291        } else {
292            let reason = self.reason.as_ref().ok_or_else(|| {
293                ConformanceError::invalid(
294                    "reason",
295                    "non-matching results require a structured reason",
296                )
297            })?;
298            validate_reason(self.status, reason)?;
299            if self.status == ConformanceStatus::UnexpectedRegression
300                && self.comparison.mode == Equivalence::NotComparable
301            {
302                return Err(ConformanceError::invalid(
303                    "comparison.mode",
304                    "an unexpected regression must identify the comparison contract",
305                ));
306            }
307        }
308        Ok(())
309    }
310
311    pub fn validate_against(&self, catalog: &Catalog) -> Result<(), ConformanceError> {
312        self.validate()?;
313        if self.catalog != catalog.identity() {
314            return Err(ConformanceError::invalid(
315                "catalog",
316                "must match the catalog supplied to validate_against",
317            ));
318        }
319        for (index, feature) in self.features.iter().enumerate() {
320            if feature.namespace != FeatureNamespace::Catalog {
321                continue;
322            }
323            match feature.kind {
324                FeatureKind::Enum => {
325                    if catalog.enum_domain(&feature.name).is_none() {
326                        return Err(ConformanceError::invalid(
327                            format!("features[{index}]"),
328                            format!("unknown canonical enum domain '{}'", feature.name),
329                        ));
330                    }
331                }
332                FeatureKind::EnumMember => {
333                    let (domain, member) = feature.name.split_once('/').ok_or_else(|| {
334                        ConformanceError::invalid(
335                            format!("features[{index}]"),
336                            "enum-member identity must contain domain/member",
337                        )
338                    })?;
339                    let known = catalog.enum_domain(domain).is_some_and(|candidate| {
340                        candidate.members.iter().any(|item| item.member == member)
341                    });
342                    if !known {
343                        return Err(ConformanceError::invalid(
344                            format!("features[{index}]"),
345                            format!("unknown canonical enum member '{domain}/{member}'"),
346                        ));
347                    }
348                }
349                kind => {
350                    let catalog_kind = match kind {
351                        FeatureKind::Event => Kind::Event,
352                        FeatureKind::Action => Kind::Action,
353                        FeatureKind::Value => Kind::Value,
354                        FeatureKind::Operator => Kind::Operator,
355                        FeatureKind::Setting => Kind::Setting,
356                        FeatureKind::Structural => Kind::Structural,
357                        _ => {
358                            return Err(ConformanceError::invalid(
359                                format!("features[{index}]"),
360                                "this feature kind cannot use the catalog namespace",
361                            ));
362                        }
363                    };
364                    if catalog.entry(catalog_kind, &feature.name).is_none() {
365                        return Err(ConformanceError::invalid(
366                            format!("features[{index}]"),
367                            format!(
368                                "unknown canonical {} '{}'",
369                                catalog_kind.as_str(),
370                                feature.name
371                            ),
372                        ));
373                    }
374                }
375            }
376        }
377        Ok(())
378    }
379
380    pub fn from_json(json: &str) -> Result<Self, ConformanceDecodeError> {
381        let result: Self = serde_json::from_str(json).map_err(ConformanceDecodeError::Json)?;
382        result.validate().map_err(ConformanceDecodeError::Invalid)?;
383        Ok(result)
384    }
385
386    pub const fn is_match(&self) -> bool {
387        self.status.is_match()
388    }
389}
390
391fn validate_catalog_identity(catalog: &CatalogIdentity) -> Result<(), ConformanceError> {
392    validate_non_empty(
393        "catalog.implementationVersion",
394        &catalog.implementation_version,
395    )?;
396    validate_non_empty("catalog.catalogVersion", &catalog.catalog_version)?;
397    if catalog.catalog_digest.as_deref().is_none_or(str::is_empty) {
398        return Err(ConformanceError::invalid(
399            "catalog.catalogDigest",
400            "must contain a catalog digest",
401        ));
402    }
403    Ok(())
404}
405
406fn validate_artifact(
407    field: &str,
408    artifact: &TestArtifact,
409    require_revision: bool,
410    require_digest: bool,
411) -> Result<(), ConformanceError> {
412    validate_non_empty(&format!("{field}.name"), &artifact.name)?;
413    if require_revision && artifact.revision.as_deref().is_none_or(str::is_empty) {
414        return Err(ConformanceError::invalid(
415            format!("{field}.revision"),
416            "must identify an immutable revision",
417        ));
418    }
419    if let Some(digest) = &artifact.sha256 {
420        if !is_sha256_digest(digest) {
421            return Err(ConformanceError::invalid(
422                format!("{field}.sha256"),
423                "must be a 64-character hexadecimal SHA-256 digest",
424            ));
425        }
426    }
427    if require_digest && artifact.sha256.is_none() {
428        return Err(ConformanceError::invalid(
429            format!("{field}.sha256"),
430            "must contain a SHA-256 digest",
431        ));
432    }
433    Ok(())
434}
435
436fn validate_comparison(
437    comparison: &Comparison,
438    source: &TestArtifact,
439) -> Result<(), ConformanceError> {
440    if comparison.mode == Equivalence::Normalized
441        && comparison
442            .normalizer
443            .as_deref()
444            .is_none_or(|normalizer| normalizer.trim().is_empty())
445    {
446        return Err(ConformanceError::invalid(
447            "comparison.normalizer",
448            "normalized comparisons require a named normalizer",
449        ));
450    }
451    if let Some(expected) = &comparison.expected {
452        validate_artifact("comparison.expected", expected, false, false)?;
453        if expected == source {
454            return Err(ConformanceError::invalid(
455                "comparison.expected",
456                "expected artifact must not be the test input",
457            ));
458        }
459    }
460    if let Some(observed) = &comparison.observed {
461        validate_artifact("comparison.observed", observed, false, false)?;
462        if observed == source {
463            return Err(ConformanceError::invalid(
464                "comparison.observed",
465                "observed artifact must not be the test input",
466            ));
467        }
468    }
469    if comparison.expected.is_some() && comparison.expected == comparison.observed {
470        return Err(ConformanceError::invalid(
471            "comparison",
472            "expected and observed artifacts must be distinct",
473        ));
474    }
475    Ok(())
476}
477
478fn validate_reason(
479    status: ConformanceStatus,
480    reason: &ConformanceReason,
481) -> Result<(), ConformanceError> {
482    validate_non_empty("reason.detail", &reason.detail)?;
483    let expected = match status {
484        ConformanceStatus::Unsupported => ReasonCode::Unsupported,
485        ConformanceStatus::KnownGap => ReasonCode::KnownGap,
486        ConformanceStatus::UnexpectedRegression => ReasonCode::UnexpectedRegression,
487        ConformanceStatus::Inconclusive => ReasonCode::Inconclusive,
488        ConformanceStatus::Matched => {
489            return Err(ConformanceError::invalid(
490                "reason",
491                "matched results must not carry a reason",
492            ));
493        }
494    };
495    if reason.code != expected {
496        return Err(ConformanceError::invalid(
497            "reason.code",
498            "reason code must match conformance status",
499        ));
500    }
501    Ok(())
502}
503
504fn validate_non_empty(field: &str, value: &str) -> Result<(), ConformanceError> {
505    if value.trim().is_empty() {
506        Err(ConformanceError::invalid(field, "must not be empty"))
507    } else {
508        Ok(())
509    }
510}
511
512pub(crate) fn is_sha256_digest(value: &str) -> bool {
513    value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit())
514}
515
516#[derive(Debug, Clone, PartialEq, Eq)]
517pub struct ConformanceError {
518    pub field: String,
519    pub message: String,
520}
521
522impl ConformanceError {
523    fn invalid(field: impl Into<String>, message: impl Into<String>) -> Self {
524        Self {
525            field: field.into(),
526            message: message.into(),
527        }
528    }
529
530    fn at(self, field: String) -> Self {
531        Self { field, ..self }
532    }
533}
534
535impl std::fmt::Display for ConformanceError {
536    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
537        write!(
538            formatter,
539            "invalid conformance {}: {}",
540            self.field, self.message
541        )
542    }
543}
544
545impl std::error::Error for ConformanceError {}
546
547#[derive(Debug)]
548pub enum ConformanceDecodeError {
549    Json(serde_json::Error),
550    Invalid(ConformanceError),
551}
552
553impl std::fmt::Display for ConformanceDecodeError {
554    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
555        match self {
556            Self::Json(error) => write!(formatter, "invalid conformance JSON: {error}"),
557            Self::Invalid(error) => error.fmt(formatter),
558        }
559    }
560}
561
562impl std::error::Error for ConformanceDecodeError {}
563
564#[cfg(test)]
565mod tests {
566    use super::*;
567
568    fn catalog() -> CatalogIdentity {
569        Catalog::builtin().expect("built-in catalog").identity()
570    }
571
572    fn source() -> TestArtifact {
573        TestArtifact {
574            name: "fixture".to_string(),
575            revision: Some("abc123".to_string()),
576            path: Some("cases/basic.ws".to_string()),
577            sha256: Some("a".repeat(64)),
578            license: Some("MIT".to_string()),
579        }
580    }
581
582    fn artifact(name: &str) -> TestArtifact {
583        TestArtifact {
584            name: name.to_string(),
585            sha256: Some("b".repeat(64)),
586            ..TestArtifact::new(name)
587        }
588    }
589
590    fn matched() -> ConformanceResult {
591        ConformanceResult {
592            schema_version: CONFORMANCE_SCHEMA_VERSION,
593            case_id: "basic-action".to_string(),
594            features: vec![
595                FeatureId::from_catalog(Kind::Action, "setHealth").expect("valid feature"),
596            ],
597            status: ConformanceStatus::Matched,
598            comparison: Comparison {
599                mode: Equivalence::Semantic,
600                expected: Some(artifact("expected")),
601                observed: Some(artifact("observed")),
602                normalizer: None,
603            },
604            source: source(),
605            catalog: catalog(),
606            locale: Some(Locale::new("en-US")),
607            reason: None,
608        }
609    }
610
611    #[test]
612    fn feature_ids_are_locale_and_provider_independent() {
613        let feature = FeatureId::from_catalog(Kind::Action, "setHealth").expect("valid feature");
614        assert_eq!(feature.kind, FeatureKind::Action);
615        assert_eq!(feature.name, "setHealth");
616        assert_eq!(
617            serde_json::to_string(&feature).unwrap(),
618            r#"{"namespace":"catalog","kind":"action","name":"setHealth"}"#
619        );
620        let member = FeatureId::from_enum_member("Hero", "ANA").expect("valid member");
621        assert_eq!(member.name, "Hero/ANA");
622    }
623
624    #[test]
625    fn result_serializes_as_test_data_without_evidence_taxonomy() {
626        let result = matched();
627        result.validate().expect("valid result");
628        result
629            .validate_against(&Catalog::builtin().expect("built-in catalog"))
630            .expect("catalog-backed feature exists");
631        let json = serde_json::to_string(&result).expect("serialize result");
632        let document: serde_json::Value = serde_json::from_str(&json).expect("valid result JSON");
633        assert!(document.get("evidence").is_none());
634        assert!(document.get("trackingRef").is_none());
635        let decoded = ConformanceResult::from_json(&json).expect("deserialize valid result");
636        assert_eq!(decoded, result);
637    }
638
639    #[test]
640    fn catalog_validation_rejects_fabricated_features_and_catalogs() {
641        let mut result = matched();
642        result.features = vec![
643            FeatureId::from_catalog(Kind::Action, "notAWorkshopAction")
644                .expect("syntactically valid feature"),
645        ];
646        assert!(
647            result
648                .validate_against(&Catalog::builtin().expect("built-in catalog"))
649                .is_err()
650        );
651
652        let mut result = matched();
653        result.catalog.catalog_digest = Some("f".repeat(64));
654        let error = result
655            .validate_against(&Catalog::builtin().expect("built-in catalog"))
656            .expect_err("result must be bound to the supplied catalog");
657        assert_eq!(error.field, "catalog");
658    }
659
660    #[test]
661    fn non_matching_results_need_the_matching_reason() {
662        let mut result = matched();
663        result.status = ConformanceStatus::KnownGap;
664        result.comparison.mode = Equivalence::NotComparable;
665        assert!(result.validate().is_err());
666        result.reason = Some(ConformanceReason {
667            code: ReasonCode::KnownGap,
668            detail: "client spelling is not available".to_string(),
669        });
670        result.validate().expect("documented gap");
671        assert!(!result.is_match());
672    }
673
674    #[test]
675    fn duplicate_features_blank_details_and_aliasing_are_invalid() {
676        let mut result = matched();
677        result.features.push(result.features[0].clone());
678        assert!(result.validate().is_err());
679
680        let mut result = matched();
681        result.status = ConformanceStatus::Inconclusive;
682        result.reason = Some(ConformanceReason {
683            code: ReasonCode::Inconclusive,
684            detail: "  \n".to_string(),
685        });
686        assert!(result.validate().is_err());
687
688        let mut result = matched();
689        result.comparison.observed = Some(result.source.clone());
690        assert!(result.validate().is_err());
691    }
692}
workshop_rs_cli/conformance.rs

workshop_rs_cli/
conformance.rs
