talos_core/
evaluation.rs

1//! Storage-neutral completion claims and independent evaluation state.
2//!
3//! This module defines the contract between an executor and the evaluator.  A claim is an
4//! assertion that work is ready to inspect; it is never a completion authority.  Reports are
5//! bound to the exact subject revision captured by the claim and derive their aggregate verdict
6//! from criterion-level results.
7
8use crate::work::{WorkIdentity, WorkKind};
9use schemars::JsonSchema;
10use serde::{Deserialize, Serialize};
11use std::collections::{HashMap, HashSet};
12use thiserror::Error;
13use uuid::Uuid;
14
15/// The exact workspace/content revision inspected by an evaluator.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
17pub struct WorkspaceRevision {
18    /// Stable identity of the workspace/content subject.
19    pub id: Uuid,
20    /// Monotonic content revision (for example a commit plus dirty-tree digest revision).
21    pub revision: u64,
22}
23
24/// Exact Mission, Goal and workspace subject to which a claim or report applies.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
26pub struct EvaluationSubject {
27    /// Mission identity and revision.
28    pub mission: WorkIdentity,
29    /// Goal identity and revision.
30    pub goal: WorkIdentity,
31    /// Workspace/content identity and revision.
32    pub workspace: WorkspaceRevision,
33}
34
35impl EvaluationSubject {
36    /// Validate that the subject contains the required Mission and Goal roles.
37    pub fn validate(&self) -> Result<(), EvaluationError> {
38        if self.mission.kind != WorkKind::Mission {
39            return Err(EvaluationError::InvalidSubject(
40                "mission identity is not a Mission",
41            ));
42        }
43        if self.goal.kind != WorkKind::Goal {
44            return Err(EvaluationError::InvalidSubject(
45                "goal identity is not a Goal",
46            ));
47        }
48        Ok(())
49    }
50
51    /// Return whether another subject is exactly the same evaluation subject.
52    #[must_use]
53    pub fn matches(&self, current: &Self) -> bool {
54        self == current
55    }
56}
57
58/// A stable reference to an artifact changed or inspected by an evaluation.
59#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
60pub struct ArtifactRef {
61    /// Stable artifact identity.
62    pub id: Uuid,
63    /// Human-readable locator, such as a path or commit object.
64    pub locator: String,
65}
66
67/// A stable reference to evidence.  Evidence is referential and never a verdict authority.
68#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
69pub struct EvidenceRef {
70    /// Stable evidence identity.
71    pub id: Uuid,
72    /// Producer or record kind (for example `validation`).
73    pub kind: String,
74}
75
76/// The semantic kind of an acceptance criterion.
77#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
78#[serde(rename_all = "snake_case")]
79pub enum CriterionKind {
80    /// A user-visible behavior requirement.
81    Behavior,
82    /// A technical/API contract requirement.
83    Technical,
84    /// A machine-verifiable validation requirement.
85    Validation,
86    /// A documentation or operational requirement.
87    Documentation,
88    /// An explicitly named project-specific criterion.
89    Custom(String),
90}
91
92/// One typed acceptance criterion for a Goal.
93#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
94pub struct AcceptanceCriterion {
95    /// Stable criterion identity.
96    pub id: Uuid,
97    /// Criterion category.
98    pub kind: CriterionKind,
99    /// Short requirement text.
100    pub statement: String,
101    /// Required criteria determine the aggregate PASS/FAIL result.
102    pub required: bool,
103}
104
105/// A completion assertion submitted by an executor.
106#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
107pub struct CompletionClaim {
108    /// Stable claim identity.
109    pub id: Uuid,
110    /// Exact subject revision being claimed.
111    pub subject: EvaluationSubject,
112    /// Immutable acceptance criteria snapshot.
113    pub criteria: Vec<AcceptanceCriterion>,
114    /// Artifacts the executor says were changed.
115    pub changed_artifacts: Vec<ArtifactRef>,
116    /// Evidence references offered as hints to the evaluator.
117    pub claimed_evidence: Vec<EvidenceRef>,
118    /// Executor summary; never an independent verdict.
119    pub executor_summary: String,
120}
121
122impl CompletionClaim {
123    /// Construct a claim in the evaluation-pending state.
124    pub fn new(
125        subject: EvaluationSubject,
126        criteria: Vec<AcceptanceCriterion>,
127        changed_artifacts: Vec<ArtifactRef>,
128        claimed_evidence: Vec<EvidenceRef>,
129        executor_summary: impl Into<String>,
130    ) -> Result<Self, EvaluationError> {
131        subject.validate()?;
132        validate_criteria(&criteria)?;
133        Ok(Self {
134            id: Uuid::new_v4(),
135            subject,
136            criteria,
137            changed_artifacts,
138            claimed_evidence,
139            executor_summary: executor_summary.into(),
140        })
141    }
142
143    /// Begin a storage-neutral evaluation state machine for this claim.
144    #[must_use]
145    pub fn evaluation(&self) -> Evaluation {
146        Evaluation {
147            claim: self.clone(),
148            state: EvaluationState::Pending,
149            report: None,
150        }
151    }
152}
153
154/// Criterion-level outcome produced by an evaluator.
155#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
156#[serde(rename_all = "snake_case")]
157pub enum CriterionVerdict {
158    /// Criterion is satisfied.
159    Pass,
160    /// Criterion is not satisfied.
161    Fail,
162    /// Available evidence is insufficient to decide.
163    Inconclusive,
164}
165
166/// Aggregate evaluation outcome derived from required criteria.
167#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
168#[serde(rename_all = "snake_case")]
169pub enum EvaluationVerdict {
170    /// Every required criterion passed.
171    Pass,
172    /// At least one required criterion failed.
173    Fail,
174    /// No required criterion failed, but at least one is inconclusive.
175    Inconclusive,
176}
177
178/// Severity assigned to one evaluator finding.
179#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
180#[serde(rename_all = "snake_case")]
181pub enum FindingSeverity {
182    /// Informational observation.
183    Info,
184    /// Non-blocking concern.
185    Warning,
186    /// Finding that prevents a passing criterion.
187    Blocking,
188}
189
190/// A bounded, criterion-linked evaluator finding.
191#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
192pub struct EvaluationFinding {
193    /// Stable finding identity.
194    pub id: Uuid,
195    /// Criterion this finding concerns, if criterion-specific.
196    pub criterion_id: Option<Uuid>,
197    /// Finding severity.
198    pub severity: FindingSeverity,
199    /// Concise explanation for rework or audit.
200    pub summary: String,
201    /// Referenced evidence; references do not issue the verdict.
202    pub evidence: Vec<EvidenceRef>,
203}
204
205/// One evaluator result for one acceptance criterion.
206#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
207pub struct CriterionEvaluation {
208    /// Criterion identity from the claim snapshot.
209    pub criterion_id: Uuid,
210    /// Evaluator outcome.
211    pub verdict: CriterionVerdict,
212    /// Evidence references supporting the outcome.
213    pub evidence: Vec<EvidenceRef>,
214    /// Findings associated with this criterion.
215    pub finding_ids: Vec<Uuid>,
216}
217
218/// A complete criterion-granular report bound to one exact claim subject.
219#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
220pub struct EvaluationReport {
221    /// Stable report identity.
222    pub id: Uuid,
223    /// Claim being evaluated.
224    pub claim_id: Uuid,
225    /// Exact subject inspected.
226    pub subject: EvaluationSubject,
227    /// Criterion-level outcomes.
228    pub results: Vec<CriterionEvaluation>,
229    /// Findings explaining outcomes.
230    pub findings: Vec<EvaluationFinding>,
231    /// Deterministically derived aggregate verdict.
232    pub verdict: EvaluationVerdict,
233}
234
235impl EvaluationReport {
236    /// Construct a report, rejecting missing, duplicate or unknown criteria and contradictions.
237    pub fn new(
238        claim: &CompletionClaim,
239        subject: EvaluationSubject,
240        results: Vec<CriterionEvaluation>,
241        findings: Vec<EvaluationFinding>,
242    ) -> Result<Self, EvaluationError> {
243        if !claim.subject.matches(&subject) {
244            return Err(EvaluationError::SubjectMismatch);
245        }
246        let criterion_ids: HashSet<_> = claim
247            .criteria
248            .iter()
249            .map(|criterion| criterion.id)
250            .collect();
251        let mut result_ids = HashSet::new();
252        if results.iter().any(|result| {
253            !criterion_ids.contains(&result.criterion_id) || !result_ids.insert(result.criterion_id)
254        }) {
255            return Err(EvaluationError::CriterionMismatch);
256        }
257        if result_ids.len() != criterion_ids.len() {
258            return Err(EvaluationError::CriterionMismatch);
259        }
260        let finding_ids: HashSet<_> = findings.iter().map(|finding| finding.id).collect();
261        if finding_ids.len() != findings.len()
262            || findings.iter().any(|finding| {
263                finding
264                    .criterion_id
265                    .is_some_and(|id| !criterion_ids.contains(&id))
266                    || finding.summary.trim().is_empty()
267            })
268            || results
269                .iter()
270                .flat_map(|result| result.finding_ids.iter())
271                .any(|id| !finding_ids.contains(id))
272        {
273            return Err(EvaluationError::FindingMismatch);
274        }
275        let verdict = aggregate_verdict(&claim.criteria, &results);
276        Ok(Self {
277            id: Uuid::new_v4(),
278            claim_id: claim.id,
279            subject,
280            results,
281            findings,
282            verdict,
283        })
284    }
285}
286
287/// Lifecycle of an evaluation report.
288#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
289#[serde(rename_all = "snake_case", tag = "kind", content = "verdict")]
290pub enum EvaluationState {
291    /// Claim is awaiting an independent evaluator.
292    Pending,
293    /// Evaluator is inspecting the exact subject.
294    Evaluating,
295    /// A report has been accepted for the current subject.
296    Verdict(EvaluationVerdict),
297    /// Subject changed after a report and the prior verdict no longer certifies it.
298    Stale,
299    /// Rework is required before a new claim can be submitted.
300    Rework,
301}
302
303/// Storage-neutral state machine for one completion claim.
304#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
305pub struct Evaluation {
306    /// Immutable claim under evaluation.
307    pub claim: CompletionClaim,
308    /// Current lifecycle state.
309    pub state: EvaluationState,
310    /// Accepted report, when one exists.
311    pub report: Option<EvaluationReport>,
312}
313
314impl Evaluation {
315    /// Return whether this evaluation contains a validated passing report for its claim.
316    #[must_use]
317    pub fn has_valid_pass(&self) -> bool {
318        if self.state != EvaluationState::Verdict(EvaluationVerdict::Pass) {
319            return false;
320        }
321        let Some(report) = &self.report else {
322            return false;
323        };
324        if report.claim_id != self.claim.id || !self.claim.subject.matches(&report.subject) {
325            return false;
326        }
327        if report.verdict != EvaluationVerdict::Pass {
328            return false;
329        }
330        EvaluationReport::new(
331            &self.claim,
332            report.subject,
333            report.results.clone(),
334            report.findings.clone(),
335        )
336        .is_ok_and(|validated| validated.verdict == report.verdict)
337    }
338
339    /// Transition `Pending` to `Evaluating`.
340    pub fn begin(&mut self) -> Result<(), EvaluationError> {
341        if self.state != EvaluationState::Pending {
342            return Err(EvaluationError::IllegalTransition);
343        }
344        self.state = EvaluationState::Evaluating;
345        Ok(())
346    }
347
348    /// Accept a report and transition `Evaluating` to its derived verdict.
349    pub fn accept_report(&mut self, report: EvaluationReport) -> Result<(), EvaluationError> {
350        if self.state != EvaluationState::Evaluating || report.claim_id != self.claim.id {
351            return Err(EvaluationError::IllegalTransition);
352        }
353        // Reports are public and deserializable, so revalidate the complete payload at the
354        // authority boundary instead of trusting a caller-provided aggregate verdict.
355        if !self.claim.subject.matches(&report.subject) {
356            return Err(EvaluationError::SubjectMismatch);
357        }
358        let validated = EvaluationReport::new(
359            &self.claim,
360            report.subject,
361            report.results.clone(),
362            report.findings.clone(),
363        )?;
364        if validated.verdict != report.verdict {
365            return Err(EvaluationError::VerdictMismatch);
366        }
367        self.state = EvaluationState::Verdict(report.verdict);
368        self.report = Some(report);
369        Ok(())
370    }
371
372    /// Mark a verdict stale when the relevant subject revision changes.
373    pub fn observe_subject(&mut self, current: EvaluationSubject) -> Result<(), EvaluationError> {
374        if self.state
375            != EvaluationState::Verdict(
376                self.report
377                    .as_ref()
378                    .map_or(EvaluationVerdict::Inconclusive, |report| report.verdict),
379            )
380        {
381            return Err(EvaluationError::IllegalTransition);
382        }
383        if !self.claim.subject.matches(&current) {
384            self.state = EvaluationState::Stale;
385        }
386        Ok(())
387    }
388
389    /// Return a failed/inconclusive evaluation to the executor for rework.
390    pub fn request_rework(&mut self) -> Result<(), EvaluationError> {
391        match self.state {
392            EvaluationState::Verdict(EvaluationVerdict::Fail | EvaluationVerdict::Inconclusive)
393            | EvaluationState::Stale => {
394                self.state = EvaluationState::Rework;
395                Ok(())
396            }
397            _ => Err(EvaluationError::IllegalTransition),
398        }
399    }
400}
401
402/// Errors raised while constructing or advancing the evaluation contract.
403#[derive(Debug, Clone, PartialEq, Eq, Error)]
404pub enum EvaluationError {
405    /// Mission/Goal roles or another subject invariant is invalid.
406    #[error("invalid evaluation subject: {0}")]
407    InvalidSubject(&'static str),
408    /// Criteria are empty, duplicated or malformed.
409    #[error("invalid acceptance criteria")]
410    InvalidCriteria,
411    /// A report does not exactly match the claim's criterion set.
412    #[error("evaluation criteria do not exactly match the claim")]
413    CriterionMismatch,
414    /// A report finding is duplicated, unknown or malformed.
415    #[error("evaluation findings do not match the report")]
416    FindingMismatch,
417    /// A report was produced for a different subject revision.
418    #[error("evaluation subject revision does not match the claim")]
419    SubjectMismatch,
420    /// An operation is not legal from the current lifecycle state.
421    #[error("illegal evaluation state transition")]
422    IllegalTransition,
423    /// A report's caller-provided aggregate verdict disagrees with its criterion results.
424    #[error("evaluation verdict contradicts criterion results")]
425    VerdictMismatch,
426}
427
428fn validate_criteria(criteria: &[AcceptanceCriterion]) -> Result<(), EvaluationError> {
429    let mut ids = HashSet::new();
430    if criteria.is_empty()
431        || criteria
432            .iter()
433            .any(|criterion| criterion.statement.trim().is_empty() || !ids.insert(criterion.id))
434    {
435        return Err(EvaluationError::InvalidCriteria);
436    }
437    Ok(())
438}
439
440fn aggregate_verdict(
441    criteria: &[AcceptanceCriterion],
442    results: &[CriterionEvaluation],
443) -> EvaluationVerdict {
444    let outcomes: HashMap<_, _> = results
445        .iter()
446        .map(|result| (result.criterion_id, result.verdict))
447        .collect();
448    let mut required = criteria.iter().filter(|criterion| criterion.required);
449    if required
450        .clone()
451        .any(|criterion| outcomes.get(&criterion.id) == Some(&CriterionVerdict::Fail))
452    {
453        EvaluationVerdict::Fail
454    } else if required
455        .any(|criterion| outcomes.get(&criterion.id) == Some(&CriterionVerdict::Inconclusive))
456    {
457        EvaluationVerdict::Inconclusive
458    } else {
459        EvaluationVerdict::Pass
460    }
461}
462
463#[cfg(test)]
464mod tests {
465    use super::*;
466
467    fn subject() -> EvaluationSubject {
468        EvaluationSubject {
469            mission: WorkIdentity {
470                id: Uuid::new_v4(),
471                kind: WorkKind::Mission,
472                revision: 1,
473            },
474            goal: WorkIdentity {
475                id: Uuid::new_v4(),
476                kind: WorkKind::Goal,
477                revision: 2,
478            },
479            workspace: WorkspaceRevision {
480                id: Uuid::new_v4(),
481                revision: 3,
482            },
483        }
484    }
485
486    fn criterion(required: bool) -> AcceptanceCriterion {
487        AcceptanceCriterion {
488            id: Uuid::new_v4(),
489            kind: CriterionKind::Behavior,
490            statement: "works".into(),
491            required,
492        }
493    }
494
495    fn result(id: Uuid, verdict: CriterionVerdict) -> CriterionEvaluation {
496        CriterionEvaluation {
497            criterion_id: id,
498            verdict,
499            evidence: vec![],
500            finding_ids: vec![],
501        }
502    }
503
504    #[test]
505    fn state_machine_rejects_self_certification_and_stales_on_revision() {
506        let claim = CompletionClaim::new(subject(), vec![criterion(true)], vec![], vec![], "done")
507            .expect("valid test fixture");
508        let mut evaluation = claim.evaluation();
509        assert_eq!(evaluation.state, EvaluationState::Pending);
510        let report = EvaluationReport::new(
511            &claim,
512            claim.subject,
513            vec![result(claim.criteria[0].id, CriterionVerdict::Pass)],
514            vec![],
515        )
516        .expect("valid test fixture");
517        assert_eq!(
518            evaluation.accept_report(report),
519            Err(EvaluationError::IllegalTransition)
520        );
521        evaluation.begin().expect("valid test fixture");
522        let report = EvaluationReport::new(
523            &claim,
524            claim.subject,
525            vec![result(claim.criteria[0].id, CriterionVerdict::Pass)],
526            vec![],
527        )
528        .expect("valid test fixture");
529        evaluation
530            .accept_report(report)
531            .expect("valid test fixture");
532        let mut changed = claim.subject;
533        changed.goal.revision += 1;
534        evaluation
535            .observe_subject(changed)
536            .expect("valid test fixture");
537        assert_eq!(evaluation.state, EvaluationState::Stale);
538        evaluation.request_rework().expect("valid test fixture");
539        assert_eq!(evaluation.state, EvaluationState::Rework);
540    }
541
542    #[test]
543    fn locale_is_not_an_evaluation_subject_component() {
544        let a = subject();
545        let b = a;
546        assert!(a.matches(&b));
547    }
548
549    #[test]
550    fn aggregate_requires_required_criteria_only() {
551        let required = criterion(true);
552        let optional = criterion(false);
553        let claim = CompletionClaim::new(
554            subject(),
555            vec![required.clone(), optional.clone()],
556            vec![],
557            vec![],
558            "done",
559        )
560        .expect("valid test fixture");
561        let report = EvaluationReport::new(
562            &claim,
563            claim.subject,
564            vec![
565                result(required.id, CriterionVerdict::Pass),
566                result(optional.id, CriterionVerdict::Fail),
567            ],
568            vec![],
569        )
570        .expect("valid test fixture");
571        assert_eq!(report.verdict, EvaluationVerdict::Pass);
572    }
573
574    #[test]
575    fn report_rejects_duplicates_and_subject_mismatch() {
576        let c = criterion(true);
577        let claim = CompletionClaim::new(subject(), vec![c.clone()], vec![], vec![], "done")
578            .expect("valid test fixture");
579        assert_eq!(
580            EvaluationReport::new(
581                &claim,
582                subject(),
583                vec![result(c.id, CriterionVerdict::Pass)],
584                vec![]
585            ),
586            Err(EvaluationError::SubjectMismatch)
587        );
588        assert_eq!(
589            EvaluationReport::new(
590                &claim,
591                claim.subject,
592                vec![
593                    result(c.id, CriterionVerdict::Pass),
594                    result(c.id, CriterionVerdict::Pass)
595                ],
596                vec![]
597            ),
598            Err(EvaluationError::CriterionMismatch)
599        );
600    }
601
602    #[test]
603    fn accepting_a_forged_report_revalidates_subject_and_verdict() {
604        let c = criterion(true);
605        let claim = CompletionClaim::new(subject(), vec![c.clone()], vec![], vec![], "done")
606            .expect("valid test claim");
607        let mut evaluation = claim.evaluation();
608        evaluation
609            .begin()
610            .expect("pending claim can begin evaluation");
611        let valid = EvaluationReport::new(
612            &claim,
613            claim.subject,
614            vec![result(c.id, CriterionVerdict::Pass)],
615            vec![],
616        )
617        .expect("valid test report");
618        let forged = EvaluationReport {
619            subject: {
620                let mut subject = claim.subject;
621                subject.goal.revision += 1;
622                subject
623            },
624            verdict: EvaluationVerdict::Fail,
625            ..valid
626        };
627        assert_eq!(
628            evaluation.accept_report(forged),
629            Err(EvaluationError::SubjectMismatch)
630        );
631        assert_eq!(evaluation.state, EvaluationState::Evaluating);
632
633        let valid = EvaluationReport::new(
634            &claim,
635            claim.subject,
636            vec![result(c.id, CriterionVerdict::Pass)],
637            vec![],
638        )
639        .expect("valid test report");
640        let forged = EvaluationReport {
641            verdict: EvaluationVerdict::Fail,
642            ..valid
643        };
644        assert_eq!(
645            evaluation.accept_report(forged),
646            Err(EvaluationError::VerdictMismatch)
647        );
648        assert_eq!(evaluation.state, EvaluationState::Evaluating);
649    }
650}
talos_core/evaluation.rs

talos_core/
evaluation.rs
