talos_agent/
evaluator.rs

1//! Independent, read-only evaluation of completion claims.
2//!
3//! The evaluator deliberately receives a bounded claim snapshot rather than the executor's
4//! conversation.  Its output is revalidated by the P2 state machine before it can become a
5//! verdict; provider failures and malformed output never become PASS.
6
7use std::collections::{HashMap, HashSet};
8use std::sync::Arc;
9use std::time::Duration;
10
11use async_trait::async_trait;
12use schemars::JsonSchema;
13use serde::{Deserialize, Serialize};
14use talos_core::evaluation::{
15    CompletionClaim, Evaluation, EvaluationError, EvaluationReport, EvidenceRef,
16};
17use talos_core::message::{AgentEvent, Message};
18use talos_core::provider::LanguageModel;
19use talos_core::tool::ToolNature;
20use thiserror::Error;
21use tokio_util::sync::CancellationToken;
22
23/// Validation status recorded in a bounded evidence snapshot.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
25#[serde(rename_all = "snake_case")]
26pub enum ValidationEvidenceStatus {
27    /// The producer completed successfully.
28    Passed,
29    /// The producer completed and found a failure.
30    Failed,
31    /// The producer could not run or did not produce a result.
32    Unavailable,
33}
34
35/// Provenance-preserving validation evidence made available to an evaluator.
36#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
37pub struct ValidationEvidence {
38    /// Stable evidence identity from the validation producer.
39    pub evidence: EvidenceRef,
40    /// Producer outcome; this is evidence, not a Goal verdict.
41    pub status: ValidationEvidenceStatus,
42    /// Digest of the producer record or artifact set.
43    pub record_digest: String,
44}
45
46impl ValidationEvidence {
47    /// Construct evidence, rejecting an absent integrity binding.
48    pub fn new(
49        evidence: EvidenceRef,
50        status: ValidationEvidenceStatus,
51        record_digest: impl Into<String>,
52    ) -> Result<Self, EvaluatorError> {
53        let record_digest = record_digest.into();
54        if record_digest.trim().is_empty() || evidence.kind.trim().is_empty() {
55            return Err(EvaluatorError::InvalidEvidence);
56        }
57        Ok(Self {
58            evidence,
59            status,
60            record_digest,
61        })
62    }
63}
64
65/// The bounded, fresh context sent to an evaluator.
66#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
67pub struct EvaluatorRequest {
68    /// Exact claim and subject identity under inspection.
69    pub claim: CompletionClaim,
70    /// Validation records available as references only.
71    pub validation_evidence: Vec<ValidationEvidence>,
72    /// Explicitly states that evaluator tools are read-only.
73    pub read_only: bool,
74}
75
76impl EvaluatorRequest {
77    fn for_claim(claim: &CompletionClaim, evidence: Vec<ValidationEvidence>) -> Self {
78        Self {
79            claim: claim.clone(),
80            validation_evidence: evidence,
81            read_only: true,
82        }
83    }
84}
85
86/// A model assessor used by the independent evaluator. It cannot execute tools through this API.
87#[async_trait]
88pub trait EvaluatorAssessor: Send + Sync {
89    /// Return one JSON [`EvaluationReport`] within the supplied deadline.
90    async fn assess(&self, request: EvaluatorRequest, deadline: Duration)
91    -> Result<String, String>;
92
93    /// Stable evaluator identity for audit records.
94    fn identity(&self) -> &str {
95        "configured-evaluator"
96    }
97}
98
99/// Provider-backed assessor that sends one tool-free request with a fresh context.
100pub struct ProviderEvaluatorAssessor {
101    provider: Arc<dyn LanguageModel>,
102    identity: String,
103}
104
105impl ProviderEvaluatorAssessor {
106    /// Create an assessor from an independent provider/runtime instance.
107    #[must_use]
108    pub fn new(provider: Arc<dyn LanguageModel>) -> Self {
109        Self {
110            provider,
111            identity: "configured-evaluator".to_owned(),
112        }
113    }
114
115    /// Set the non-secret identity exposed in audit records.
116    #[must_use]
117    pub fn with_identity(mut self, identity: impl Into<String>) -> Self {
118        self.identity = identity.into();
119        self
120    }
121}
122
123#[async_trait]
124impl EvaluatorAssessor for ProviderEvaluatorAssessor {
125    async fn assess(
126        &self,
127        request: EvaluatorRequest,
128        deadline: Duration,
129    ) -> Result<String, String> {
130        let payload = serde_json::to_string(&request).map_err(|error| error.to_string())?;
131        let messages = vec![
132            Message::System {
133                content: "You are an independent evaluator. Return only one JSON EvaluationReport. Use the exact claim subject and criterion IDs. Do not use tools, infer missing evidence, or certify from executor reasoning.".to_owned(),
134                cache_markers: Vec::new(),
135            },
136            Message::User {
137                content: format!("Evaluate this bounded claim snapshot; return JSON only:\n{payload}"),
138            },
139        ];
140        let mut events = self
141            .provider
142            .stream(&messages)
143            .await
144            .map_err(|error| error.to_string())?;
145        let mut output = String::new();
146        let deadline = tokio::time::sleep(deadline);
147        tokio::pin!(deadline);
148        loop {
149            tokio::select! {
150                _ = &mut deadline => return Err("evaluator deadline exceeded".to_owned()),
151                event = events.recv() => match event {
152                    Some(AgentEvent::TextDelta { delta }) => output.push_str(&delta),
153                    Some(AgentEvent::ToolCall { .. }) => return Err("evaluator tool use is forbidden".to_owned()),
154                    Some(AgentEvent::Error { message }) => return Err(message),
155                    Some(AgentEvent::TurnEnd { .. }) | None => break,
156                    Some(_) => {}
157                },
158            }
159        }
160        if output.trim().is_empty() {
161            return Err("evaluator returned no report".to_owned());
162        }
163        Ok(output)
164    }
165
166    fn identity(&self) -> &str {
167        &self.identity
168    }
169}
170
171/// Read-only admission policy for evaluator tools.
172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
173pub struct EvaluatorAdmission {
174    read_only: bool,
175}
176
177impl Default for EvaluatorAdmission {
178    fn default() -> Self {
179        Self { read_only: true }
180    }
181}
182
183impl EvaluatorAdmission {
184    /// Returns whether a tool nature can be admitted by the default evaluator policy.
185    #[must_use]
186    pub const fn allows(self, nature: ToolNature) -> bool {
187        self.read_only && matches!(nature, ToolNature::Read | ToolNature::Internal)
188    }
189}
190
191/// Explicit non-PASS outcome when evaluation cannot safely produce a report.
192#[derive(Debug, Clone, PartialEq, Eq)]
193pub struct EvaluatorFailure {
194    /// Stable evaluator identity.
195    pub evaluator: String,
196    /// Bounded reason suitable for audit/status output.
197    pub reason: String,
198}
199
200/// Result of one independent evaluation attempt.
201#[derive(Debug, Clone, PartialEq, Eq)]
202pub enum EvaluatorOutcome {
203    /// A report was accepted by the P2 state machine.
204    Report { evaluation: Box<Evaluation> },
205    /// Evaluation ended safely without a PASS report.
206    Failure(EvaluatorFailure),
207}
208
209/// Errors raised before a safe evaluator outcome can be constructed.
210#[derive(Debug, Error)]
211pub enum EvaluatorError {
212    /// Evidence omitted its producer identity or integrity digest.
213    #[error("validation evidence is missing provenance or integrity binding")]
214    InvalidEvidence,
215    /// The claim subject or report violated the P2 contract.
216    #[error("evaluation contract rejected evaluator report: {0}")]
217    Contract(#[from] EvaluationError),
218}
219
220/// Independent evaluator coordinator with bounded, fail-closed execution.
221pub struct IndependentEvaluator {
222    assessor: Arc<dyn EvaluatorAssessor>,
223    deadline: Duration,
224    admission: EvaluatorAdmission,
225}
226
227impl IndependentEvaluator {
228    /// Create an evaluator. The deadline is clamped to a bounded 1ms-30s range.
229    #[must_use]
230    pub fn new(assessor: Arc<dyn EvaluatorAssessor>, deadline: Duration) -> Self {
231        Self {
232            assessor,
233            deadline: deadline.clamp(Duration::from_millis(1), Duration::from_secs(30)),
234            admission: EvaluatorAdmission::default(),
235        }
236    }
237
238    /// Returns the read-only admission policy used by this evaluator.
239    #[must_use]
240    pub const fn admission(&self) -> EvaluatorAdmission {
241        self.admission
242    }
243
244    /// Evaluate one exact claim, returning an explicit non-PASS failure on unsafe conditions.
245    pub async fn evaluate(
246        &self,
247        claim: &CompletionClaim,
248        validation_evidence: Vec<ValidationEvidence>,
249    ) -> EvaluatorOutcome {
250        self.evaluate_with_cancellation(claim, validation_evidence, CancellationToken::new())
251            .await
252    }
253
254    /// Evaluate with caller-owned cancellation. Cancellation is always an explicit non-PASS.
255    pub async fn evaluate_with_cancellation(
256        &self,
257        claim: &CompletionClaim,
258        validation_evidence: Vec<ValidationEvidence>,
259        cancellation: CancellationToken,
260    ) -> EvaluatorOutcome {
261        let request = EvaluatorRequest::for_claim(claim, validation_evidence);
262        let evaluator = self.assessor.identity().to_owned();
263        let mut supplied_records = HashMap::new();
264        for evidence in &request.validation_evidence {
265            if evidence.record_digest.trim().is_empty() || evidence.evidence.kind.trim().is_empty()
266            {
267                return EvaluatorOutcome::Failure(EvaluatorFailure {
268                    evaluator,
269                    reason: "validation evidence lacks provenance or integrity binding".to_owned(),
270                });
271            }
272            let record = (evidence.status, evidence.record_digest.as_str());
273            if let Some(previous) = supplied_records.insert(evidence.evidence.clone(), record)
274                && previous != record
275            {
276                return EvaluatorOutcome::Failure(EvaluatorFailure {
277                    evaluator,
278                    reason: "validation evidence has conflicting records".to_owned(),
279                });
280            }
281        }
282        let valid_evidence: HashSet<_> = request
283            .validation_evidence
284            .iter()
285            .filter(|evidence| {
286                !evidence.record_digest.trim().is_empty()
287                    && !evidence.evidence.kind.trim().is_empty()
288                    && evidence.status == ValidationEvidenceStatus::Passed
289            })
290            .map(|evidence| evidence.evidence.clone())
291            .collect();
292        let assessment = self.assessor.assess(request, self.deadline);
293        tokio::pin!(assessment);
294        let raw = tokio::select! {
295            _ = cancellation.cancelled() => {
296                return EvaluatorOutcome::Failure(EvaluatorFailure {
297                    evaluator,
298                    reason: "evaluator cancelled".to_owned(),
299                });
300            }
301            result = tokio::time::timeout(self.deadline, &mut assessment) => match result {
302                Ok(Ok(raw)) => raw,
303                Ok(Err(reason)) => {
304                    return EvaluatorOutcome::Failure(EvaluatorFailure { evaluator, reason });
305                }
306                Err(_) => {
307                    return EvaluatorOutcome::Failure(EvaluatorFailure {
308                        evaluator,
309                        reason: "evaluator deadline exceeded".to_owned(),
310                    });
311                }
312            }
313        };
314        let report: EvaluationReport = match serde_json::from_str(&raw) {
315            Ok(report) => report,
316            Err(error) => {
317                return EvaluatorOutcome::Failure(EvaluatorFailure {
318                    evaluator,
319                    reason: format!("malformed evaluator report: {error}"),
320                });
321            }
322        };
323        if let Err(reason) = validate_report_evidence(claim, &report, &valid_evidence) {
324            return EvaluatorOutcome::Failure(EvaluatorFailure { evaluator, reason });
325        }
326        let mut evaluation = claim.evaluation();
327        if let Err(error) = evaluation.begin() {
328            return EvaluatorOutcome::Failure(EvaluatorFailure {
329                evaluator,
330                reason: error.to_string(),
331            });
332        }
333        if let Err(error) = evaluation.accept_report(report) {
334            return EvaluatorOutcome::Failure(EvaluatorFailure {
335                evaluator,
336                reason: error.to_string(),
337            });
338        }
339        EvaluatorOutcome::Report {
340            evaluation: Box::new(evaluation),
341        }
342    }
343}
344
345fn validate_report_evidence(
346    claim: &CompletionClaim,
347    report: &EvaluationReport,
348    valid_evidence: &HashSet<EvidenceRef>,
349) -> Result<(), String> {
350    for result in &report.results {
351        let Some(criterion) = claim
352            .criteria
353            .iter()
354            .find(|criterion| criterion.id == result.criterion_id)
355        else {
356            return Err("evaluator report references an unknown criterion".to_owned());
357        };
358        if criterion.required
359            && result.verdict == talos_core::evaluation::CriterionVerdict::Pass
360            && (result.evidence.is_empty()
361                || result
362                    .evidence
363                    .iter()
364                    .any(|evidence| !valid_evidence.contains(evidence)))
365        {
366            return Err("required PASS criterion lacks valid supplied evidence".to_owned());
367        }
368    }
369    Ok(())
370}
371
372#[cfg(test)]
373mod tests {
374    use super::*;
375    use std::sync::Arc;
376    use talos_core::evaluation::{
377        AcceptanceCriterion, CriterionEvaluation, CriterionKind, CriterionVerdict,
378        EvaluationSubject, EvaluationVerdict, WorkspaceRevision,
379    };
380    use talos_core::work::{WorkIdentity, WorkKind};
381    use uuid::Uuid;
382
383    fn claim() -> CompletionClaim {
384        CompletionClaim::new(
385            EvaluationSubject {
386                mission: WorkIdentity {
387                    id: Uuid::new_v4(),
388                    kind: WorkKind::Mission,
389                    revision: 1,
390                },
391                goal: WorkIdentity {
392                    id: Uuid::new_v4(),
393                    kind: WorkKind::Goal,
394                    revision: 1,
395                },
396                workspace: WorkspaceRevision {
397                    id: Uuid::new_v4(),
398                    revision: 1,
399                },
400            },
401            vec![AcceptanceCriterion {
402                id: Uuid::new_v4(),
403                kind: CriterionKind::Technical,
404                statement: "works".into(),
405                required: true,
406            }],
407            Vec::new(),
408            Vec::new(),
409            "executor hint",
410        )
411        .expect("claim")
412    }
413
414    struct Assessor(String);
415
416    #[async_trait]
417    impl EvaluatorAssessor for Assessor {
418        async fn assess(
419            &self,
420            _request: EvaluatorRequest,
421            _deadline: Duration,
422        ) -> Result<String, String> {
423            Ok(self.0.clone())
424        }
425    }
426
427    struct HangingAssessor;
428
429    #[async_trait]
430    impl EvaluatorAssessor for HangingAssessor {
431        async fn assess(
432            &self,
433            _request: EvaluatorRequest,
434            _deadline: Duration,
435        ) -> Result<String, String> {
436            tokio::time::sleep(Duration::from_secs(60)).await;
437            Ok(String::new())
438        }
439    }
440
441    #[tokio::test]
442    async fn malformed_output_is_explicit_failure() {
443        let evaluator = IndependentEvaluator::new(
444            Arc::new(Assessor("not-json".into())),
445            Duration::from_secs(1),
446        );
447        assert!(matches!(
448            evaluator.evaluate(&claim(), Vec::new()).await,
449            EvaluatorOutcome::Failure(_)
450        ));
451    }
452
453    #[tokio::test]
454    async fn assessor_that_ignores_deadline_is_bounded() {
455        let evaluator =
456            IndependentEvaluator::new(Arc::new(HangingAssessor), Duration::from_millis(5));
457        let outcome = evaluator.evaluate(&claim(), Vec::new()).await;
458        assert!(
459            matches!(outcome, EvaluatorOutcome::Failure(EvaluatorFailure { reason, .. }) if reason.contains("deadline"))
460        );
461    }
462
463    #[tokio::test]
464    async fn cancellation_is_explicit_failure() {
465        let evaluator =
466            IndependentEvaluator::new(Arc::new(HangingAssessor), Duration::from_secs(1));
467        let cancellation = CancellationToken::new();
468        cancellation.cancel();
469        let outcome = evaluator
470            .evaluate_with_cancellation(&claim(), Vec::new(), cancellation)
471            .await;
472        assert!(
473            matches!(outcome, EvaluatorOutcome::Failure(EvaluatorFailure { reason, .. }) if reason.contains("cancelled"))
474        );
475    }
476
477    #[tokio::test]
478    async fn valid_report_is_revalidated_and_accepted() {
479        let claim = claim();
480        let evidence = EvidenceRef {
481            id: Uuid::new_v4(),
482            kind: "validation".into(),
483        };
484        let result = CriterionEvaluation {
485            criterion_id: claim.criteria[0].id,
486            verdict: CriterionVerdict::Pass,
487            evidence: vec![evidence.clone()],
488            finding_ids: Vec::new(),
489        };
490        let report =
491            EvaluationReport::new(&claim, claim.subject, vec![result], Vec::new()).expect("report");
492        let raw = serde_json::to_string(&report).expect("json");
493        let evaluator = IndependentEvaluator::new(Arc::new(Assessor(raw)), Duration::from_secs(1));
494        let supplied =
495            ValidationEvidence::new(evidence, ValidationEvidenceStatus::Passed, "digest")
496                .expect("evidence");
497        let outcome = evaluator.evaluate(&claim, vec![supplied]).await;
498        match outcome {
499            EvaluatorOutcome::Report { evaluation } => assert_eq!(
500                evaluation.state,
501                talos_core::evaluation::EvaluationState::Verdict(EvaluationVerdict::Pass)
502            ),
503            EvaluatorOutcome::Failure(error) => panic!("unexpected failure: {}", error.reason),
504        }
505    }
506
507    #[tokio::test]
508    async fn pass_without_supplied_evidence_is_rejected() {
509        let claim = claim();
510        let result = CriterionEvaluation {
511            criterion_id: claim.criteria[0].id,
512            verdict: CriterionVerdict::Pass,
513            evidence: vec![EvidenceRef {
514                id: Uuid::new_v4(),
515                kind: "validation".into(),
516            }],
517            finding_ids: Vec::new(),
518        };
519        let report =
520            EvaluationReport::new(&claim, claim.subject, vec![result], Vec::new()).expect("report");
521        let evaluator = IndependentEvaluator::new(
522            Arc::new(Assessor(serde_json::to_string(&report).expect("json"))),
523            Duration::from_secs(1),
524        );
525        assert!(
526            matches!(evaluator.evaluate(&claim, Vec::new()).await, EvaluatorOutcome::Failure(EvaluatorFailure { reason, .. }) if reason.contains("supplied evidence"))
527        );
528    }
529
530    #[tokio::test]
531    async fn claimed_evidence_hint_cannot_authorize_required_pass() {
532        let mut claim = claim();
533        let evidence = EvidenceRef {
534            id: Uuid::new_v4(),
535            kind: "validation".into(),
536        };
537        claim.claimed_evidence.push(evidence.clone());
538        let result = CriterionEvaluation {
539            criterion_id: claim.criteria[0].id,
540            verdict: CriterionVerdict::Pass,
541            evidence: vec![evidence],
542            finding_ids: Vec::new(),
543        };
544        let report =
545            EvaluationReport::new(&claim, claim.subject, vec![result], Vec::new()).expect("report");
546        let evaluator = IndependentEvaluator::new(
547            Arc::new(Assessor(serde_json::to_string(&report).expect("json"))),
548            Duration::from_secs(1),
549        );
550        assert!(matches!(
551            evaluator.evaluate(&claim, Vec::new()).await,
552            EvaluatorOutcome::Failure(EvaluatorFailure { reason, .. })
553                if reason.contains("supplied evidence")
554        ));
555    }
556
557    #[tokio::test]
558    async fn evidence_kind_mismatch_cannot_authorize_required_pass() {
559        let claim = claim();
560        let supplied = EvidenceRef {
561            id: Uuid::new_v4(),
562            kind: "validation".into(),
563        };
564        let reported = EvidenceRef {
565            id: supplied.id,
566            kind: "executor-claim".into(),
567        };
568        let result = CriterionEvaluation {
569            criterion_id: claim.criteria[0].id,
570            verdict: CriterionVerdict::Pass,
571            evidence: vec![reported],
572            finding_ids: Vec::new(),
573        };
574        let report =
575            EvaluationReport::new(&claim, claim.subject, vec![result], Vec::new()).expect("report");
576        let evaluator = IndependentEvaluator::new(
577            Arc::new(Assessor(serde_json::to_string(&report).expect("json"))),
578            Duration::from_secs(1),
579        );
580        let supplied =
581            ValidationEvidence::new(supplied, ValidationEvidenceStatus::Passed, "digest")
582                .expect("evidence");
583        assert!(matches!(
584            evaluator.evaluate(&claim, vec![supplied]).await,
585            EvaluatorOutcome::Failure(EvaluatorFailure { reason, .. })
586                if reason.contains("supplied evidence")
587        ));
588    }
589
590    #[tokio::test]
591    async fn failed_evidence_cannot_authorize_required_pass() {
592        let claim = claim();
593        let evidence = EvidenceRef {
594            id: Uuid::new_v4(),
595            kind: "validation".into(),
596        };
597        let result = CriterionEvaluation {
598            criterion_id: claim.criteria[0].id,
599            verdict: CriterionVerdict::Pass,
600            evidence: vec![evidence.clone()],
601            finding_ids: Vec::new(),
602        };
603        let report =
604            EvaluationReport::new(&claim, claim.subject, vec![result], Vec::new()).expect("report");
605        let evaluator = IndependentEvaluator::new(
606            Arc::new(Assessor(serde_json::to_string(&report).expect("json"))),
607            Duration::from_secs(1),
608        );
609        let supplied =
610            ValidationEvidence::new(evidence, ValidationEvidenceStatus::Failed, "digest")
611                .expect("evidence");
612        assert!(matches!(
613            evaluator.evaluate(&claim, vec![supplied]).await,
614            EvaluatorOutcome::Failure(EvaluatorFailure { reason, .. })
615                if reason.contains("supplied evidence")
616        ));
617    }
618
619    #[test]
620    fn admission_rejects_side_effecting_tools() {
621        let policy = EvaluatorAdmission::default();
622        assert!(policy.allows(ToolNature::Read));
623        assert!(policy.allows(ToolNature::Internal));
624        assert!(!policy.allows(ToolNature::Write));
625        assert!(!policy.allows(ToolNature::Execute));
626        assert!(!policy.allows(ToolNature::Network));
627    }
628
629    #[test]
630    fn evidence_requires_integrity_binding() {
631        let reference = EvidenceRef {
632            id: Uuid::new_v4(),
633            kind: "validation".into(),
634        };
635        assert!(
636            ValidationEvidence::new(reference.clone(), ValidationEvidenceStatus::Passed, "")
637                .is_err()
638        );
639        assert!(
640            ValidationEvidence::new(reference, ValidationEvidenceStatus::Passed, "digest").is_ok()
641        );
642    }
643}
talos_agent/evaluator.rs

talos_agent/
evaluator.rs
