a3s_code_core/evaluation/
result.rs

1//! Generic evaluation result contracts and a bounded result store.
2
3use super::identity::{digest_json, validate_digest, ExecutionTargetV1};
4use async_trait::async_trait;
5use serde::{Deserialize, Serialize};
6use std::collections::{HashMap, VecDeque};
7use std::sync::{Arc, RwLock};
8use thiserror::Error;
9
10pub const EVALUATION_RESULT_SCHEMA_V1: &str = "a3s.code.evaluation-result.v1";
11pub const EVALUATION_RECORD_SCHEMA_V1: &str = "a3s.code.evaluation-record.v1";
12const MAX_EVALUATOR_ID_BYTES: usize = 256;
13const MAX_DECISION_BYTES: usize = 128;
14const MAX_SUMMARY_BYTES: usize = 16 * 1024;
15
16/// A host-defined outcome token.  Core validates shape and provenance only;
17/// it deliberately does not enumerate reviewer/business dispositions.
18#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
19#[serde(deny_unknown_fields)]
20pub struct EvaluationResultV1 {
21    pub schema: String,
22    pub evaluator_id: String,
23    pub target: ExecutionTargetV1,
24    pub auxiliary_run_id: String,
25    pub decision: String,
26    #[serde(default, skip_serializing_if = "Option::is_none")]
27    pub confidence_bps: Option<u16>,
28    #[serde(default, skip_serializing_if = "Option::is_none")]
29    pub summary: Option<String>,
30    pub payload: serde_json::Value,
31    pub evidence_digest: String,
32    pub result_digest: String,
33}
34
35impl EvaluationResultV1 {
36    pub fn new(
37        evaluator_id: impl Into<String>,
38        target: ExecutionTargetV1,
39        auxiliary_run_id: impl Into<String>,
40        decision: impl Into<String>,
41        payload: serde_json::Value,
42        evidence_digest: impl Into<String>,
43    ) -> Result<Self, EvaluationStoreError> {
44        let mut result = Self {
45            schema: EVALUATION_RESULT_SCHEMA_V1.to_string(),
46            evaluator_id: evaluator_id.into(),
47            target,
48            auxiliary_run_id: auxiliary_run_id.into(),
49            decision: decision.into(),
50            confidence_bps: None,
51            summary: None,
52            payload,
53            evidence_digest: evidence_digest.into(),
54            result_digest: String::new(),
55        };
56        result.validate_without_digest()?;
57        result.result_digest = result.expected_digest()?;
58        Ok(result)
59    }
60
61    pub fn with_confidence(mut self, confidence_bps: u16) -> Self {
62        // Keep invalid caller input visible to `validate` instead of silently
63        // changing a host decision.  The builder remains infallible for
64        // ergonomics, while `finalize`/store admission fail closed.
65        self.confidence_bps = Some(confidence_bps);
66        self.result_digest = String::new();
67        self
68    }
69
70    pub fn with_summary(mut self, summary: impl Into<String>) -> Self {
71        self.summary = Some(summary.into());
72        self.result_digest = String::new();
73        self
74    }
75
76    /// Recompute the result digest after using one of the builder methods.
77    pub fn finalize(mut self) -> Result<Self, EvaluationStoreError> {
78        self.validate_without_digest()?;
79        self.result_digest = self.expected_digest()?;
80        Ok(self)
81    }
82
83    pub fn validate(&self) -> Result<(), EvaluationStoreError> {
84        self.validate_without_digest()?;
85        validate_digest(&self.result_digest)
86            .map_err(|_| EvaluationStoreError::InvalidField("result_digest"))?;
87        if self.result_digest != self.expected_digest()? {
88            return Err(EvaluationStoreError::DigestMismatch("result_digest"));
89        }
90        Ok(())
91    }
92
93    fn expected_digest(&self) -> Result<String, EvaluationStoreError> {
94        #[derive(Serialize)]
95        struct Identity<'a> {
96            schema: &'a str,
97            evaluator_id: &'a str,
98            target: &'a ExecutionTargetV1,
99            auxiliary_run_id: &'a str,
100            decision: &'a str,
101            confidence_bps: Option<u16>,
102            summary: Option<&'a str>,
103            payload: &'a serde_json::Value,
104            evidence_digest: &'a str,
105        }
106        digest_json(
107            "a3s.code.evaluation-result.identity.v1",
108            &Identity {
109                schema: &self.schema,
110                evaluator_id: &self.evaluator_id,
111                target: &self.target,
112                auxiliary_run_id: &self.auxiliary_run_id,
113                decision: &self.decision,
114                confidence_bps: self.confidence_bps,
115                summary: self.summary.as_deref(),
116                payload: &self.payload,
117                evidence_digest: &self.evidence_digest,
118            },
119        )
120        .map_err(|error| EvaluationStoreError::Serialization(error.to_string()))
121    }
122
123    fn validate_without_digest(&self) -> Result<(), EvaluationStoreError> {
124        if self.schema != EVALUATION_RESULT_SCHEMA_V1 {
125            return Err(EvaluationStoreError::UnsupportedSchema);
126        }
127        self.target
128            .validate()
129            .map_err(|_| EvaluationStoreError::InvalidField("target"))?;
130        validate_text("evaluator_id", &self.evaluator_id, MAX_EVALUATOR_ID_BYTES)?;
131        validate_text(
132            "auxiliary_run_id",
133            &self.auxiliary_run_id,
134            MAX_EVALUATOR_ID_BYTES,
135        )?;
136        validate_text("decision", &self.decision, MAX_DECISION_BYTES)?;
137        if self.confidence_bps.is_some_and(|value| value > 10_000) {
138            return Err(EvaluationStoreError::InvalidField("confidence_bps"));
139        }
140        if self
141            .summary
142            .as_ref()
143            .is_some_and(|value| value.len() > MAX_SUMMARY_BYTES || value.contains('\0'))
144        {
145            return Err(EvaluationStoreError::InvalidField("summary"));
146        }
147        validate_digest(&self.evidence_digest)
148            .map_err(|_| EvaluationStoreError::InvalidField("evidence_digest"))?;
149        let payload_bytes = serde_json::to_vec(&self.payload)
150            .map_err(|error| EvaluationStoreError::Serialization(error.to_string()))?
151            .len();
152        if payload_bytes > MAX_SUMMARY_BYTES * 16 {
153            return Err(EvaluationStoreError::InvalidField("payload"));
154        }
155        Ok(())
156    }
157}
158
159#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
160#[serde(deny_unknown_fields)]
161pub struct EvaluationRecordV1 {
162    pub schema: String,
163    pub result: EvaluationResultV1,
164    pub observed_at_ms: u64,
165    pub record_digest: String,
166}
167
168impl EvaluationRecordV1 {
169    pub fn new(
170        result: EvaluationResultV1,
171        observed_at_ms: u64,
172    ) -> Result<Self, EvaluationStoreError> {
173        result.validate()?;
174        if observed_at_ms == 0 {
175            return Err(EvaluationStoreError::InvalidField("observed_at_ms"));
176        }
177        let mut record = Self {
178            schema: EVALUATION_RECORD_SCHEMA_V1.to_string(),
179            result,
180            observed_at_ms,
181            record_digest: String::new(),
182        };
183        record.record_digest = record.expected_digest()?;
184        Ok(record)
185    }
186
187    pub fn validate(&self) -> Result<(), EvaluationStoreError> {
188        if self.schema != EVALUATION_RECORD_SCHEMA_V1 {
189            return Err(EvaluationStoreError::UnsupportedSchema);
190        }
191        self.result.validate()?;
192        if self.observed_at_ms == 0 {
193            return Err(EvaluationStoreError::InvalidField("observed_at_ms"));
194        }
195        validate_digest(&self.record_digest)
196            .map_err(|_| EvaluationStoreError::InvalidField("record_digest"))?;
197        if self.record_digest != self.expected_digest()? {
198            return Err(EvaluationStoreError::DigestMismatch("record_digest"));
199        }
200        Ok(())
201    }
202
203    fn expected_digest(&self) -> Result<String, EvaluationStoreError> {
204        #[derive(Serialize)]
205        struct Identity<'a> {
206            schema: &'a str,
207            result: &'a EvaluationResultV1,
208            observed_at_ms: u64,
209        }
210        digest_json(
211            "a3s.code.evaluation-record.identity.v1",
212            &Identity {
213                schema: &self.schema,
214                result: &self.result,
215                observed_at_ms: self.observed_at_ms,
216            },
217        )
218        .map_err(|error| EvaluationStoreError::Serialization(error.to_string()))
219    }
220}
221
222#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
223#[serde(deny_unknown_fields)]
224pub struct EvaluationWriteOutcomeV1 {
225    pub written: bool,
226    pub replayed: bool,
227}
228
229#[derive(Debug, Clone, PartialEq, Eq, Error)]
230pub enum EvaluationStoreError {
231    #[error("evaluation result schema is unsupported")]
232    UnsupportedSchema,
233    #[error("evaluation result field `{0}` is invalid")]
234    InvalidField(&'static str),
235    #[error("evaluation result digest for `{0}` does not match")]
236    DigestMismatch(&'static str),
237    #[error("evaluation result conflicts with an existing record")]
238    Conflict,
239    #[error("evaluation result serialization failed: {0}")]
240    Serialization(String),
241    #[error("evaluation result store I/O failed: {0}")]
242    Storage(String),
243    #[error("evaluation result store is corrupt: {0}")]
244    Corrupt(String),
245    #[error("evaluation result store exceeds its configured size limit")]
246    SizeLimit,
247}
248
249#[async_trait]
250pub trait EvaluationResultSink: Send + Sync {
251    async fn write(
252        &self,
253        record: EvaluationRecordV1,
254    ) -> Result<EvaluationWriteOutcomeV1, EvaluationStoreError>;
255    async fn get(&self, record_digest: &str) -> Option<EvaluationRecordV1>;
256    async fn list_for_target(&self, target: &ExecutionTargetV1) -> Vec<EvaluationRecordV1>;
257
258    /// Error-reporting read variant for hosts that need to distinguish a
259    /// missing record from a corrupt or unavailable backing store.  Legacy
260    /// implementations may use the compatibility methods above; the default
261    /// keeps those implementations source-compatible.
262    async fn get_checked(
263        &self,
264        record_digest: &str,
265    ) -> Result<Option<EvaluationRecordV1>, EvaluationStoreError> {
266        Ok(self.get(record_digest).await)
267    }
268
269    async fn list_for_target_checked(
270        &self,
271        target: &ExecutionTargetV1,
272    ) -> Result<Vec<EvaluationRecordV1>, EvaluationStoreError> {
273        Ok(self.list_for_target(target).await)
274    }
275}
276
277#[derive(Debug, Default)]
278struct ResultState {
279    by_digest: HashMap<String, EvaluationRecordV1>,
280    by_identity: HashMap<EvaluationIdentityKey, String>,
281    order: VecDeque<String>,
282}
283
284#[derive(Debug, Clone, PartialEq, Eq, Hash)]
285struct EvaluationIdentityKey {
286    target: ExecutionTargetV1,
287    evaluator_id: String,
288    auxiliary_run_id: String,
289}
290
291/// In-memory CAS result sink.  A durable host can implement the same trait
292/// with an append-only object store and an external retention policy.
293#[derive(Debug, Clone)]
294pub struct InMemoryEvaluationResultStore {
295    state: Arc<RwLock<ResultState>>,
296    max_records: Option<usize>,
297}
298
299impl InMemoryEvaluationResultStore {
300    pub fn new() -> Self {
301        Self::with_max_records(None)
302    }
303
304    pub fn with_max_records(max_records: Option<usize>) -> Self {
305        Self {
306            state: Arc::new(RwLock::new(ResultState::default())),
307            max_records,
308        }
309    }
310}
311
312impl Default for InMemoryEvaluationResultStore {
313    fn default() -> Self {
314        Self::new()
315    }
316}
317
318#[async_trait]
319impl EvaluationResultSink for InMemoryEvaluationResultStore {
320    async fn write(
321        &self,
322        record: EvaluationRecordV1,
323    ) -> Result<EvaluationWriteOutcomeV1, EvaluationStoreError> {
324        record.validate()?;
325        let digest = record.record_digest.clone();
326        let mut state = self
327            .state
328            .write()
329            .map_err(|_| EvaluationStoreError::Conflict)?;
330        if let Some(existing) = state.by_digest.get(&digest) {
331            if existing == &record {
332                return Ok(EvaluationWriteOutcomeV1 {
333                    written: false,
334                    replayed: true,
335                });
336            }
337            return Err(EvaluationStoreError::Conflict);
338        }
339        let identity = EvaluationIdentityKey {
340            target: record.result.target.clone(),
341            evaluator_id: record.result.evaluator_id.clone(),
342            auxiliary_run_id: record.result.auxiliary_run_id.clone(),
343        };
344        if state.by_identity.contains_key(&identity) {
345            // A single evaluator/auxiliary pair is immutable.  This catches a
346            // conflicting result even when the caller recomputed a different
347            // valid content digest, while exact replay above remains
348            // idempotent.
349            return Err(EvaluationStoreError::Conflict);
350        }
351        state.order.push_back(digest.clone());
352        state.by_identity.insert(identity, digest.clone());
353        state.by_digest.insert(digest, record);
354        if let Some(limit) = self.max_records {
355            while state.order.len() > limit {
356                if let Some(oldest) = state.order.pop_front() {
357                    if let Some(removed) = state.by_digest.remove(&oldest) {
358                        let identity = EvaluationIdentityKey {
359                            target: removed.result.target,
360                            evaluator_id: removed.result.evaluator_id,
361                            auxiliary_run_id: removed.result.auxiliary_run_id,
362                        };
363                        if state
364                            .by_identity
365                            .get(&identity)
366                            .is_some_and(|digest| digest == &oldest)
367                        {
368                            state.by_identity.remove(&identity);
369                        }
370                    }
371                }
372            }
373        }
374        Ok(EvaluationWriteOutcomeV1 {
375            written: true,
376            replayed: false,
377        })
378    }
379
380    async fn get(&self, record_digest: &str) -> Option<EvaluationRecordV1> {
381        self.state
382            .read()
383            .ok()?
384            .by_digest
385            .get(record_digest)
386            .cloned()
387    }
388
389    async fn list_for_target(&self, target: &ExecutionTargetV1) -> Vec<EvaluationRecordV1> {
390        let Ok(state) = self.state.read() else {
391            return Vec::new();
392        };
393        state
394            .order
395            .iter()
396            .filter_map(|digest| state.by_digest.get(digest))
397            .filter(|record| record.result.target == *target)
398            .cloned()
399            .collect()
400    }
401}
402
403fn validate_text(
404    field: &'static str,
405    value: &str,
406    max_bytes: usize,
407) -> Result<(), EvaluationStoreError> {
408    if value.is_empty()
409        || value.len() > max_bytes
410        || value.contains('\0')
411        || value.contains(['\r', '\n'])
412    {
413        return Err(EvaluationStoreError::InvalidField(field));
414    }
415    Ok(())
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421
422    fn target() -> ExecutionTargetV1 {
423        ExecutionTargetV1::new("session-1", "run-1")
424    }
425
426    fn result() -> EvaluationResultV1 {
427        EvaluationResultV1::new(
428            "fixture-evaluator",
429            target(),
430            "aux-1",
431            "inconclusive",
432            serde_json::json!({"issues": []}),
433            super::super::identity::digest_bytes("evidence", b"fixture"),
434        )
435        .unwrap()
436    }
437
438    #[test]
439    fn evaluator_identity_fields_reject_trailing_line_endings() {
440        for evaluator_id in ["reviewer\n", "reviewer\r", "reviewer\r\n"] {
441            assert!(matches!(
442                EvaluationResultV1::new(
443                    evaluator_id,
444                    target(),
445                    "aux-1",
446                    "observed",
447                    serde_json::json!({"finding_count": 1}),
448                    "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
449                ),
450                Err(EvaluationStoreError::InvalidField("evaluator_id"))
451            ));
452        }
453    }
454
455    #[test]
456    fn result_builder_requires_finalization_after_mutation() {
457        let result = result().with_summary("bounded summary");
458        assert!(result.validate().is_err());
459        let result = result.finalize().unwrap();
460        assert!(result.validate().is_ok());
461    }
462
463    #[tokio::test]
464    async fn result_store_is_idempotent_and_lists_by_target() {
465        let store = InMemoryEvaluationResultStore::new();
466        let record = EvaluationRecordV1::new(result(), 1).unwrap();
467        let first = store.write(record.clone()).await.unwrap();
468        assert!(first.written);
469        let replay = store.write(record.clone()).await.unwrap();
470        assert!(replay.replayed);
471        assert_eq!(store.list_for_target(&target()).await, vec![record.clone()]);
472        assert_eq!(store.get(&record.record_digest).await, Some(record));
473    }
474
475    #[test]
476    fn host_decision_is_open_text_not_a_core_enum() {
477        let mut result = result();
478        result.decision = "product-specific-token".to_string();
479        result = result.finalize().unwrap();
480        assert!(result.validate().is_ok());
481    }
482
483    #[test]
484    fn confidence_overflow_is_rejected_instead_of_clamped() {
485        let result = result().with_confidence(10_001).finalize();
486        assert!(matches!(
487            result,
488            Err(EvaluationStoreError::InvalidField("confidence_bps"))
489        ));
490    }
491
492    #[tokio::test]
493    async fn result_store_rejects_conflicting_identity() {
494        let store = InMemoryEvaluationResultStore::new();
495        let first = EvaluationRecordV1::new(result(), 1).unwrap();
496        store.write(first).await.unwrap();
497        let second_result = EvaluationResultV1::new(
498            "fixture-evaluator",
499            target(),
500            "aux-1",
501            "inconclusive",
502            serde_json::json!({"issues": ["different"]}),
503            super::super::identity::digest_bytes("evidence", b"fixture"),
504        )
505        .unwrap();
506        let second = EvaluationRecordV1::new(second_result, 2).unwrap();
507        assert!(matches!(
508            store.write(second).await,
509            Err(EvaluationStoreError::Conflict)
510        ));
511    }
512}
a3s_code_core/evaluation/result.rs

a3s_code_core/evaluation/
result.rs
