car_ir/
outcome.rs

1//! Agent execution outcome semantics.
2//!
3//! Provides typed outcomes for agent execution loops, replacing ad-hoc
4//! success/failure heuristics with structured completion semantics.
5
6use chrono::{DateTime, Utc};
7use serde::{Deserialize, Serialize};
8use serde_json::Value;
9
10/// Default timestamp for outcomes deserialized from caller-built JSON that
11/// omits `timestamp` (e.g. the canonical harness, which sends only
12/// `{status, summary, evidence, metrics, tools_called}`). `DateTime` has no
13/// `Default`, so we provide one explicitly.
14fn default_outcome_timestamp() -> DateTime<Utc> {
15    Utc::now()
16}
17
18/// The outcome of an agent execution loop.
19///
20/// Deserialization is tolerant of the caller-built shape: `timestamp` is
21/// optional (defaults to now), `summary`/`evidence`/`metrics` are optional
22/// (default to empty), and unknown fields (e.g. the harness's extra
23/// `tools_called`) are ignored (no `deny_unknown_fields`). Serialization is
24/// unchanged and always emits all fields.
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct AgentOutcome {
27    /// The outcome type.
28    pub status: OutcomeStatus,
29    /// Human-readable summary of what happened.
30    #[serde(default)]
31    pub summary: String,
32    /// Evidence supporting the outcome classification.
33    #[serde(default)]
34    pub evidence: Vec<Evidence>,
35    /// Metrics from the execution.
36    #[serde(default)]
37    pub metrics: OutcomeMetrics,
38    /// When the outcome was determined.
39    #[serde(default = "default_outcome_timestamp")]
40    pub timestamp: DateTime<Utc>,
41}
42
43/// Outcome classification for an agent execution.
44#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
45#[serde(rename_all = "snake_case")]
46pub enum OutcomeStatus {
47    /// Task completed successfully with all goals met.
48    Success,
49    /// Task completed but only some goals were met.
50    PartialSuccess,
51    /// Agent explicitly determined it cannot complete the task.
52    GiveUp,
53    /// Execution exceeded time or step limits.
54    Timeout,
55    /// Execution failed due to errors.
56    Failure,
57    /// Agent explicitly signaled it is done (neutral -- may or may not have succeeded).
58    Done,
59    /// Repeated actions made no progress.
60    Stalled,
61    /// The model did not call a required terminal tool after a reminder.
62    NoTerminal,
63}
64
65impl OutcomeStatus {
66    /// Whether this outcome represents any form of completion (not failure/timeout).
67    pub fn is_completed(&self) -> bool {
68        matches!(self, Self::Success | Self::PartialSuccess | Self::Done)
69    }
70
71    /// Whether this outcome represents a terminal state (no more work possible).
72    pub fn is_terminal(&self) -> bool {
73        true // All outcome statuses are terminal
74    }
75}
76
77/// Evidence supporting an outcome classification.
78#[derive(Debug, Clone, Serialize, Deserialize)]
79pub struct Evidence {
80    /// What kind of evidence this is.
81    pub kind: EvidenceKind,
82    /// Human-readable description.
83    pub description: String,
84    /// Optional structured data.
85    pub data: Option<Value>,
86}
87
88/// Types of evidence that can support an outcome.
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
90#[serde(rename_all = "snake_case")]
91pub enum EvidenceKind {
92    /// Agent's own assessment of task completion.
93    SelfAssessment,
94    /// Tool call results that demonstrate completion.
95    ToolResult,
96    /// State changes that demonstrate completion.
97    StateChange,
98    /// External verification (e.g., test passed).
99    ExternalVerification,
100    /// The reason execution stopped (timeout, max steps, etc.).
101    StopReason,
102    /// Product-level evaluator result.
103    Evaluator,
104}
105
106/// Execution metrics associated with an outcome.
107///
108/// All fields are `#[serde(default)]` so a partial metrics object (e.g. the
109/// harness's `{turns, tool_calls, actions_succeeded, actions_failed}`, which
110/// omits `duration_ms` and `retries`) deserializes cleanly.
111#[derive(Debug, Clone, Default, Serialize, Deserialize)]
112pub struct OutcomeMetrics {
113    /// Total turns/steps executed.
114    #[serde(default)]
115    pub turns: u32,
116    /// Total tool calls made.
117    #[serde(default)]
118    pub tool_calls: u32,
119    /// Wall-clock duration in milliseconds.
120    #[serde(default)]
121    pub duration_ms: f64,
122    /// Number of retries/replans attempted.
123    #[serde(default)]
124    pub retries: u32,
125    /// Number of actions that succeeded.
126    #[serde(default)]
127    pub actions_succeeded: u32,
128    /// Number of actions that failed.
129    #[serde(default)]
130    pub actions_failed: u32,
131}
132
133impl AgentOutcome {
134    /// Create a successful outcome.
135    pub fn success(summary: &str) -> Self {
136        Self {
137            status: OutcomeStatus::Success,
138            summary: summary.to_string(),
139            evidence: Vec::new(),
140            metrics: OutcomeMetrics::default(),
141            timestamp: Utc::now(),
142        }
143    }
144
145    /// Create a failure outcome.
146    pub fn failure(summary: &str) -> Self {
147        Self {
148            status: OutcomeStatus::Failure,
149            summary: summary.to_string(),
150            evidence: Vec::new(),
151            metrics: OutcomeMetrics::default(),
152            timestamp: Utc::now(),
153        }
154    }
155
156    /// Create a timeout outcome.
157    pub fn timeout(summary: &str, turns: u32, max_turns: u32) -> Self {
158        Self {
159            status: OutcomeStatus::Timeout,
160            summary: summary.to_string(),
161            evidence: vec![Evidence {
162                kind: EvidenceKind::StopReason,
163                description: format!("Reached {} of {} max turns", turns, max_turns),
164                data: Some(serde_json::json!({
165                    "turns": turns,
166                    "max_turns": max_turns,
167                })),
168            }],
169            metrics: OutcomeMetrics {
170                turns,
171                ..Default::default()
172            },
173            timestamp: Utc::now(),
174        }
175    }
176
177    /// Create a give-up outcome.
178    pub fn give_up(reason: &str) -> Self {
179        Self {
180            status: OutcomeStatus::GiveUp,
181            summary: reason.to_string(),
182            evidence: vec![Evidence {
183                kind: EvidenceKind::SelfAssessment,
184                description: reason.to_string(),
185                data: None,
186            }],
187            metrics: OutcomeMetrics::default(),
188            timestamp: Utc::now(),
189        }
190    }
191
192    /// Add evidence to this outcome.
193    pub fn with_evidence(mut self, evidence: Evidence) -> Self {
194        self.evidence.push(evidence);
195        self
196    }
197
198    /// Set metrics on this outcome.
199    pub fn with_metrics(mut self, metrics: OutcomeMetrics) -> Self {
200        self.metrics = metrics;
201        self
202    }
203}
204
205#[cfg(test)]
206mod tests {
207    use super::*;
208
209    #[test]
210    fn test_outcome_status_classification() {
211        assert!(OutcomeStatus::Success.is_completed());
212        assert!(OutcomeStatus::PartialSuccess.is_completed());
213        assert!(OutcomeStatus::Done.is_completed());
214        assert!(!OutcomeStatus::Failure.is_completed());
215        assert!(!OutcomeStatus::Timeout.is_completed());
216        assert!(!OutcomeStatus::GiveUp.is_completed());
217        assert!(!OutcomeStatus::Stalled.is_completed());
218        assert!(!OutcomeStatus::NoTerminal.is_completed());
219    }
220
221    #[test]
222    fn test_all_statuses_are_terminal() {
223        assert!(OutcomeStatus::Success.is_terminal());
224        assert!(OutcomeStatus::PartialSuccess.is_terminal());
225        assert!(OutcomeStatus::Done.is_terminal());
226        assert!(OutcomeStatus::Failure.is_terminal());
227        assert!(OutcomeStatus::Timeout.is_terminal());
228        assert!(OutcomeStatus::GiveUp.is_terminal());
229    }
230
231    #[test]
232    fn test_timeout_outcome() {
233        let outcome = AgentOutcome::timeout("Exceeded step limit", 10, 10);
234        assert_eq!(outcome.status, OutcomeStatus::Timeout);
235        assert_eq!(outcome.metrics.turns, 10);
236        assert_eq!(outcome.evidence.len(), 1);
237        assert_eq!(outcome.evidence[0].kind, EvidenceKind::StopReason);
238    }
239
240    #[test]
241    fn test_outcome_with_evidence() {
242        let outcome = AgentOutcome::success("Task done")
243            .with_evidence(Evidence {
244                kind: EvidenceKind::ToolResult,
245                description: "File created".to_string(),
246                data: Some(serde_json::json!({"path": "/tmp/out.txt"})),
247            })
248            .with_evidence(Evidence {
249                kind: EvidenceKind::ExternalVerification,
250                description: "Tests passed".to_string(),
251                data: None,
252            });
253        assert_eq!(outcome.evidence.len(), 2);
254    }
255
256    #[test]
257    fn test_give_up_outcome() {
258        let outcome = AgentOutcome::give_up("Cannot access required API");
259        assert_eq!(outcome.status, OutcomeStatus::GiveUp);
260        assert_eq!(outcome.evidence.len(), 1);
261        assert_eq!(outcome.evidence[0].kind, EvidenceKind::SelfAssessment);
262    }
263
264    #[test]
265    fn test_failure_outcome() {
266        let outcome = AgentOutcome::failure("Connection refused");
267        assert_eq!(outcome.status, OutcomeStatus::Failure);
268        assert!(!outcome.status.is_completed());
269    }
270
271    #[test]
272    fn test_outcome_serde_roundtrip() {
273        let outcome = AgentOutcome::success("Done")
274            .with_evidence(Evidence {
275                kind: EvidenceKind::ToolResult,
276                description: "ok".to_string(),
277                data: Some(serde_json::json!(42)),
278            })
279            .with_metrics(OutcomeMetrics {
280                turns: 5,
281                tool_calls: 3,
282                duration_ms: 1234.5,
283                retries: 1,
284                actions_succeeded: 4,
285                actions_failed: 1,
286            });
287
288        let json = serde_json::to_string(&outcome).unwrap();
289        let roundtripped: AgentOutcome = serde_json::from_str(&json).unwrap();
290
291        assert_eq!(roundtripped.status, OutcomeStatus::Success);
292        assert_eq!(roundtripped.summary, "Done");
293        assert_eq!(roundtripped.evidence.len(), 1);
294        assert_eq!(roundtripped.metrics.turns, 5);
295        assert_eq!(roundtripped.metrics.tool_calls, 3);
296    }
297
298    /// FIX 1: the canonical harness builds the outcome itself and sends
299    /// `{status, summary, evidence, metrics, tools_called}` — NO `timestamp`,
300    /// an EXTRA `tools_called`, and a PARTIAL `metrics` object (no
301    /// `duration_ms`/`retries`). All of that must deserialize: timestamp
302    /// defaults to now, the extra field is ignored, and the missing metrics
303    /// fields default to zero.
304    #[test]
305    fn test_outcome_deserializes_harness_shape_without_timestamp() {
306        let harness = r#"{"status":"success","summary":"Created file","evidence":[],"metrics":{"turns":3,"tool_calls":3,"actions_succeeded":3,"actions_failed":0},"tools_called":["drive_cli","check_outcome","finish"]}"#;
307        let outcome: AgentOutcome =
308            serde_json::from_str(harness).expect("harness shape must deserialize");
309        assert_eq!(outcome.status, OutcomeStatus::Success);
310        assert_eq!(outcome.summary, "Created file");
311        assert!(outcome.evidence.is_empty());
312        assert_eq!(outcome.metrics.turns, 3);
313        assert_eq!(outcome.metrics.tool_calls, 3);
314        assert_eq!(outcome.metrics.actions_succeeded, 3);
315        assert_eq!(outcome.metrics.actions_failed, 0);
316        // Omitted metrics fields default to zero.
317        assert_eq!(outcome.metrics.duration_ms, 0.0);
318        assert_eq!(outcome.metrics.retries, 0);
319        // Serialize still emits a timestamp (default-now), unchanged shape.
320        let json = serde_json::to_string(&outcome).unwrap();
321        assert!(json.contains("\"timestamp\""));
322    }
323
324    #[test]
325    fn test_outcome_status_snake_case_serde() {
326        assert_eq!(
327            serde_json::to_string(&OutcomeStatus::PartialSuccess).unwrap(),
328            "\"partial_success\""
329        );
330        assert_eq!(
331            serde_json::to_string(&OutcomeStatus::GiveUp).unwrap(),
332            "\"give_up\""
333        );
334    }
335}
car_ir/outcome.rs

car_ir/
outcome.rs
