ai_agents_eval/
evidence.rs

1use std::collections::HashMap;
2
3use ai_agents_core::{ChatMessage, Role};
4use ai_agents_hitl::{ApprovalRequest, ApprovalResolvedOutcome, ApprovalResult, ApprovalTrigger};
5use ai_agents_observability::manager::ObservabilityCursor;
6use ai_agents_observability::{ObservabilityReport, ObservationEvent};
7use ai_agents_runtime::RuntimeAgent;
8use chrono::{DateTime, Utc};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11
12use crate::fixtures::RecordingToolLog;
13
14/// Source category for a recorded tool execution.
15#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
16#[serde(rename_all = "snake_case")]
17pub enum ToolExecutionSource {
18    Llm,
19    Skill,
20    /// Tool step requested by the plan-and-execute runtime.
21    Plan,
22    StateAction,
23    OnEnter,
24    OnExit,
25    PostTransition,
26    Spawner,
27    Orchestration,
28    Mock,
29}
30
31/// Structured record for one tool execution observed during eval.
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct ToolExecutionRecord {
34    /// Unique ID for this recorded tool call.
35    pub call_id: String,
36    /// Canonical tool ID executed by the registry.
37    pub tool_id: String,
38    /// Tool name requested by the model or runtime.
39    pub requested_name: String,
40    /// Source category assigned to this execution.
41    pub source: ToolExecutionSource,
42    /// Current or expected state name.
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub state: Option<String>,
45    /// Actor ID associated with this evidence.
46    #[serde(default, skip_serializing_if = "Option::is_none")]
47    pub actor_id: Option<String>,
48    /// Original tool arguments before execution.
49    pub arguments_original: Value,
50    /// Arguments passed to the wrapped tool.
51    pub arguments_executed: Value,
52    /// Whether the wrapped tool implementation was invoked.
53    #[serde(default = "default_executed_true")]
54    pub executed: bool,
55    /// Whether the operation succeeded.
56    pub success: bool,
57    /// Directory where output artifacts are written.
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub output: Option<Value>,
60    /// Error text for failed execution.
61    #[serde(default, skip_serializing_if = "Option::is_none")]
62    pub error: Option<String>,
63    /// Optional response or tool metadata.
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub metadata: Option<Value>,
66    /// UTC timestamp when execution started.
67    pub started_at: DateTime<Utc>,
68    /// Duration in milliseconds.
69    pub duration_ms: u64,
70    /// Optional observability span ID.
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub observability_span_id: Option<String>,
73}
74
75fn default_executed_true() -> bool {
76    true
77}
78
79/// Skill routing evidence inferred or reported for a turn.
80#[derive(Debug, Clone, Serialize, Deserialize)]
81pub struct SkillEvidence {
82    /// Skill ID selected by routing, if available.
83    pub selected_skill_id: Option<String>,
84    /// Skill ID actually executed, if available.
85    pub executed_skill_id: Option<String>,
86    /// Whether routing found no matching skill.
87    pub no_match: bool,
88    /// Whether clarification was requested.
89    pub clarification_requested: bool,
90}
91
92/// Normalized status values for disambiguation evidence.
93#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
94#[serde(rename_all = "snake_case")]
95pub enum DisambiguationStatus {
96    Clear,
97    Skipped,
98    Triggered,
99    Clarified,
100    BestGuess,
101    Abandoned,
102    GiveUp,
103    Escalated,
104}
105
106/// Disambiguation evidence inferred or reported for a turn.
107#[derive(Debug, Clone, Serialize, Deserialize)]
108pub struct DisambiguationEvidence {
109    /// Final or normalized status value.
110    pub status: DisambiguationStatus,
111    /// Ambiguity type reported by detection, if available.
112    #[serde(default, skip_serializing_if = "Option::is_none")]
113    pub ambiguity_type: Option<String>,
114    /// Detection confidence reported by the system.
115    #[serde(default, skip_serializing_if = "Option::is_none")]
116    pub confidence: Option<f32>,
117    /// Resolved payload when available.
118    #[serde(default, skip_serializing_if = "Option::is_none")]
119    pub resolved: Option<Value>,
120}
121
122/// Actor fact evidence captured around one turn.
123#[derive(Debug, Clone, Serialize, Deserialize)]
124pub struct FactsEvidence {
125    /// Actor ID associated with this evidence.
126    pub actor_id: Option<String>,
127    /// Serialized fact records visible after the turn.
128    pub facts: Vec<Value>,
129    /// Number of facts before the turn when known.
130    pub before_count: Option<usize>,
131    /// Number of facts after the turn when known.
132    pub after_count: Option<usize>,
133}
134
135/// Relationship memory evidence captured around one turn.
136#[derive(Debug, Clone, Serialize, Deserialize)]
137pub struct RelationshipEvidence {
138    /// Actor ID associated with this evidence.
139    pub actor_id: Option<String>,
140    /// Model or relationship model name.
141    pub model: Option<String>,
142    /// Perspectives available for assertions.
143    pub available_perspectives: Vec<String>,
144    /// Current serialized relationship state.
145    pub current: Option<Value>,
146    /// State before the turn when available.
147    pub before: Option<Value>,
148    /// State after the turn when available.
149    pub after: Option<Value>,
150}
151
152/// Persona reveal and evolution evidence for one turn.
153#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct PersonaEvidence {
155    /// Whether any persona secret is currently revealed.
156    pub secret_revealed: bool,
157    /// IDs of revealed secrets when stable IDs are available.
158    pub revealed_secret_ids: Vec<String>,
159    /// Number of revealed secrets.
160    pub revealed_secret_count: usize,
161    /// Number of persona evolution events recorded.
162    pub evolution_events: usize,
163}
164
165/// Observability evidence attached to one evaluated turn.
166#[derive(Debug, Clone, Serialize, Deserialize)]
167pub struct TurnObservabilityEvidence {
168    /// Trace ID associated with the turn when available.
169    pub trace_id: Option<String>,
170    /// Span IDs observed during the turn.
171    pub span_ids: Vec<String>,
172    /// Observability report snapshot generated after the turn.
173    pub report: Option<ObservabilityReport>,
174}
175
176/// Normalized approval decision used by eval evidence and assertions.
177#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
178#[serde(rename_all = "snake_case")]
179pub enum ApprovalDecision {
180    Approved,
181    Rejected,
182    Modified,
183    Timeout,
184    Error,
185}
186
187/// Normalized trigger that caused an approval request.
188#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
189#[serde(tag = "type", rename_all = "snake_case")]
190pub enum ApprovalTriggerEvidence {
191    Tool { name: String },
192    Condition { name: String, matched: String },
193    State { from: Option<String>, to: String },
194}
195
196impl From<&ApprovalTrigger> for ApprovalTriggerEvidence {
197    fn from(trigger: &ApprovalTrigger) -> Self {
198        match trigger {
199            ApprovalTrigger::Tool { name, .. } => Self::Tool { name: name.clone() },
200            ApprovalTrigger::Condition { name, matched } => Self::Condition {
201                name: name.clone(),
202                matched: matched.clone(),
203            },
204            ApprovalTrigger::State { from, to } => Self::State {
205                from: from.clone(),
206                to: to.clone(),
207            },
208        }
209    }
210}
211
212/// In-memory evidence for one fully resolved approval request.
213#[derive(Debug, Clone, Serialize, Deserialize)]
214pub struct ApprovalEvidence {
215    /// Stable request ID assigned by the HITL runtime.
216    pub request_id: String,
217    /// Normalized request trigger without tool argument values.
218    pub trigger: ApprovalTriggerEvidence,
219    /// Decision returned directly by the approval handler.
220    pub raw_decision: ApprovalDecision,
221    /// Decision after runtime timeout and error resolution.
222    pub effective_decision: ApprovalDecision,
223    /// Original tool arguments supplied with the request.
224    #[serde(default, skip_serializing)]
225    pub original_args: Option<Value>,
226    /// Complete tool arguments after an effective modification.
227    #[serde(default, skip_serializing)]
228    pub modified_args: Option<Value>,
229    /// Tool arguments that would be executed after the effective decision.
230    #[serde(default, skip_serializing)]
231    pub effective_args: Option<Value>,
232    /// Localized message shown to the approval handler.
233    #[serde(default, skip_serializing)]
234    pub message: String,
235    /// Rejection reason from the effective result, when present.
236    #[serde(default, skip_serializing)]
237    pub rejection_reason: Option<String>,
238    /// Resolution error from the effective result, when present.
239    #[serde(default, skip_serializing)]
240    pub error: Option<String>,
241}
242
243impl ApprovalEvidence {
244    /// Normalize one approval hook resolution into assertion-time evidence.
245    pub fn from_resolution(
246        request: &ApprovalRequest,
247        raw_result: &ApprovalResult,
248        effective_result: &ApprovalResolvedOutcome,
249    ) -> Self {
250        let original_args = match &request.trigger {
251            ApprovalTrigger::Tool { args, .. } => Some(args.clone()),
252            _ => None,
253        };
254        let (effective_decision, changes, rejection_reason, error) = match effective_result {
255            ApprovalResolvedOutcome::Approved => (ApprovalDecision::Approved, None, None, None),
256            ApprovalResolvedOutcome::Rejected { reason } => {
257                (ApprovalDecision::Rejected, None, reason.clone(), None)
258            }
259            ApprovalResolvedOutcome::Modified { changes } => {
260                (ApprovalDecision::Modified, Some(changes), None, None)
261            }
262            ApprovalResolvedOutcome::Error { message } => {
263                (ApprovalDecision::Error, None, None, Some(message.clone()))
264            }
265        };
266        let modified_args = changes.and_then(|changes| {
267            original_args
268                .as_ref()
269                .map(|original| apply_argument_changes(original, changes))
270        });
271        let effective_args = match effective_decision {
272            ApprovalDecision::Approved => original_args.clone(),
273            ApprovalDecision::Modified => modified_args.clone(),
274            ApprovalDecision::Rejected | ApprovalDecision::Timeout | ApprovalDecision::Error => {
275                None
276            }
277        };
278
279        Self {
280            request_id: request.id.clone(),
281            trigger: ApprovalTriggerEvidence::from(&request.trigger),
282            raw_decision: approval_result_decision(raw_result),
283            effective_decision,
284            original_args,
285            modified_args,
286            effective_args,
287            message: request.message.clone(),
288            rejection_reason,
289            error,
290        }
291    }
292}
293
294fn approval_result_decision(result: &ApprovalResult) -> ApprovalDecision {
295    match result {
296        ApprovalResult::Approved => ApprovalDecision::Approved,
297        ApprovalResult::Rejected { .. } => ApprovalDecision::Rejected,
298        ApprovalResult::Modified { .. } => ApprovalDecision::Modified,
299        ApprovalResult::Timeout => ApprovalDecision::Timeout,
300    }
301}
302
303fn apply_argument_changes(original: &Value, changes: &HashMap<String, Value>) -> Value {
304    let mut modified = original.clone();
305    if let Value::Object(arguments) = &mut modified {
306        for (key, value) in changes {
307            arguments.insert(key.clone(), value.clone());
308        }
309        modified
310    } else {
311        Value::Object(changes.clone().into_iter().collect())
312    }
313}
314
315/// One message sent as part of an LLM request.
316#[derive(Debug, Clone, Serialize, Deserialize)]
317pub struct LlmMessageEvidence {
318    /// Role assigned to the message.
319    pub role: Role,
320    /// Complete message content retained only for in-memory assertions.
321    #[serde(default, skip_serializing)]
322    pub content: String,
323}
324
325/// One complete message list supplied to an LLM call.
326#[derive(Debug, Clone, Serialize, Deserialize)]
327pub struct LlmRequestEvidence {
328    /// Messages captured together by one LLM start hook invocation.
329    pub messages: Vec<LlmMessageEvidence>,
330}
331
332impl LlmRequestEvidence {
333    /// Copy a hook message slice while preserving its request boundary.
334    pub fn from_messages(messages: &[ChatMessage]) -> Self {
335        Self {
336            messages: messages
337                .iter()
338                .map(|message| LlmMessageEvidence {
339                    role: message.role,
340                    content: message.content.clone(),
341                })
342                .collect(),
343        }
344    }
345}
346
347/// Full assertion-time evidence collected after a turn.
348#[derive(Debug, Clone, Serialize, Deserialize)]
349pub struct TurnEvidence {
350    /// Response metadata produced by the runtime.
351    pub response_metadata: Option<Value>,
352    /// Current or expected state name.
353    pub state: Option<String>,
354    /// State transition history observed by the runtime.
355    pub state_history: Vec<ai_agents_core::StateTransitionEvent>,
356    /// Runtime or fixture context value.
357    pub context: Value,
358    /// Tool calls recorded during this turn.
359    pub tool_executions: Vec<ToolExecutionRecord>,
360    /// Fully resolved approval requests recorded during this turn.
361    #[serde(default)]
362    pub approvals: Vec<ApprovalEvidence>,
363    /// Complete LLM requests retained only for in-memory assertions.
364    #[serde(default, skip_serializing)]
365    pub llm_requests: Vec<LlmRequestEvidence>,
366    /// Skill evidence for this turn, if available.
367    pub skill: Option<SkillEvidence>,
368    /// Expected disambiguation status or evidence.
369    pub disambiguation: Option<DisambiguationEvidence>,
370    /// Serialized fact records visible after the turn.
371    pub facts: Option<FactsEvidence>,
372    /// Relationship memory assertion or evidence.
373    pub relationship: Option<RelationshipEvidence>,
374    /// persona value for TurnEvidence.
375    pub persona: Option<PersonaEvidence>,
376    /// Orchestration metadata assertion or evidence.
377    pub orchestration: Option<Value>,
378    /// Observability assertion, setting, or report value.
379    pub observability: Option<TurnObservabilityEvidence>,
380}
381
382/// Collects post-turn runtime evidence with the original cumulative observability snapshot.
383pub fn collect_turn_evidence(
384    agent: &RuntimeAgent,
385    response_metadata: Option<HashMap<String, Value>>,
386    tool_log: &RecordingToolLog,
387    tool_start_index: usize,
388    before_relationship: Option<Value>,
389) -> TurnEvidence {
390    let observability = agent.observability().map(|manager| {
391        let report = manager.generate_report();
392        let events = manager.raw_events();
393        observability_evidence(events, report)
394    });
395    collect_turn_evidence_with_observability(
396        agent,
397        response_metadata,
398        tool_log,
399        tool_start_index,
400        before_relationship,
401        observability,
402    )
403}
404
405/// Collects EvalRunner evidence with observability scoped after the pre-turn cursor.
406pub(crate) fn collect_turn_evidence_since(
407    agent: &RuntimeAgent,
408    response_metadata: Option<HashMap<String, Value>>,
409    tool_log: &RecordingToolLog,
410    tool_start_index: usize,
411    before_relationship: Option<Value>,
412    observability_cursor: Option<ObservabilityCursor>,
413) -> TurnEvidence {
414    let observability = agent
415        .observability()
416        .zip(observability_cursor)
417        .map(|(manager, cursor)| {
418            let snapshot = manager.snapshot_since(cursor);
419            observability_evidence(snapshot.events, snapshot.report)
420        });
421    collect_turn_evidence_with_observability(
422        agent,
423        response_metadata,
424        tool_log,
425        tool_start_index,
426        before_relationship,
427        observability,
428    )
429}
430
431// Assembles common runtime evidence after the caller selects cumulative or scoped observability.
432fn collect_turn_evidence_with_observability(
433    agent: &RuntimeAgent,
434    response_metadata: Option<HashMap<String, Value>>,
435    tool_log: &RecordingToolLog,
436    tool_start_index: usize,
437    before_relationship: Option<Value>,
438    observability: Option<TurnObservabilityEvidence>,
439) -> TurnEvidence {
440    let context_map = agent.get_context();
441    let context = serde_json::to_value(&context_map).unwrap_or(Value::Null);
442    let metadata_value = response_metadata
443        .clone()
444        .and_then(|metadata| serde_json::to_value(metadata).ok());
445    let orchestration = metadata_value
446        .as_ref()
447        .and_then(|metadata| metadata.get("orchestration").cloned())
448        .or_else(|| context.get("orchestration").cloned());
449    let disambiguation = infer_disambiguation(metadata_value.as_ref(), &context);
450    let skill = infer_skill(metadata_value.as_ref(), disambiguation.as_ref());
451    let actor_id = agent.actor_id();
452    let facts = Some(FactsEvidence {
453        actor_id: actor_id.clone(),
454        facts: agent
455            .actor_facts()
456            .into_iter()
457            .filter_map(|fact| serde_json::to_value(fact).ok())
458            .collect(),
459        before_count: None,
460        after_count: Some(agent.actor_facts().len()),
461    });
462    let relationship = collect_relationship(agent, actor_id.clone(), before_relationship);
463    let persona = collect_persona(agent, &context_map);
464
465    TurnEvidence {
466        response_metadata: metadata_value,
467        state: agent.current_state(),
468        state_history: agent.state_history(),
469        context,
470        tool_executions: tool_log.records_since(tool_start_index),
471        approvals: Vec::new(),
472        llm_requests: Vec::new(),
473        skill,
474        disambiguation,
475        facts,
476        relationship,
477        persona,
478        orchestration,
479        observability,
480    }
481}
482
483// Converts one selected event vector and its matching report into eval evidence.
484fn observability_evidence(
485    events: Vec<ObservationEvent>,
486    report: ObservabilityReport,
487) -> TurnObservabilityEvidence {
488    TurnObservabilityEvidence {
489        trace_id: events.last().map(|event| event.trace_id.clone()),
490        span_ids: events.iter().map(|event| event.span_id.clone()).collect(),
491        report: Some(report),
492    }
493}
494
495pub fn relationship_snapshot(agent: &RuntimeAgent) -> Option<Value> {
496    let actor_id = agent.actor_id()?;
497    let manager = agent.relationship_manager()?;
498    manager.relationship_as_value(&actor_id).ok().flatten()
499}
500
501fn infer_disambiguation(
502    metadata: Option<&Value>,
503    context: &Value,
504) -> Option<DisambiguationEvidence> {
505    if let Some(disambiguation) = metadata.and_then(|m| m.get("disambiguation")) {
506        let status = match disambiguation
507            .get("status")
508            .and_then(Value::as_str)
509            .unwrap_or("triggered")
510        {
511            "awaiting_clarification" | "awaiting_confirmation" => DisambiguationStatus::Triggered,
512            "clarified" => DisambiguationStatus::Clarified,
513            "best_guess" => DisambiguationStatus::BestGuess,
514            "abandoned" => DisambiguationStatus::Abandoned,
515            "give_up" => DisambiguationStatus::GiveUp,
516            "escalated" => DisambiguationStatus::Escalated,
517            "skipped" => DisambiguationStatus::Skipped,
518            "clear" => DisambiguationStatus::Clear,
519            _ => DisambiguationStatus::Triggered,
520        };
521        let detection = disambiguation.get("detection");
522        return Some(DisambiguationEvidence {
523            status,
524            ambiguity_type: detection.and_then(|d| d.get("type")).map(|v| v.to_string()),
525            confidence: detection
526                .and_then(|d| d.get("confidence"))
527                .and_then(Value::as_f64)
528                .map(|v| v as f32),
529            resolved: disambiguation.get("resolved").cloned(),
530        });
531    }
532
533    if context
534        .pointer("/disambiguation/resolved")
535        .and_then(Value::as_bool)
536        .unwrap_or(false)
537    {
538        return Some(DisambiguationEvidence {
539            status: DisambiguationStatus::Clarified,
540            ambiguity_type: None,
541            confidence: None,
542            resolved: context.get("disambiguation").cloned(),
543        });
544    }
545
546    None
547}
548
549fn infer_skill(
550    metadata: Option<&Value>,
551    disambiguation: Option<&DisambiguationEvidence>,
552) -> Option<SkillEvidence> {
553    let skill_id = metadata
554        .and_then(|m| m.get("skill_id"))
555        .and_then(Value::as_str)
556        .map(str::to_string)
557        .or_else(|| {
558            metadata
559                .and_then(|m| m.get("disambiguation"))
560                .and_then(|d| d.get("skill_id"))
561                .and_then(Value::as_str)
562                .map(str::to_string)
563        });
564
565    if skill_id.is_none() && disambiguation.is_none() {
566        return None;
567    }
568
569    Some(SkillEvidence {
570        selected_skill_id: skill_id.clone(),
571        executed_skill_id: skill_id,
572        no_match: false,
573        clarification_requested: disambiguation
574            .map(|d| d.status == DisambiguationStatus::Triggered)
575            .unwrap_or(false),
576    })
577}
578
579fn collect_relationship(
580    agent: &RuntimeAgent,
581    actor_id: Option<String>,
582    before: Option<Value>,
583) -> Option<RelationshipEvidence> {
584    let actor_id = actor_id?;
585    let manager = agent.relationship_manager()?;
586    let current = manager.relationship_as_value(&actor_id).ok().flatten();
587    let model = current
588        .as_ref()
589        .and_then(|value| value.get("model"))
590        .and_then(Value::as_str)
591        .map(str::to_string);
592    let mut available = vec!["agent_to_actor".to_string(), "mutual".to_string()];
593    if model.as_deref() == Some("two_sided") {
594        available.push("perceived_actor_to_agent".to_string());
595    }
596    Some(RelationshipEvidence {
597        actor_id: Some(actor_id),
598        model,
599        available_perspectives: available,
600        before,
601        after: current.clone(),
602        current,
603    })
604}
605
606fn collect_persona(
607    agent: &RuntimeAgent,
608    context_map: &HashMap<String, Value>,
609) -> Option<PersonaEvidence> {
610    let manager = agent.persona_manager()?;
611    let revealed_count = manager.revealed_secrets(context_map).len();
612    Some(PersonaEvidence {
613        secret_revealed: revealed_count > 0,
614        revealed_secret_ids: Vec::new(),
615        revealed_secret_count: revealed_count,
616        evolution_events: manager.history().len(),
617    })
618}
619
620#[cfg(test)]
621mod tests {
622    use super::*;
623    use serde_json::json;
624
625    /// Locks the public collector's patch-compatible argument list at compile time.
626    type CollectTurnEvidenceFn = fn(
627        &RuntimeAgent,
628        Option<HashMap<String, Value>>,
629        &RecordingToolLog,
630        usize,
631        Option<Value>,
632    ) -> TurnEvidence;
633
634    #[test]
635    fn collect_turn_evidence_keeps_five_argument_signature() {
636        let _: CollectTurnEvidenceFn = collect_turn_evidence;
637    }
638
639    #[test]
640    fn llm_request_preserves_roles_and_keeps_content_in_memory() {
641        let evidence = LlmRequestEvidence::from_messages(&[
642            ChatMessage::system("persona and reasoning prompt"),
643            ChatMessage::user("private user history"),
644            ChatMessage::assistant("private assistant history"),
645        ]);
646
647        assert_eq!(evidence.messages.len(), 3);
648        assert_eq!(evidence.messages[0].role, Role::System);
649        assert_eq!(evidence.messages[1].content, "private user history");
650
651        let serialized = serde_json::to_string(&evidence).unwrap();
652        assert!(!serialized.contains("persona and reasoning prompt"));
653        assert!(!serialized.contains("private user history"));
654        assert!(!serialized.contains("private assistant history"));
655    }
656
657    #[test]
658    fn normalizes_awaiting_confirmation_as_triggered_disambiguation() {
659        let metadata = json!({
660            "disambiguation": {
661                "status": "awaiting_confirmation",
662                "detection": {
663                    "type": "missing_target",
664                    "confidence": 0.25
665                }
666            }
667        });
668
669        let evidence = infer_disambiguation(Some(&metadata), &Value::Null).unwrap();
670        assert_eq!(evidence.status, DisambiguationStatus::Triggered);
671        assert_eq!(evidence.confidence, Some(0.25));
672    }
673
674    #[test]
675    fn normalizes_modified_approval_and_keeps_sensitive_values_in_memory() {
676        let request = ApprovalRequest::new(
677            ApprovalTrigger::tool("transfer", json!({"amount": 100, "currency": "USD"})),
678            "Approve this transfer?",
679        );
680        let mut changes = HashMap::new();
681        changes.insert("amount".to_string(), json!(25));
682        let evidence = ApprovalEvidence::from_resolution(
683            &request,
684            &ApprovalResult::Modified {
685                changes: changes.clone(),
686            },
687            &ApprovalResolvedOutcome::Modified { changes },
688        );
689
690        assert_eq!(evidence.raw_decision, ApprovalDecision::Modified);
691        assert_eq!(evidence.effective_decision, ApprovalDecision::Modified);
692        assert_eq!(
693            evidence.original_args,
694            Some(json!({"amount": 100, "currency": "USD"}))
695        );
696        assert_eq!(
697            evidence.modified_args,
698            Some(json!({"amount": 25, "currency": "USD"}))
699        );
700        assert_eq!(evidence.effective_args, evidence.modified_args);
701        assert_eq!(evidence.message, "Approve this transfer?");
702
703        let serialized = serde_json::to_value(&evidence).unwrap();
704        assert!(serialized.get("original_args").is_none());
705        assert!(serialized.get("modified_args").is_none());
706        assert!(serialized.get("effective_args").is_none());
707        assert!(serialized.get("message").is_none());
708    }
709
710    #[test]
711    fn normalizes_timeout_to_effective_error_without_executable_arguments() {
712        let request = ApprovalRequest::new(
713            ApprovalTrigger::tool("delete", json!({"path": "/private"})),
714            "Delete file?",
715        );
716        let evidence = ApprovalEvidence::from_resolution(
717            &request,
718            &ApprovalResult::Timeout,
719            &ApprovalResolvedOutcome::Error {
720                message: "approval timed out".to_string(),
721            },
722        );
723
724        assert_eq!(evidence.raw_decision, ApprovalDecision::Timeout);
725        assert_eq!(evidence.effective_decision, ApprovalDecision::Error);
726        assert_eq!(evidence.error.as_deref(), Some("approval timed out"));
727        assert!(evidence.effective_args.is_none());
728    }
729}
ai_agents_eval/evidence.rs

ai_agents_eval/
evidence.rs
