ai_agents_eval/
evidence.rs

1use std::collections::HashMap;
2
3use ai_agents_core::{ChatMessage, Role};
4use ai_agents_hitl::{ApprovalRequest, ApprovalResolvedOutcome, ApprovalResult, ApprovalTrigger};
5use ai_agents_observability::ObservabilityReport;
6use ai_agents_runtime::RuntimeAgent;
7use chrono::{DateTime, Utc};
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10
11use crate::fixtures::RecordingToolLog;
12
13/// Source category for a recorded tool execution.
14#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
15#[serde(rename_all = "snake_case")]
16pub enum ToolExecutionSource {
17    Llm,
18    Skill,
19    /// Tool step requested by the plan-and-execute runtime.
20    Plan,
21    StateAction,
22    OnEnter,
23    OnExit,
24    PostTransition,
25    Spawner,
26    Orchestration,
27    Mock,
28}
29
30/// Structured record for one tool execution observed during eval.
31#[derive(Debug, Clone, Serialize, Deserialize)]
32pub struct ToolExecutionRecord {
33    /// Unique ID for this recorded tool call.
34    pub call_id: String,
35    /// Canonical tool ID executed by the registry.
36    pub tool_id: String,
37    /// Tool name requested by the model or runtime.
38    pub requested_name: String,
39    /// Source category assigned to this execution.
40    pub source: ToolExecutionSource,
41    /// Current or expected state name.
42    #[serde(default, skip_serializing_if = "Option::is_none")]
43    pub state: Option<String>,
44    /// Actor ID associated with this evidence.
45    #[serde(default, skip_serializing_if = "Option::is_none")]
46    pub actor_id: Option<String>,
47    /// Original tool arguments before execution.
48    pub arguments_original: Value,
49    /// Arguments passed to the wrapped tool.
50    pub arguments_executed: Value,
51    /// Whether the wrapped tool implementation was invoked.
52    #[serde(default = "default_executed_true")]
53    pub executed: bool,
54    /// Whether the operation succeeded.
55    pub success: bool,
56    /// Directory where output artifacts are written.
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub output: Option<Value>,
59    /// Error text for failed execution.
60    #[serde(default, skip_serializing_if = "Option::is_none")]
61    pub error: Option<String>,
62    /// Optional response or tool metadata.
63    #[serde(default, skip_serializing_if = "Option::is_none")]
64    pub metadata: Option<Value>,
65    /// UTC timestamp when execution started.
66    pub started_at: DateTime<Utc>,
67    /// Duration in milliseconds.
68    pub duration_ms: u64,
69    /// Optional observability span ID.
70    #[serde(default, skip_serializing_if = "Option::is_none")]
71    pub observability_span_id: Option<String>,
72}
73
74fn default_executed_true() -> bool {
75    true
76}
77
78/// Skill routing evidence inferred or reported for a turn.
79#[derive(Debug, Clone, Serialize, Deserialize)]
80pub struct SkillEvidence {
81    /// Skill ID selected by routing, if available.
82    pub selected_skill_id: Option<String>,
83    /// Skill ID actually executed, if available.
84    pub executed_skill_id: Option<String>,
85    /// Whether routing found no matching skill.
86    pub no_match: bool,
87    /// Whether clarification was requested.
88    pub clarification_requested: bool,
89}
90
91/// Normalized status values for disambiguation evidence.
92#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
93#[serde(rename_all = "snake_case")]
94pub enum DisambiguationStatus {
95    Clear,
96    Skipped,
97    Triggered,
98    Clarified,
99    BestGuess,
100    Abandoned,
101    GiveUp,
102    Escalated,
103}
104
105/// Disambiguation evidence inferred or reported for a turn.
106#[derive(Debug, Clone, Serialize, Deserialize)]
107pub struct DisambiguationEvidence {
108    /// Final or normalized status value.
109    pub status: DisambiguationStatus,
110    /// Ambiguity type reported by detection, if available.
111    #[serde(default, skip_serializing_if = "Option::is_none")]
112    pub ambiguity_type: Option<String>,
113    /// Detection confidence reported by the system.
114    #[serde(default, skip_serializing_if = "Option::is_none")]
115    pub confidence: Option<f32>,
116    /// Resolved payload when available.
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub resolved: Option<Value>,
119}
120
121/// Actor fact evidence captured around one turn.
122#[derive(Debug, Clone, Serialize, Deserialize)]
123pub struct FactsEvidence {
124    /// Actor ID associated with this evidence.
125    pub actor_id: Option<String>,
126    /// Serialized fact records visible after the turn.
127    pub facts: Vec<Value>,
128    /// Number of facts before the turn when known.
129    pub before_count: Option<usize>,
130    /// Number of facts after the turn when known.
131    pub after_count: Option<usize>,
132}
133
134/// Relationship memory evidence captured around one turn.
135#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct RelationshipEvidence {
137    /// Actor ID associated with this evidence.
138    pub actor_id: Option<String>,
139    /// Model or relationship model name.
140    pub model: Option<String>,
141    /// Perspectives available for assertions.
142    pub available_perspectives: Vec<String>,
143    /// Current serialized relationship state.
144    pub current: Option<Value>,
145    /// State before the turn when available.
146    pub before: Option<Value>,
147    /// State after the turn when available.
148    pub after: Option<Value>,
149}
150
151/// Persona reveal and evolution evidence for one turn.
152#[derive(Debug, Clone, Serialize, Deserialize)]
153pub struct PersonaEvidence {
154    /// Whether any persona secret is currently revealed.
155    pub secret_revealed: bool,
156    /// IDs of revealed secrets when stable IDs are available.
157    pub revealed_secret_ids: Vec<String>,
158    /// Number of revealed secrets.
159    pub revealed_secret_count: usize,
160    /// Number of persona evolution events recorded.
161    pub evolution_events: usize,
162}
163
164/// Observability evidence attached to one evaluated turn.
165#[derive(Debug, Clone, Serialize, Deserialize)]
166pub struct TurnObservabilityEvidence {
167    /// Trace ID associated with the turn when available.
168    pub trace_id: Option<String>,
169    /// Span IDs observed during the turn.
170    pub span_ids: Vec<String>,
171    /// Observability report snapshot generated after the turn.
172    pub report: Option<ObservabilityReport>,
173}
174
175/// Normalized approval decision used by eval evidence and assertions.
176#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
177#[serde(rename_all = "snake_case")]
178pub enum ApprovalDecision {
179    Approved,
180    Rejected,
181    Modified,
182    Timeout,
183    Error,
184}
185
186/// Normalized trigger that caused an approval request.
187#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
188#[serde(tag = "type", rename_all = "snake_case")]
189pub enum ApprovalTriggerEvidence {
190    Tool { name: String },
191    Condition { name: String, matched: String },
192    State { from: Option<String>, to: String },
193}
194
195impl From<&ApprovalTrigger> for ApprovalTriggerEvidence {
196    fn from(trigger: &ApprovalTrigger) -> Self {
197        match trigger {
198            ApprovalTrigger::Tool { name, .. } => Self::Tool { name: name.clone() },
199            ApprovalTrigger::Condition { name, matched } => Self::Condition {
200                name: name.clone(),
201                matched: matched.clone(),
202            },
203            ApprovalTrigger::State { from, to } => Self::State {
204                from: from.clone(),
205                to: to.clone(),
206            },
207        }
208    }
209}
210
211/// In-memory evidence for one fully resolved approval request.
212#[derive(Debug, Clone, Serialize, Deserialize)]
213pub struct ApprovalEvidence {
214    /// Stable request ID assigned by the HITL runtime.
215    pub request_id: String,
216    /// Normalized request trigger without tool argument values.
217    pub trigger: ApprovalTriggerEvidence,
218    /// Decision returned directly by the approval handler.
219    pub raw_decision: ApprovalDecision,
220    /// Decision after runtime timeout and error resolution.
221    pub effective_decision: ApprovalDecision,
222    /// Original tool arguments supplied with the request.
223    #[serde(default, skip_serializing)]
224    pub original_args: Option<Value>,
225    /// Complete tool arguments after an effective modification.
226    #[serde(default, skip_serializing)]
227    pub modified_args: Option<Value>,
228    /// Tool arguments that would be executed after the effective decision.
229    #[serde(default, skip_serializing)]
230    pub effective_args: Option<Value>,
231    /// Localized message shown to the approval handler.
232    #[serde(default, skip_serializing)]
233    pub message: String,
234    /// Rejection reason from the effective result, when present.
235    #[serde(default, skip_serializing)]
236    pub rejection_reason: Option<String>,
237    /// Resolution error from the effective result, when present.
238    #[serde(default, skip_serializing)]
239    pub error: Option<String>,
240}
241
242impl ApprovalEvidence {
243    /// Normalize one approval hook resolution into assertion-time evidence.
244    pub fn from_resolution(
245        request: &ApprovalRequest,
246        raw_result: &ApprovalResult,
247        effective_result: &ApprovalResolvedOutcome,
248    ) -> Self {
249        let original_args = match &request.trigger {
250            ApprovalTrigger::Tool { args, .. } => Some(args.clone()),
251            _ => None,
252        };
253        let (effective_decision, changes, rejection_reason, error) = match effective_result {
254            ApprovalResolvedOutcome::Approved => (ApprovalDecision::Approved, None, None, None),
255            ApprovalResolvedOutcome::Rejected { reason } => {
256                (ApprovalDecision::Rejected, None, reason.clone(), None)
257            }
258            ApprovalResolvedOutcome::Modified { changes } => {
259                (ApprovalDecision::Modified, Some(changes), None, None)
260            }
261            ApprovalResolvedOutcome::Error { message } => {
262                (ApprovalDecision::Error, None, None, Some(message.clone()))
263            }
264        };
265        let modified_args = changes.and_then(|changes| {
266            original_args
267                .as_ref()
268                .map(|original| apply_argument_changes(original, changes))
269        });
270        let effective_args = match effective_decision {
271            ApprovalDecision::Approved => original_args.clone(),
272            ApprovalDecision::Modified => modified_args.clone(),
273            ApprovalDecision::Rejected | ApprovalDecision::Timeout | ApprovalDecision::Error => {
274                None
275            }
276        };
277
278        Self {
279            request_id: request.id.clone(),
280            trigger: ApprovalTriggerEvidence::from(&request.trigger),
281            raw_decision: approval_result_decision(raw_result),
282            effective_decision,
283            original_args,
284            modified_args,
285            effective_args,
286            message: request.message.clone(),
287            rejection_reason,
288            error,
289        }
290    }
291}
292
293fn approval_result_decision(result: &ApprovalResult) -> ApprovalDecision {
294    match result {
295        ApprovalResult::Approved => ApprovalDecision::Approved,
296        ApprovalResult::Rejected { .. } => ApprovalDecision::Rejected,
297        ApprovalResult::Modified { .. } => ApprovalDecision::Modified,
298        ApprovalResult::Timeout => ApprovalDecision::Timeout,
299    }
300}
301
302fn apply_argument_changes(original: &Value, changes: &HashMap<String, Value>) -> Value {
303    let mut modified = original.clone();
304    if let Value::Object(arguments) = &mut modified {
305        for (key, value) in changes {
306            arguments.insert(key.clone(), value.clone());
307        }
308        modified
309    } else {
310        Value::Object(changes.clone().into_iter().collect())
311    }
312}
313
314/// One message sent as part of an LLM request.
315#[derive(Debug, Clone, Serialize, Deserialize)]
316pub struct LlmMessageEvidence {
317    /// Role assigned to the message.
318    pub role: Role,
319    /// Complete message content retained only for in-memory assertions.
320    #[serde(default, skip_serializing)]
321    pub content: String,
322}
323
324/// One complete message list supplied to an LLM call.
325#[derive(Debug, Clone, Serialize, Deserialize)]
326pub struct LlmRequestEvidence {
327    /// Messages captured together by one LLM start hook invocation.
328    pub messages: Vec<LlmMessageEvidence>,
329}
330
331impl LlmRequestEvidence {
332    /// Copy a hook message slice while preserving its request boundary.
333    pub fn from_messages(messages: &[ChatMessage]) -> Self {
334        Self {
335            messages: messages
336                .iter()
337                .map(|message| LlmMessageEvidence {
338                    role: message.role,
339                    content: message.content.clone(),
340                })
341                .collect(),
342        }
343    }
344}
345
346/// Full assertion-time evidence collected after a turn.
347#[derive(Debug, Clone, Serialize, Deserialize)]
348pub struct TurnEvidence {
349    /// Response metadata produced by the runtime.
350    pub response_metadata: Option<Value>,
351    /// Current or expected state name.
352    pub state: Option<String>,
353    /// State transition history observed by the runtime.
354    pub state_history: Vec<ai_agents_core::StateTransitionEvent>,
355    /// Runtime or fixture context value.
356    pub context: Value,
357    /// Tool calls recorded during this turn.
358    pub tool_executions: Vec<ToolExecutionRecord>,
359    /// Fully resolved approval requests recorded during this turn.
360    #[serde(default)]
361    pub approvals: Vec<ApprovalEvidence>,
362    /// Complete LLM requests retained only for in-memory assertions.
363    #[serde(default, skip_serializing)]
364    pub llm_requests: Vec<LlmRequestEvidence>,
365    /// Skill evidence for this turn, if available.
366    pub skill: Option<SkillEvidence>,
367    /// Expected disambiguation status or evidence.
368    pub disambiguation: Option<DisambiguationEvidence>,
369    /// Serialized fact records visible after the turn.
370    pub facts: Option<FactsEvidence>,
371    /// Relationship memory assertion or evidence.
372    pub relationship: Option<RelationshipEvidence>,
373    /// persona value for TurnEvidence.
374    pub persona: Option<PersonaEvidence>,
375    /// Orchestration metadata assertion or evidence.
376    pub orchestration: Option<Value>,
377    /// Observability assertion, setting, or report value.
378    pub observability: Option<TurnObservabilityEvidence>,
379}
380
381pub fn collect_turn_evidence(
382    agent: &RuntimeAgent,
383    response_metadata: Option<HashMap<String, Value>>,
384    tool_log: &RecordingToolLog,
385    tool_start_index: usize,
386    before_relationship: Option<Value>,
387) -> TurnEvidence {
388    let context_map = agent.get_context();
389    let context = serde_json::to_value(&context_map).unwrap_or(Value::Null);
390    let metadata_value = response_metadata
391        .clone()
392        .and_then(|metadata| serde_json::to_value(metadata).ok());
393    let orchestration = metadata_value
394        .as_ref()
395        .and_then(|metadata| metadata.get("orchestration").cloned())
396        .or_else(|| context.get("orchestration").cloned());
397    let disambiguation = infer_disambiguation(metadata_value.as_ref(), &context);
398    let skill = infer_skill(metadata_value.as_ref(), disambiguation.as_ref());
399    let actor_id = agent.actor_id();
400    let facts = Some(FactsEvidence {
401        actor_id: actor_id.clone(),
402        facts: agent
403            .actor_facts()
404            .into_iter()
405            .filter_map(|fact| serde_json::to_value(fact).ok())
406            .collect(),
407        before_count: None,
408        after_count: Some(agent.actor_facts().len()),
409    });
410    let relationship = collect_relationship(agent, actor_id.clone(), before_relationship);
411    let persona = collect_persona(agent, &context_map);
412    let observability = agent.observability().map(|manager| {
413        let report = manager.generate_report();
414        let raw_events = manager.raw_events();
415        TurnObservabilityEvidence {
416            trace_id: raw_events.last().map(|event| event.trace_id.clone()),
417            span_ids: raw_events
418                .iter()
419                .map(|event| event.span_id.clone())
420                .collect(),
421            report: Some(report),
422        }
423    });
424
425    TurnEvidence {
426        response_metadata: metadata_value,
427        state: agent.current_state(),
428        state_history: agent.state_history(),
429        context,
430        tool_executions: tool_log.records_since(tool_start_index),
431        approvals: Vec::new(),
432        llm_requests: Vec::new(),
433        skill,
434        disambiguation,
435        facts,
436        relationship,
437        persona,
438        orchestration,
439        observability,
440    }
441}
442
443pub fn relationship_snapshot(agent: &RuntimeAgent) -> Option<Value> {
444    let actor_id = agent.actor_id()?;
445    let manager = agent.relationship_manager()?;
446    manager.relationship_as_value(&actor_id).ok().flatten()
447}
448
449fn infer_disambiguation(
450    metadata: Option<&Value>,
451    context: &Value,
452) -> Option<DisambiguationEvidence> {
453    if let Some(disambiguation) = metadata.and_then(|m| m.get("disambiguation")) {
454        let status = match disambiguation
455            .get("status")
456            .and_then(Value::as_str)
457            .unwrap_or("triggered")
458        {
459            "awaiting_clarification" | "awaiting_confirmation" => DisambiguationStatus::Triggered,
460            "clarified" => DisambiguationStatus::Clarified,
461            "best_guess" => DisambiguationStatus::BestGuess,
462            "abandoned" => DisambiguationStatus::Abandoned,
463            "give_up" => DisambiguationStatus::GiveUp,
464            "escalated" => DisambiguationStatus::Escalated,
465            "skipped" => DisambiguationStatus::Skipped,
466            "clear" => DisambiguationStatus::Clear,
467            _ => DisambiguationStatus::Triggered,
468        };
469        let detection = disambiguation.get("detection");
470        return Some(DisambiguationEvidence {
471            status,
472            ambiguity_type: detection.and_then(|d| d.get("type")).map(|v| v.to_string()),
473            confidence: detection
474                .and_then(|d| d.get("confidence"))
475                .and_then(Value::as_f64)
476                .map(|v| v as f32),
477            resolved: disambiguation.get("resolved").cloned(),
478        });
479    }
480
481    if context
482        .pointer("/disambiguation/resolved")
483        .and_then(Value::as_bool)
484        .unwrap_or(false)
485    {
486        return Some(DisambiguationEvidence {
487            status: DisambiguationStatus::Clarified,
488            ambiguity_type: None,
489            confidence: None,
490            resolved: context.get("disambiguation").cloned(),
491        });
492    }
493
494    None
495}
496
497fn infer_skill(
498    metadata: Option<&Value>,
499    disambiguation: Option<&DisambiguationEvidence>,
500) -> Option<SkillEvidence> {
501    let skill_id = metadata
502        .and_then(|m| m.get("skill_id"))
503        .and_then(Value::as_str)
504        .map(str::to_string)
505        .or_else(|| {
506            metadata
507                .and_then(|m| m.get("disambiguation"))
508                .and_then(|d| d.get("skill_id"))
509                .and_then(Value::as_str)
510                .map(str::to_string)
511        });
512
513    if skill_id.is_none() && disambiguation.is_none() {
514        return None;
515    }
516
517    Some(SkillEvidence {
518        selected_skill_id: skill_id.clone(),
519        executed_skill_id: skill_id,
520        no_match: false,
521        clarification_requested: disambiguation
522            .map(|d| d.status == DisambiguationStatus::Triggered)
523            .unwrap_or(false),
524    })
525}
526
527fn collect_relationship(
528    agent: &RuntimeAgent,
529    actor_id: Option<String>,
530    before: Option<Value>,
531) -> Option<RelationshipEvidence> {
532    let actor_id = actor_id?;
533    let manager = agent.relationship_manager()?;
534    let current = manager.relationship_as_value(&actor_id).ok().flatten();
535    let model = current
536        .as_ref()
537        .and_then(|value| value.get("model"))
538        .and_then(Value::as_str)
539        .map(str::to_string);
540    let mut available = vec!["agent_to_actor".to_string(), "mutual".to_string()];
541    if model.as_deref() == Some("two_sided") {
542        available.push("perceived_actor_to_agent".to_string());
543    }
544    Some(RelationshipEvidence {
545        actor_id: Some(actor_id),
546        model,
547        available_perspectives: available,
548        before,
549        after: current.clone(),
550        current,
551    })
552}
553
554fn collect_persona(
555    agent: &RuntimeAgent,
556    context_map: &HashMap<String, Value>,
557) -> Option<PersonaEvidence> {
558    let manager = agent.persona_manager()?;
559    let revealed_count = manager.revealed_secrets(context_map).len();
560    Some(PersonaEvidence {
561        secret_revealed: revealed_count > 0,
562        revealed_secret_ids: Vec::new(),
563        revealed_secret_count: revealed_count,
564        evolution_events: manager.history().len(),
565    })
566}
567
568#[cfg(test)]
569mod tests {
570    use super::*;
571    use serde_json::json;
572
573    #[test]
574    fn llm_request_preserves_roles_and_keeps_content_in_memory() {
575        let evidence = LlmRequestEvidence::from_messages(&[
576            ChatMessage::system("persona and reasoning prompt"),
577            ChatMessage::user("private user history"),
578            ChatMessage::assistant("private assistant history"),
579        ]);
580
581        assert_eq!(evidence.messages.len(), 3);
582        assert_eq!(evidence.messages[0].role, Role::System);
583        assert_eq!(evidence.messages[1].content, "private user history");
584
585        let serialized = serde_json::to_string(&evidence).unwrap();
586        assert!(!serialized.contains("persona and reasoning prompt"));
587        assert!(!serialized.contains("private user history"));
588        assert!(!serialized.contains("private assistant history"));
589    }
590
591    #[test]
592    fn normalizes_awaiting_confirmation_as_triggered_disambiguation() {
593        let metadata = json!({
594            "disambiguation": {
595                "status": "awaiting_confirmation",
596                "detection": {
597                    "type": "missing_target",
598                    "confidence": 0.25
599                }
600            }
601        });
602
603        let evidence = infer_disambiguation(Some(&metadata), &Value::Null).unwrap();
604        assert_eq!(evidence.status, DisambiguationStatus::Triggered);
605        assert_eq!(evidence.confidence, Some(0.25));
606    }
607
608    #[test]
609    fn normalizes_modified_approval_and_keeps_sensitive_values_in_memory() {
610        let request = ApprovalRequest::new(
611            ApprovalTrigger::tool("transfer", json!({"amount": 100, "currency": "USD"})),
612            "Approve this transfer?",
613        );
614        let mut changes = HashMap::new();
615        changes.insert("amount".to_string(), json!(25));
616        let evidence = ApprovalEvidence::from_resolution(
617            &request,
618            &ApprovalResult::Modified {
619                changes: changes.clone(),
620            },
621            &ApprovalResolvedOutcome::Modified { changes },
622        );
623
624        assert_eq!(evidence.raw_decision, ApprovalDecision::Modified);
625        assert_eq!(evidence.effective_decision, ApprovalDecision::Modified);
626        assert_eq!(
627            evidence.original_args,
628            Some(json!({"amount": 100, "currency": "USD"}))
629        );
630        assert_eq!(
631            evidence.modified_args,
632            Some(json!({"amount": 25, "currency": "USD"}))
633        );
634        assert_eq!(evidence.effective_args, evidence.modified_args);
635        assert_eq!(evidence.message, "Approve this transfer?");
636
637        let serialized = serde_json::to_value(&evidence).unwrap();
638        assert!(serialized.get("original_args").is_none());
639        assert!(serialized.get("modified_args").is_none());
640        assert!(serialized.get("effective_args").is_none());
641        assert!(serialized.get("message").is_none());
642    }
643
644    #[test]
645    fn normalizes_timeout_to_effective_error_without_executable_arguments() {
646        let request = ApprovalRequest::new(
647            ApprovalTrigger::tool("delete", json!({"path": "/private"})),
648            "Delete file?",
649        );
650        let evidence = ApprovalEvidence::from_resolution(
651            &request,
652            &ApprovalResult::Timeout,
653            &ApprovalResolvedOutcome::Error {
654                message: "approval timed out".to_string(),
655            },
656        );
657
658        assert_eq!(evidence.raw_decision, ApprovalDecision::Timeout);
659        assert_eq!(evidence.effective_decision, ApprovalDecision::Error);
660        assert_eq!(evidence.error.as_deref(), Some("approval timed out"));
661        assert!(evidence.effective_args.is_none());
662    }
663}
ai_agents_eval/evidence.rs

ai_agents_eval/
evidence.rs
