ai_agents_eval/
evidence.rs

1use std::collections::HashMap;
2
3use ai_agents_observability::ObservabilityReport;
4use ai_agents_runtime::RuntimeAgent;
5use chrono::{DateTime, Utc};
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8
9use crate::fixtures::RecordingToolLog;
10
11/// Source category for a recorded tool execution.
12#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
13#[serde(rename_all = "snake_case")]
14pub enum ToolExecutionSource {
15    Llm,
16    Skill,
17    StateAction,
18    OnEnter,
19    OnExit,
20    PostTransition,
21    Spawner,
22    Orchestration,
23    Mock,
24}
25
26/// Structured record for one tool execution observed during eval.
27#[derive(Debug, Clone, Serialize, Deserialize)]
28pub struct ToolExecutionRecord {
29    /// Unique ID for this recorded tool call.
30    pub call_id: String,
31    /// Canonical tool ID executed by the registry.
32    pub tool_id: String,
33    /// Tool name requested by the model or runtime.
34    pub requested_name: String,
35    /// Source category assigned to this execution.
36    pub source: ToolExecutionSource,
37    /// Current or expected state name.
38    #[serde(default, skip_serializing_if = "Option::is_none")]
39    pub state: Option<String>,
40    /// Actor ID associated with this evidence.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub actor_id: Option<String>,
43    /// Original tool arguments before execution.
44    pub arguments_original: Value,
45    /// Arguments passed to the wrapped tool.
46    pub arguments_executed: Value,
47    /// Whether the wrapped tool implementation was invoked.
48    #[serde(default = "default_executed_true")]
49    pub executed: bool,
50    /// Whether the operation succeeded.
51    pub success: bool,
52    /// Directory where output artifacts are written.
53    #[serde(default, skip_serializing_if = "Option::is_none")]
54    pub output: Option<Value>,
55    /// Error text for failed execution.
56    #[serde(default, skip_serializing_if = "Option::is_none")]
57    pub error: Option<String>,
58    /// Optional response or tool metadata.
59    #[serde(default, skip_serializing_if = "Option::is_none")]
60    pub metadata: Option<Value>,
61    /// UTC timestamp when execution started.
62    pub started_at: DateTime<Utc>,
63    /// Duration in milliseconds.
64    pub duration_ms: u64,
65    /// Optional observability span ID.
66    #[serde(default, skip_serializing_if = "Option::is_none")]
67    pub observability_span_id: Option<String>,
68}
69
70fn default_executed_true() -> bool {
71    true
72}
73
74/// Skill routing evidence inferred or reported for a turn.
75#[derive(Debug, Clone, Serialize, Deserialize)]
76pub struct SkillEvidence {
77    /// Skill ID selected by routing, if available.
78    pub selected_skill_id: Option<String>,
79    /// Skill ID actually executed, if available.
80    pub executed_skill_id: Option<String>,
81    /// Whether routing found no matching skill.
82    pub no_match: bool,
83    /// Whether clarification was requested.
84    pub clarification_requested: bool,
85}
86
87/// Normalized status values for disambiguation evidence.
88#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
89#[serde(rename_all = "snake_case")]
90pub enum DisambiguationStatus {
91    Clear,
92    Skipped,
93    Triggered,
94    Clarified,
95    BestGuess,
96    Abandoned,
97    GiveUp,
98    Escalated,
99}
100
101/// Disambiguation evidence inferred or reported for a turn.
102#[derive(Debug, Clone, Serialize, Deserialize)]
103pub struct DisambiguationEvidence {
104    /// Final or normalized status value.
105    pub status: DisambiguationStatus,
106    /// Ambiguity type reported by detection, if available.
107    #[serde(default, skip_serializing_if = "Option::is_none")]
108    pub ambiguity_type: Option<String>,
109    /// Detection confidence reported by the system.
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub confidence: Option<f32>,
112    /// Resolved payload when available.
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub resolved: Option<Value>,
115}
116
117/// Actor fact evidence captured around one turn.
118#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct FactsEvidence {
120    /// Actor ID associated with this evidence.
121    pub actor_id: Option<String>,
122    /// Serialized fact records visible after the turn.
123    pub facts: Vec<Value>,
124    /// Number of facts before the turn when known.
125    pub before_count: Option<usize>,
126    /// Number of facts after the turn when known.
127    pub after_count: Option<usize>,
128}
129
130/// Relationship memory evidence captured around one turn.
131#[derive(Debug, Clone, Serialize, Deserialize)]
132pub struct RelationshipEvidence {
133    /// Actor ID associated with this evidence.
134    pub actor_id: Option<String>,
135    /// Model or relationship model name.
136    pub model: Option<String>,
137    /// Perspectives available for assertions.
138    pub available_perspectives: Vec<String>,
139    /// Current serialized relationship state.
140    pub current: Option<Value>,
141    /// State before the turn when available.
142    pub before: Option<Value>,
143    /// State after the turn when available.
144    pub after: Option<Value>,
145}
146
147/// Persona reveal and evolution evidence for one turn.
148#[derive(Debug, Clone, Serialize, Deserialize)]
149pub struct PersonaEvidence {
150    /// Whether any persona secret is currently revealed.
151    pub secret_revealed: bool,
152    /// IDs of revealed secrets when stable IDs are available.
153    pub revealed_secret_ids: Vec<String>,
154    /// Number of revealed secrets.
155    pub revealed_secret_count: usize,
156    /// Number of persona evolution events recorded.
157    pub evolution_events: usize,
158}
159
160/// Observability evidence attached to one evaluated turn.
161#[derive(Debug, Clone, Serialize, Deserialize)]
162pub struct TurnObservabilityEvidence {
163    /// Trace ID associated with the turn when available.
164    pub trace_id: Option<String>,
165    /// Span IDs observed during the turn.
166    pub span_ids: Vec<String>,
167    /// Observability report snapshot generated after the turn.
168    pub report: Option<ObservabilityReport>,
169}
170
171/// Full assertion-time evidence collected after a turn.
172#[derive(Debug, Clone, Serialize, Deserialize)]
173pub struct TurnEvidence {
174    /// Response metadata produced by the runtime.
175    pub response_metadata: Option<Value>,
176    /// Current or expected state name.
177    pub state: Option<String>,
178    /// State transition history observed by the runtime.
179    pub state_history: Vec<ai_agents_core::StateTransitionEvent>,
180    /// Runtime or fixture context value.
181    pub context: Value,
182    /// Tool calls recorded during this turn.
183    pub tool_executions: Vec<ToolExecutionRecord>,
184    /// Skill evidence for this turn, if available.
185    pub skill: Option<SkillEvidence>,
186    /// Expected disambiguation status or evidence.
187    pub disambiguation: Option<DisambiguationEvidence>,
188    /// Serialized fact records visible after the turn.
189    pub facts: Option<FactsEvidence>,
190    /// Relationship memory assertion or evidence.
191    pub relationship: Option<RelationshipEvidence>,
192    /// persona value for TurnEvidence.
193    pub persona: Option<PersonaEvidence>,
194    /// Orchestration metadata assertion or evidence.
195    pub orchestration: Option<Value>,
196    /// Observability assertion, setting, or report value.
197    pub observability: Option<TurnObservabilityEvidence>,
198}
199
200pub fn collect_turn_evidence(
201    agent: &RuntimeAgent,
202    response_metadata: Option<HashMap<String, Value>>,
203    tool_log: &RecordingToolLog,
204    tool_start_index: usize,
205    before_relationship: Option<Value>,
206) -> TurnEvidence {
207    let context_map = agent.get_context();
208    let context = serde_json::to_value(&context_map).unwrap_or(Value::Null);
209    let metadata_value = response_metadata
210        .clone()
211        .and_then(|metadata| serde_json::to_value(metadata).ok());
212    let orchestration = metadata_value
213        .as_ref()
214        .and_then(|metadata| metadata.get("orchestration").cloned())
215        .or_else(|| context.get("orchestration").cloned());
216    let disambiguation = infer_disambiguation(metadata_value.as_ref(), &context);
217    let skill = infer_skill(metadata_value.as_ref(), disambiguation.as_ref());
218    let actor_id = agent.actor_id();
219    let facts = Some(FactsEvidence {
220        actor_id: actor_id.clone(),
221        facts: agent
222            .actor_facts()
223            .into_iter()
224            .filter_map(|fact| serde_json::to_value(fact).ok())
225            .collect(),
226        before_count: None,
227        after_count: Some(agent.actor_facts().len()),
228    });
229    let relationship = collect_relationship(agent, actor_id.clone(), before_relationship);
230    let persona = collect_persona(agent, &context_map);
231    let observability = agent.observability().map(|manager| {
232        let report = manager.generate_report();
233        let raw_events = manager.raw_events();
234        TurnObservabilityEvidence {
235            trace_id: raw_events.last().map(|event| event.trace_id.clone()),
236            span_ids: raw_events
237                .iter()
238                .map(|event| event.span_id.clone())
239                .collect(),
240            report: Some(report),
241        }
242    });
243
244    TurnEvidence {
245        response_metadata: metadata_value,
246        state: agent.current_state(),
247        state_history: agent.state_history(),
248        context,
249        tool_executions: tool_log.records_since(tool_start_index),
250        skill,
251        disambiguation,
252        facts,
253        relationship,
254        persona,
255        orchestration,
256        observability,
257    }
258}
259
260pub fn relationship_snapshot(agent: &RuntimeAgent) -> Option<Value> {
261    let actor_id = agent.actor_id()?;
262    let manager = agent.relationship_manager()?;
263    manager.relationship_as_value(&actor_id).ok().flatten()
264}
265
266fn infer_disambiguation(
267    metadata: Option<&Value>,
268    context: &Value,
269) -> Option<DisambiguationEvidence> {
270    if let Some(disambiguation) = metadata.and_then(|m| m.get("disambiguation")) {
271        let status = match disambiguation
272            .get("status")
273            .and_then(Value::as_str)
274            .unwrap_or("triggered")
275        {
276            "awaiting_clarification" => DisambiguationStatus::Triggered,
277            "clarified" => DisambiguationStatus::Clarified,
278            "best_guess" => DisambiguationStatus::BestGuess,
279            "abandoned" => DisambiguationStatus::Abandoned,
280            "give_up" => DisambiguationStatus::GiveUp,
281            "escalated" => DisambiguationStatus::Escalated,
282            "skipped" => DisambiguationStatus::Skipped,
283            "clear" => DisambiguationStatus::Clear,
284            _ => DisambiguationStatus::Triggered,
285        };
286        let detection = disambiguation.get("detection");
287        return Some(DisambiguationEvidence {
288            status,
289            ambiguity_type: detection.and_then(|d| d.get("type")).map(|v| v.to_string()),
290            confidence: detection
291                .and_then(|d| d.get("confidence"))
292                .and_then(Value::as_f64)
293                .map(|v| v as f32),
294            resolved: disambiguation.get("resolved").cloned(),
295        });
296    }
297
298    if context
299        .pointer("/disambiguation/resolved")
300        .and_then(Value::as_bool)
301        .unwrap_or(false)
302    {
303        return Some(DisambiguationEvidence {
304            status: DisambiguationStatus::Clarified,
305            ambiguity_type: None,
306            confidence: None,
307            resolved: context.get("disambiguation").cloned(),
308        });
309    }
310
311    None
312}
313
314fn infer_skill(
315    metadata: Option<&Value>,
316    disambiguation: Option<&DisambiguationEvidence>,
317) -> Option<SkillEvidence> {
318    let skill_id = metadata
319        .and_then(|m| m.get("skill_id"))
320        .and_then(Value::as_str)
321        .map(str::to_string)
322        .or_else(|| {
323            metadata
324                .and_then(|m| m.get("disambiguation"))
325                .and_then(|d| d.get("skill_id"))
326                .and_then(Value::as_str)
327                .map(str::to_string)
328        });
329
330    if skill_id.is_none() && disambiguation.is_none() {
331        return None;
332    }
333
334    Some(SkillEvidence {
335        selected_skill_id: skill_id.clone(),
336        executed_skill_id: skill_id,
337        no_match: false,
338        clarification_requested: disambiguation
339            .map(|d| d.status == DisambiguationStatus::Triggered)
340            .unwrap_or(false),
341    })
342}
343
344fn collect_relationship(
345    agent: &RuntimeAgent,
346    actor_id: Option<String>,
347    before: Option<Value>,
348) -> Option<RelationshipEvidence> {
349    let actor_id = actor_id?;
350    let manager = agent.relationship_manager()?;
351    let current = manager.relationship_as_value(&actor_id).ok().flatten();
352    let model = current
353        .as_ref()
354        .and_then(|value| value.get("model"))
355        .and_then(Value::as_str)
356        .map(str::to_string);
357    let mut available = vec!["agent_to_actor".to_string(), "mutual".to_string()];
358    if model.as_deref() == Some("two_sided") {
359        available.push("perceived_actor_to_agent".to_string());
360    }
361    Some(RelationshipEvidence {
362        actor_id: Some(actor_id),
363        model,
364        available_perspectives: available,
365        before,
366        after: current.clone(),
367        current,
368    })
369}
370
371fn collect_persona(
372    agent: &RuntimeAgent,
373    context_map: &HashMap<String, Value>,
374) -> Option<PersonaEvidence> {
375    let manager = agent.persona_manager()?;
376    let revealed_count = manager.revealed_secrets(context_map).len();
377    Some(PersonaEvidence {
378        secret_revealed: revealed_count > 0,
379        revealed_secret_ids: Vec::new(),
380        revealed_secret_count: revealed_count,
381        evolution_events: manager.history().len(),
382    })
383}
ai_agents_eval/evidence.rs

ai_agents_eval/
evidence.rs
