adk_agent/team/
evaluation.rs

1use std::collections::{BTreeSet, HashSet};
2
3use schemars::JsonSchema;
4use serde::{Deserialize, Serialize};
5use thiserror::Error;
6
7use super::{RelationshipKind, TeamExecutionSnapshot, TeamExecutionStatus, TeamSpec};
8
9/// Provider-free evaluation summary derived from a team execution receipt.
10#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
11#[serde(rename_all = "camelCase")]
12pub struct TeamExecutionAnalysis {
13    /// Number of exact relationships declared by the specification.
14    pub declared_relationships: usize,
15    /// Number of distinct declared relationships observed in the receipt.
16    pub covered_relationships: usize,
17    /// Relationship coverage in basis points (10,000 = 100%).
18    pub coverage_basis_points: u32,
19    /// Delegation executions observed, including retries.
20    pub delegations: usize,
21    /// Handoff executions observed, including retries.
22    pub handoffs: usize,
23    /// Failed relationship executions.
24    pub failed_edges: usize,
25    /// Maximum causal relationship depth in the receipt.
26    pub max_causal_depth: usize,
27    /// Declared exact relationships that were not exercised.
28    pub uncovered: Vec<String>,
29}
30
31/// Integrity failure found while validating an execution receipt for replay/evaluation.
32#[derive(Debug, Error, Clone, PartialEq, Eq)]
33pub enum TeamReplayError {
34    /// Receipt belongs to a different team.
35    #[error("receipt belongs to team '{actual}', expected '{expected}'")]
36    TeamMismatch {
37        /// Expected team.
38        expected: String,
39        /// Receipt team.
40        actual: String,
41    },
42    /// Relationship execution identifier is duplicated.
43    #[error("duplicate relationship execution id '{0}'")]
44    DuplicateEdgeId(String),
45    /// Receipt contains an edge not declared by the spec.
46    #[error("receipt contains undeclared {kind:?} edge from '{from}' to '{to}'")]
47    UndeclaredEdge {
48        /// Source member.
49        from: String,
50        /// Target member.
51        to: String,
52        /// Relationship semantics.
53        kind: RelationshipKind,
54    },
55    /// A causal parent is absent or appears after its child.
56    #[error("edge '{edge}' references unavailable causal parent '{parent}'")]
57    InvalidParent {
58        /// Child execution.
59        edge: String,
60        /// Invalid parent execution.
61        parent: String,
62    },
63    /// A terminal timestamp precedes its start.
64    #[error("edge '{0}' finishes before it starts")]
65    InvalidTimestamp(String),
66}
67
68/// Validates receipt integrity against an immutable [`TeamSpec`].
69pub fn validate_team_replay(
70    spec: &TeamSpec,
71    snapshot: &TeamExecutionSnapshot,
72) -> std::result::Result<(), TeamReplayError> {
73    if snapshot.team != spec.name {
74        return Err(TeamReplayError::TeamMismatch {
75            expected: spec.name.clone(),
76            actual: snapshot.team.clone(),
77        });
78    }
79    let declared: BTreeSet<_> = spec
80        .relationships
81        .iter()
82        .map(|edge| (edge.from.as_str(), edge.to.as_str(), edge.kind))
83        .collect();
84    let mut seen = HashSet::new();
85    for edge in &snapshot.edges {
86        if !seen.insert(edge.id.as_str()) {
87            return Err(TeamReplayError::DuplicateEdgeId(edge.id.clone()));
88        }
89        if !declared.contains(&(edge.from.as_str(), edge.to.as_str(), edge.kind)) {
90            return Err(TeamReplayError::UndeclaredEdge {
91                from: edge.from.clone(),
92                to: edge.to.clone(),
93                kind: edge.kind,
94            });
95        }
96        if let Some(parent) = &edge.parent_id
97            && !seen.contains(parent.as_str())
98        {
99            return Err(TeamReplayError::InvalidParent {
100                edge: edge.id.clone(),
101                parent: parent.clone(),
102            });
103        }
104        if edge.finished_at_ms.is_some_and(|finished| finished < edge.started_at_ms) {
105            return Err(TeamReplayError::InvalidTimestamp(edge.id.clone()));
106        }
107    }
108    Ok(())
109}
110
111/// Computes deterministic topology coverage and failure metrics.
112pub fn analyze_team_execution(
113    spec: &TeamSpec,
114    snapshot: &TeamExecutionSnapshot,
115) -> TeamExecutionAnalysis {
116    let declared: BTreeSet<_> = spec
117        .relationships
118        .iter()
119        .map(|edge| (edge.from.clone(), edge.to.clone(), edge.kind))
120        .collect();
121    let covered: BTreeSet<_> =
122        snapshot.edges.iter().map(|edge| (edge.from.clone(), edge.to.clone(), edge.kind)).collect();
123    let covered_count = declared.intersection(&covered).count();
124    let coverage_basis_points = if declared.is_empty() {
125        10_000
126    } else {
127        u32::try_from(covered_count.saturating_mul(10_000) / declared.len()).unwrap_or(10_000)
128    };
129    let ids: std::collections::HashMap<_, _> =
130        snapshot.edges.iter().map(|edge| (edge.id.as_str(), edge.parent_id.as_deref())).collect();
131    let max_causal_depth = snapshot
132        .edges
133        .iter()
134        .map(|edge| {
135            let mut depth = 1;
136            let mut parent = edge.parent_id.as_deref();
137            let mut guard = HashSet::new();
138            while let Some(id) = parent {
139                if !guard.insert(id) {
140                    break;
141                }
142                depth += 1;
143                parent = ids.get(id).copied().flatten();
144            }
145            depth
146        })
147        .max()
148        .unwrap_or(0);
149    TeamExecutionAnalysis {
150        declared_relationships: declared.len(),
151        covered_relationships: covered_count,
152        coverage_basis_points,
153        delegations: snapshot
154            .edges
155            .iter()
156            .filter(|edge| edge.kind == RelationshipKind::Delegate)
157            .count(),
158        handoffs: snapshot
159            .edges
160            .iter()
161            .filter(|edge| edge.kind == RelationshipKind::Handoff)
162            .count(),
163        failed_edges: snapshot
164            .edges
165            .iter()
166            .filter(|edge| edge.status == TeamExecutionStatus::Failed)
167            .count(),
168        max_causal_depth,
169        uncovered: declared
170            .difference(&covered)
171            .map(|(from, to, kind)| format!("{from} -{kind:?}-> {to}"))
172            .collect(),
173    }
174}
adk_agent/team/evaluation.rs

adk_agent/team/
evaluation.rs
