assay_core/otel/
mod.rs

1use crate::model::TestResultRow;
2
3pub mod genai;
4pub(crate) mod mcp_ingest;
5pub mod metrics;
6pub mod projection;
7pub mod redaction;
8pub mod semconv;
9
10#[derive(Debug, Clone, Default)]
11pub struct OTelConfig {
12    pub jsonl_path: Option<std::path::PathBuf>,
13    pub redact_prompts: bool,
14}
15
16pub fn export_jsonl(
17    cfg: &OTelConfig,
18    _suite: &str,
19    results: &[TestResultRow],
20) -> anyhow::Result<()> {
21    let Some(path) = &cfg.jsonl_path else {
22        return Ok(());
23    };
24    let mut f = std::fs::OpenOptions::new()
25        .create(true)
26        .append(true)
27        .open(path)?;
28    for r in results {
29        // GenAI Semantic Conventions (simplified for MVP)
30        // https://opentelemetry.io/docs/specs/semconv/gen-ai/
31        let row = serde_json::json!({
32            "timestamp": chrono::Utc::now().to_rfc3339(),
33            "attributes": {
34                "gen_ai.system": "assay",
35                "gen_ai.request.model": "unknown", // can be enriched if we track it better
36                "gen_ai.response.completion_tokens": 0, // placeholder
37                "assay.test_id": r.test_id,
38                "assay.status": format!("{:?}", r.status),
39                "assay.score": r.score,
40                "assay.cached": r.cached,
41                "assay.duration_ms": r.duration_ms,
42            }
43        });
44
45        // Use details/meta if available to populate standard fields
46        // checking details logic would go here
47
48        use std::io::Write;
49        writeln!(f, "{}", row)?;
50    }
51    Ok(())
52}
53
54/// A single observed tool effect to emit as an OTel GenAI `execute_tool` span,
55/// carrying the Assay claim-class outcome (the claimed-versus-actual surface).
56#[derive(Debug, Clone)]
57pub struct ToolObservation {
58    /// Tool / effect name (e.g. an MCP tool name or a sandbox effect kind).
59    pub tool_name: String,
60    /// Assay claim-class outcome: `supported` | `degraded` | `blocked` | `not_evaluable`.
61    pub claim_class_outcome: String,
62    /// Optional subject (e.g. a path or resource).
63    pub subject: Option<String>,
64}
65
66/// Emit observed tool effects as OTel GenAI `execute_tool` spans in the
67/// semconv-shaped JSONL collector format (the same pattern as [`export_jsonl`]),
68/// each carrying the Assay claim-class outcome as an attribute. Pinned to GenAI
69/// semconv 1.28.0. A no-op unless `cfg.jsonl_path` is set.
70///
71/// This is the emit side of the claimed-versus-actual surface: a downstream OTel
72/// collector ingests these spans alongside the agent's self-reported spans, so a
73/// consumer can compare declared behavior against the independently observed
74/// effect and the claim it actually supports.
75pub fn export_tool_spans_jsonl(
76    cfg: &OTelConfig,
77    run: &str,
78    observations: &[ToolObservation],
79) -> anyhow::Result<()> {
80    let Some(path) = &cfg.jsonl_path else {
81        return Ok(());
82    };
83    let mut f = std::fs::OpenOptions::new()
84        .create(true)
85        .append(true)
86        .open(path)?;
87    use std::io::Write;
88    for (seq, obs) in observations.iter().enumerate() {
89        // OTel GenAI execute-tool span (semconv 1.28.0), plus the assay claim-class
90        // outcome as a vendor extension attribute.
91        let row = serde_json::json!({
92            "timestamp": chrono::Utc::now().to_rfc3339(),
93            "name": "execute_tool",
94            "attributes": {
95                "gen_ai.system": "assay",
96                "gen_ai.operation.name": "execute_tool",
97                "gen_ai.tool.name": obs.tool_name,
98                "assay.claim_class.outcome": obs.claim_class_outcome,
99                "assay.run": run,
100                "assay.seq": seq,
101                "assay.subject": obs.subject,
102            },
103        });
104        writeln!(f, "{}", row)?;
105    }
106    Ok(())
107}
108
109#[cfg(test)]
110mod tool_span_tests {
111    use super::*;
112
113    #[test]
114    fn export_tool_spans_writes_execute_tool_rows_with_claim_class() {
115        let path = std::env::temp_dir().join(format!(
116            "assay-otel-tool-spans-{}.jsonl",
117            std::process::id()
118        ));
119        let _ = std::fs::remove_file(&path);
120        let cfg = OTelConfig {
121            jsonl_path: Some(path.clone()),
122            redact_prompts: false,
123        };
124        let observations = vec![
125            ToolObservation {
126                tool_name: "fs.write".into(),
127                claim_class_outcome: "supported".into(),
128                subject: Some("/tmp/out.txt".into()),
129            },
130            ToolObservation {
131                tool_name: "net.connect".into(),
132                claim_class_outcome: "blocked".into(),
133                subject: None,
134            },
135        ];
136
137        export_tool_spans_jsonl(&cfg, "sandbox_testrun", &observations).expect("export");
138
139        let body = std::fs::read_to_string(&path).expect("read jsonl");
140        let lines: Vec<&str> = body.lines().collect();
141        assert_eq!(lines.len(), 2);
142        let first: serde_json::Value = serde_json::from_str(lines[0]).unwrap();
143        assert_eq!(first["name"], "execute_tool");
144        assert_eq!(first["attributes"]["gen_ai.operation.name"], "execute_tool");
145        assert_eq!(first["attributes"]["gen_ai.tool.name"], "fs.write");
146        assert_eq!(
147            first["attributes"]["assay.claim_class.outcome"],
148            "supported"
149        );
150        let second: serde_json::Value = serde_json::from_str(lines[1]).unwrap();
151        assert_eq!(second["attributes"]["assay.claim_class.outcome"], "blocked");
152
153        std::fs::remove_file(&path).ok();
154    }
155}
assay_core/otel/mod.rs

assay_core/otel/
mod.rs
