assay_core/otel/
mod.rs

1use crate::model::TestResultRow;
2
3pub mod genai;
4pub mod metrics;
5pub mod redaction;
6pub mod semconv;
7
8#[derive(Debug, Clone, Default)]
9pub struct OTelConfig {
10    pub jsonl_path: Option<std::path::PathBuf>,
11    pub redact_prompts: bool,
12}
13
14pub fn export_jsonl(
15    cfg: &OTelConfig,
16    _suite: &str,
17    results: &[TestResultRow],
18) -> anyhow::Result<()> {
19    let Some(path) = &cfg.jsonl_path else {
20        return Ok(());
21    };
22    let mut f = std::fs::OpenOptions::new()
23        .create(true)
24        .append(true)
25        .open(path)?;
26    for r in results {
27        // GenAI Semantic Conventions (simplified for MVP)
28        // https://opentelemetry.io/docs/specs/semconv/gen-ai/
29        let row = serde_json::json!({
30            "timestamp": chrono::Utc::now().to_rfc3339(),
31            "attributes": {
32                "gen_ai.system": "assay",
33                "gen_ai.request.model": "unknown", // can be enriched if we track it better
34                "gen_ai.response.completion_tokens": 0, // placeholder
35                "assay.test_id": r.test_id,
36                "assay.status": format!("{:?}", r.status),
37                "assay.score": r.score,
38                "assay.cached": r.cached,
39                "assay.duration_ms": r.duration_ms,
40            }
41        });
42
43        // Use details/meta if available to populate standard fields
44        // checking details logic would go here
45
46        use std::io::Write;
47        writeln!(f, "{}", row)?;
48    }
49    Ok(())
50}
51
52/// A single observed tool effect to emit as an OTel GenAI `execute_tool` span,
53/// carrying the Assay claim-class outcome (the claimed-versus-actual surface).
54#[derive(Debug, Clone)]
55pub struct ToolObservation {
56    /// Tool / effect name (e.g. an MCP tool name or a sandbox effect kind).
57    pub tool_name: String,
58    /// Assay claim-class outcome: `supported` | `degraded` | `blocked` | `not_evaluable`.
59    pub claim_class_outcome: String,
60    /// Optional subject (e.g. a path or resource).
61    pub subject: Option<String>,
62}
63
64/// Emit observed tool effects as OTel GenAI `execute_tool` spans in the
65/// semconv-shaped JSONL collector format (the same pattern as [`export_jsonl`]),
66/// each carrying the Assay claim-class outcome as an attribute. Pinned to GenAI
67/// semconv 1.28.0. A no-op unless `cfg.jsonl_path` is set.
68///
69/// This is the emit side of the claimed-versus-actual surface: a downstream OTel
70/// collector ingests these spans alongside the agent's self-reported spans, so a
71/// consumer can compare declared behavior against the independently observed
72/// effect and the claim it actually supports.
73pub fn export_tool_spans_jsonl(
74    cfg: &OTelConfig,
75    run: &str,
76    observations: &[ToolObservation],
77) -> anyhow::Result<()> {
78    let Some(path) = &cfg.jsonl_path else {
79        return Ok(());
80    };
81    let mut f = std::fs::OpenOptions::new()
82        .create(true)
83        .append(true)
84        .open(path)?;
85    use std::io::Write;
86    for (seq, obs) in observations.iter().enumerate() {
87        // OTel GenAI execute-tool span (semconv 1.28.0), plus the assay claim-class
88        // outcome as a vendor extension attribute.
89        let row = serde_json::json!({
90            "timestamp": chrono::Utc::now().to_rfc3339(),
91            "name": "execute_tool",
92            "attributes": {
93                "gen_ai.system": "assay",
94                "gen_ai.operation.name": "execute_tool",
95                "gen_ai.tool.name": obs.tool_name,
96                "assay.claim_class.outcome": obs.claim_class_outcome,
97                "assay.run": run,
98                "assay.seq": seq,
99                "assay.subject": obs.subject,
100            },
101        });
102        writeln!(f, "{}", row)?;
103    }
104    Ok(())
105}
106
107#[cfg(test)]
108mod tool_span_tests {
109    use super::*;
110
111    #[test]
112    fn export_tool_spans_writes_execute_tool_rows_with_claim_class() {
113        let path = std::env::temp_dir().join(format!(
114            "assay-otel-tool-spans-{}.jsonl",
115            std::process::id()
116        ));
117        let _ = std::fs::remove_file(&path);
118        let cfg = OTelConfig {
119            jsonl_path: Some(path.clone()),
120            redact_prompts: false,
121        };
122        let observations = vec![
123            ToolObservation {
124                tool_name: "fs.write".into(),
125                claim_class_outcome: "supported".into(),
126                subject: Some("/tmp/out.txt".into()),
127            },
128            ToolObservation {
129                tool_name: "net.connect".into(),
130                claim_class_outcome: "blocked".into(),
131                subject: None,
132            },
133        ];
134
135        export_tool_spans_jsonl(&cfg, "sandbox_testrun", &observations).expect("export");
136
137        let body = std::fs::read_to_string(&path).expect("read jsonl");
138        let lines: Vec<&str> = body.lines().collect();
139        assert_eq!(lines.len(), 2);
140        let first: serde_json::Value = serde_json::from_str(lines[0]).unwrap();
141        assert_eq!(first["name"], "execute_tool");
142        assert_eq!(first["attributes"]["gen_ai.operation.name"], "execute_tool");
143        assert_eq!(first["attributes"]["gen_ai.tool.name"], "fs.write");
144        assert_eq!(
145            first["attributes"]["assay.claim_class.outcome"],
146            "supported"
147        );
148        let second: serde_json::Value = serde_json::from_str(lines[1]).unwrap();
149        assert_eq!(second["attributes"]["assay.claim_class.outcome"], "blocked");
150
151        std::fs::remove_file(&path).ok();
152    }
153}
assay_core/otel/mod.rs

assay_core/otel/
mod.rs
