assay_core/otel/
mod.rs

1use crate::model::TestResultRow;
2
3pub mod genai;
4pub mod metrics;
5pub mod projection;
6pub mod redaction;
7pub mod semconv;
8
9#[derive(Debug, Clone, Default)]
10pub struct OTelConfig {
11    pub jsonl_path: Option<std::path::PathBuf>,
12    pub redact_prompts: bool,
13}
14
15pub fn export_jsonl(
16    cfg: &OTelConfig,
17    _suite: &str,
18    results: &[TestResultRow],
19) -> anyhow::Result<()> {
20    let Some(path) = &cfg.jsonl_path else {
21        return Ok(());
22    };
23    let mut f = std::fs::OpenOptions::new()
24        .create(true)
25        .append(true)
26        .open(path)?;
27    for r in results {
28        // GenAI Semantic Conventions (simplified for MVP)
29        // https://opentelemetry.io/docs/specs/semconv/gen-ai/
30        let row = serde_json::json!({
31            "timestamp": chrono::Utc::now().to_rfc3339(),
32            "attributes": {
33                "gen_ai.system": "assay",
34                "gen_ai.request.model": "unknown", // can be enriched if we track it better
35                "gen_ai.response.completion_tokens": 0, // placeholder
36                "assay.test_id": r.test_id,
37                "assay.status": format!("{:?}", r.status),
38                "assay.score": r.score,
39                "assay.cached": r.cached,
40                "assay.duration_ms": r.duration_ms,
41            }
42        });
43
44        // Use details/meta if available to populate standard fields
45        // checking details logic would go here
46
47        use std::io::Write;
48        writeln!(f, "{}", row)?;
49    }
50    Ok(())
51}
52
53/// A single observed tool effect to emit as an OTel GenAI `execute_tool` span,
54/// carrying the Assay claim-class outcome (the claimed-versus-actual surface).
55#[derive(Debug, Clone)]
56pub struct ToolObservation {
57    /// Tool / effect name (e.g. an MCP tool name or a sandbox effect kind).
58    pub tool_name: String,
59    /// Assay claim-class outcome: `supported` | `degraded` | `blocked` | `not_evaluable`.
60    pub claim_class_outcome: String,
61    /// Optional subject (e.g. a path or resource).
62    pub subject: Option<String>,
63}
64
65/// Emit observed tool effects as OTel GenAI `execute_tool` spans in the
66/// semconv-shaped JSONL collector format (the same pattern as [`export_jsonl`]),
67/// each carrying the Assay claim-class outcome as an attribute. Pinned to GenAI
68/// semconv 1.28.0. A no-op unless `cfg.jsonl_path` is set.
69///
70/// This is the emit side of the claimed-versus-actual surface: a downstream OTel
71/// collector ingests these spans alongside the agent's self-reported spans, so a
72/// consumer can compare declared behavior against the independently observed
73/// effect and the claim it actually supports.
74pub fn export_tool_spans_jsonl(
75    cfg: &OTelConfig,
76    run: &str,
77    observations: &[ToolObservation],
78) -> anyhow::Result<()> {
79    let Some(path) = &cfg.jsonl_path else {
80        return Ok(());
81    };
82    let mut f = std::fs::OpenOptions::new()
83        .create(true)
84        .append(true)
85        .open(path)?;
86    use std::io::Write;
87    for (seq, obs) in observations.iter().enumerate() {
88        // OTel GenAI execute-tool span (semconv 1.28.0), plus the assay claim-class
89        // outcome as a vendor extension attribute.
90        let row = serde_json::json!({
91            "timestamp": chrono::Utc::now().to_rfc3339(),
92            "name": "execute_tool",
93            "attributes": {
94                "gen_ai.system": "assay",
95                "gen_ai.operation.name": "execute_tool",
96                "gen_ai.tool.name": obs.tool_name,
97                "assay.claim_class.outcome": obs.claim_class_outcome,
98                "assay.run": run,
99                "assay.seq": seq,
100                "assay.subject": obs.subject,
101            },
102        });
103        writeln!(f, "{}", row)?;
104    }
105    Ok(())
106}
107
108#[cfg(test)]
109mod tool_span_tests {
110    use super::*;
111
112    #[test]
113    fn export_tool_spans_writes_execute_tool_rows_with_claim_class() {
114        let path = std::env::temp_dir().join(format!(
115            "assay-otel-tool-spans-{}.jsonl",
116            std::process::id()
117        ));
118        let _ = std::fs::remove_file(&path);
119        let cfg = OTelConfig {
120            jsonl_path: Some(path.clone()),
121            redact_prompts: false,
122        };
123        let observations = vec![
124            ToolObservation {
125                tool_name: "fs.write".into(),
126                claim_class_outcome: "supported".into(),
127                subject: Some("/tmp/out.txt".into()),
128            },
129            ToolObservation {
130                tool_name: "net.connect".into(),
131                claim_class_outcome: "blocked".into(),
132                subject: None,
133            },
134        ];
135
136        export_tool_spans_jsonl(&cfg, "sandbox_testrun", &observations).expect("export");
137
138        let body = std::fs::read_to_string(&path).expect("read jsonl");
139        let lines: Vec<&str> = body.lines().collect();
140        assert_eq!(lines.len(), 2);
141        let first: serde_json::Value = serde_json::from_str(lines[0]).unwrap();
142        assert_eq!(first["name"], "execute_tool");
143        assert_eq!(first["attributes"]["gen_ai.operation.name"], "execute_tool");
144        assert_eq!(first["attributes"]["gen_ai.tool.name"], "fs.write");
145        assert_eq!(
146            first["attributes"]["assay.claim_class.outcome"],
147            "supported"
148        );
149        let second: serde_json::Value = serde_json::from_str(lines[1]).unwrap();
150        assert_eq!(second["attributes"]["assay.claim_class.outcome"], "blocked");
151
152        std::fs::remove_file(&path).ok();
153    }
154}
assay_core/otel/mod.rs

assay_core/otel/
mod.rs
