lc_callbacks/tracing/
backend.rs

1use std::sync::Mutex;
2
3#[cfg(feature = "opentelemetry")]
4use std::collections::HashMap;
5
6#[cfg(feature = "opentelemetry")]
7use crate::semconv;
8
9use super::span::{build_tree, TraceSpan};
10use super::TracingBackend;
11
12// ---------------------------------------------------------------------------
13// In-memory backend
14// ---------------------------------------------------------------------------
15
16/// In-memory tracing backend for development and testing.
17pub struct InMemoryTracingBackend {
18    spans: Mutex<Vec<TraceSpan>>,
19}
20
21impl InMemoryTracingBackend {
22    /// Create a new empty in-memory backend.
23    pub fn new() -> Self {
24        Self {
25            spans: Mutex::new(Vec::new()),
26        }
27    }
28
29    /// Return a snapshot of all recorded spans.
30    pub fn spans(&self) -> Vec<TraceSpan> {
31        self.spans.lock().unwrap_or_else(|e| e.into_inner()).clone()
32    }
33
34    /// Remove all recorded spans.
35    pub fn clear(&self) {
36        self.spans.lock().unwrap_or_else(|e| e.into_inner()).clear();
37    }
38
39    /// Get the full trace tree rooted at `root_id`.
40    ///
41    /// Returns `None` if no span with that ID exists.
42    pub fn trace_tree(&self, root_id: &str) -> Option<super::TraceNode> {
43        let spans = self.spans.lock().unwrap_or_else(|e| e.into_inner());
44        let root = spans.iter().find(|s| s.id == root_id)?;
45        Some(build_tree(root, &spans))
46    }
47}
48
49impl Default for InMemoryTracingBackend {
50    fn default() -> Self {
51        Self::new()
52    }
53}
54
55impl TracingBackend for InMemoryTracingBackend {
56    fn start_span(&self, span: &TraceSpan) {
57        self.spans
58            .lock()
59            .unwrap_or_else(|e| e.into_inner())
60            .push(span.clone());
61    }
62
63    fn end_span(&self, span: &TraceSpan) {
64        let mut spans = self.spans.lock().unwrap_or_else(|e| e.into_inner());
65        if let Some(existing) = spans.iter_mut().find(|s| s.id == span.id) {
66            *existing = span.clone();
67        }
68    }
69
70    fn flush(&self) {
71        // In-memory backend has nothing to flush.
72    }
73}
74
75// ---------------------------------------------------------------------------
76// Console / logging backend
77// ---------------------------------------------------------------------------
78
79/// Console logging backend that prints span lifecycle events.
80pub struct ConsoleTracingBackend;
81
82impl TracingBackend for ConsoleTracingBackend {
83    fn start_span(&self, span: &TraceSpan) {
84        println!("[TRACE START] {} ({})", span.name, span.kind);
85    }
86
87    fn end_span(&self, span: &TraceSpan) {
88        let latency = span.latency_ms.unwrap_or(0);
89        let status_str = match &span.status {
90            super::SpanStatus::Ok => "OK".to_string(),
91            super::SpanStatus::Error(e) => format!("ERROR: {}", e),
92        };
93        println!(
94            "[TRACE END]   {} latency={}ms status={}",
95            span.name, latency, status_str
96        );
97    }
98
99    fn flush(&self) {}
100}
101
102// ---------------------------------------------------------------------------
103// OpenTelemetry backend (feature-gated)
104// ---------------------------------------------------------------------------
105
106/// OpenTelemetry tracing backend (requires `opentelemetry` feature).
107///
108/// Converts framework trace spans into OTel spans via the global tracer,
109/// emitting the stabilized GenAI semantic-convention attributes carried on
110/// [`TraceSpan`] (same vocabulary as `OtelHandler`). Parent spans are linked
111/// through the framework `parent_id`, so the exported trace topology matches
112/// the in-process trace tree.
113#[cfg(feature = "opentelemetry")]
114pub struct OtelTracingBackend {
115    tracer: opentelemetry::global::BoxedTracer,
116    /// Active OTel spans keyed by framework span ID.
117    spans: Mutex<HashMap<String, opentelemetry::global::BoxedSpan>>,
118}
119
120#[cfg(feature = "opentelemetry")]
121impl OtelTracingBackend {
122    /// Create a new backend with the given tracer.
123    pub fn new(tracer: opentelemetry::global::BoxedTracer) -> Self {
124        Self {
125            tracer,
126            spans: Mutex::new(HashMap::new()),
127        }
128    }
129
130    /// Create a backend using the global tracer provider.
131    pub fn from_global(name: &str) -> Self {
132        Self::new(opentelemetry::global::tracer(name.to_string()))
133    }
134}
135
136#[cfg(feature = "opentelemetry")]
137impl TracingBackend for OtelTracingBackend {
138    fn start_span(&self, span: &TraceSpan) {
139        use opentelemetry::trace::{Span as _, TraceContextExt, Tracer as _};
140        use opentelemetry::{Context, KeyValue};
141
142        // Establish the OTel parent from the framework parent_id, cloning the
143        // SpanContext while the lock is held (matching OtelHandler).
144        let parent_cx = {
145            let spans = self.spans.lock().unwrap_or_else(|e| e.into_inner());
146            span.parent_id
147                .as_ref()
148                .and_then(|pid| spans.get(pid))
149                .map(|parent| {
150                    Context::new().with_remote_span_context(parent.span_context().clone())
151                })
152        };
153
154        let name = semconv_span_name(span);
155        let mut otel_span = match parent_cx {
156            Some(cx) => self.tracer.start_with_context(name, &cx),
157            None => self.tracer.start(name),
158        };
159
160        if let Some(op) = &span.gen_ai_operation_name {
161            otel_span.set_attribute(KeyValue::new(semconv::GEN_AI_OPERATION_NAME, op.clone()));
162        }
163        if let Some(system) = &span.gen_ai_system {
164            otel_span.set_attribute(KeyValue::new(semconv::GEN_AI_PROVIDER_NAME, system.clone()));
165        }
166        if let Some(model) = &span.gen_ai_request_model {
167            otel_span.set_attribute(KeyValue::new(semconv::GEN_AI_REQUEST_MODEL, model.clone()));
168        }
169        if let Some(max) = span.gen_ai_request_max_tokens {
170            otel_span.set_attribute(KeyValue::new(
171                semconv::GEN_AI_REQUEST_MAX_TOKENS,
172                max as i64,
173            ));
174        }
175        if let Some(temp) = span.gen_ai_request_temperature {
176            otel_span.set_attribute(KeyValue::new(semconv::GEN_AI_REQUEST_TEMPERATURE, temp));
177        }
178        if let Some(tool) = &span.gen_ai_tool_name {
179            otel_span.set_attribute(KeyValue::new(semconv::GEN_AI_TOOL_NAME, tool.clone()));
180        }
181        // Eval/trace join keys when the caller stamped them on metadata.
182        if let Some(obj) = span.metadata.as_object() {
183            for (key, attr) in [
184                ("run_id", semconv::RUN_ID_ATTR),
185                ("trace_id", semconv::TRACE_ID_ATTR),
186            ] {
187                if let Some(v) = obj.get(key).and_then(|v| v.as_str()) {
188                    otel_span.set_attribute(KeyValue::new(attr, v.to_string()));
189                }
190            }
191        }
192
193        self.spans
194            .lock()
195            .unwrap_or_else(|e| e.into_inner())
196            .insert(span.id.clone(), otel_span);
197    }
198
199    fn end_span(&self, span: &TraceSpan) {
200        use opentelemetry::trace::{Span as _, Status};
201        use opentelemetry::{Array, KeyValue, StringValue, Value};
202
203        let mut spans = self.spans.lock().unwrap_or_else(|e| e.into_inner());
204        if let Some(mut s) = spans.remove(&span.id) {
205            if let Some(model) = &span.gen_ai_response_model {
206                s.set_attribute(KeyValue::new(semconv::GEN_AI_RESPONSE_MODEL, model.clone()));
207            }
208            if let Some(tokens) = &span.tokens {
209                s.set_attribute(KeyValue::new(
210                    semconv::GEN_AI_USAGE_INPUT_TOKENS,
211                    tokens.prompt_tokens as i64,
212                ));
213                s.set_attribute(KeyValue::new(
214                    semconv::GEN_AI_USAGE_OUTPUT_TOKENS,
215                    tokens.completion_tokens as i64,
216                ));
217                // T10: Development-stability usage extensions, mirroring
218                // OtelHandler's provider-token attribution (the two OTel
219                // surfaces share these names via `semconv`).
220                for (value, attr) in [
221                    (
222                        tokens.cache_read_input_tokens,
223                        semconv::GEN_AI_USAGE_CACHE_READ,
224                    ),
225                    (
226                        tokens.cache_write_input_tokens,
227                        semconv::GEN_AI_USAGE_CACHE_WRITE,
228                    ),
229                    (
230                        tokens.reasoning_output_tokens,
231                        semconv::GEN_AI_USAGE_REASONING,
232                    ),
233                ] {
234                    if let Some(n) = value {
235                        s.set_attribute(KeyValue::new(attr, n as i64));
236                    }
237                }
238            }
239            if let Some(reason) = &span.gen_ai_finish_reason {
240                s.set_attribute(KeyValue::new(
241                    semconv::GEN_AI_RESPONSE_FINISH_REASONS,
242                    Value::Array(Array::String(vec![StringValue::from(reason.clone())])),
243                ));
244            }
245            if let crate::SpanStatus::Error(error) = &span.status {
246                s.set_status(Status::error(error.clone()));
247                s.set_attribute(KeyValue::new(
248                    semconv::ERROR_TYPE,
249                    crate::semconv::error_type(error),
250                ));
251                s.add_event(
252                    "exception".to_string(),
253                    vec![KeyValue::new(
254                        "exception.message",
255                        crate::semconv::truncate(error, 1024),
256                    )],
257                );
258            }
259            s.end();
260        }
261    }
262
263    fn flush(&self) {}
264}
265
266/// Picks the semconv span name for a trace span: `chat {model}` for LLM
267/// spans, `execute_tool {tool}` for tool spans; falls back to the recorded
268/// name.
269#[cfg(feature = "opentelemetry")]
270fn semconv_span_name(span: &TraceSpan) -> String {
271    use super::span::SpanKind;
272    match &span.kind {
273        SpanKind::Llm => match &span.gen_ai_request_model {
274            Some(model) => format!("chat {model}"),
275            None => "chat".to_string(),
276        },
277        crate::SpanKind::Tool => match &span.gen_ai_tool_name {
278            Some(tool) => format!("execute_tool {tool}"),
279            None => span.name.clone(),
280        },
281        _ => span.name.clone(),
282    }
283}
284
285#[cfg(all(test, feature = "opentelemetry"))]
286mod otel_backend_tests {
287    use super::*;
288    use crate::semconv;
289    use crate::tracing::span::make_span;
290    use opentelemetry::trace::TracerProvider as _;
291    use opentelemetry::Value;
292    use opentelemetry_sdk::testing::trace::InMemorySpanExporterBuilder;
293    use opentelemetry_sdk::trace::{SimpleSpanProcessor, TracerProvider};
294
295    fn backend_with_exporter() -> (
296        OtelTracingBackend,
297        opentelemetry_sdk::testing::trace::InMemorySpanExporter,
298    ) {
299        let exporter = InMemorySpanExporterBuilder::new().build();
300        let provider = TracerProvider::builder()
301            .with_span_processor(SimpleSpanProcessor::new(Box::new(exporter.clone())))
302            .build();
303        let tracer: opentelemetry::global::BoxedTracer =
304            opentelemetry::global::BoxedTracer::new(Box::new(provider.tracer("test")));
305        (OtelTracingBackend::new(tracer), exporter)
306    }
307
308    fn attr_str(span: &opentelemetry_sdk::export::trace::SpanData, key: &str) -> Option<String> {
309        span.attributes
310            .iter()
311            .find(|kv| kv.key.as_str() == key)
312            .map(|kv| kv.value.as_str().into_owned())
313    }
314
315    #[test]
316    fn llm_span_emits_semconv_name_and_request_attributes() {
317        let (backend, exporter) = backend_with_exporter();
318        let mut span = make_span(
319            "root".into(),
320            None,
321            "gpt-4o-mini call",
322            crate::SpanKind::Llm,
323        );
324        span.gen_ai_system = Some("openai".into());
325        span.gen_ai_request_model = Some("gpt-4o-mini".into());
326        span.gen_ai_operation_name = Some("chat".into());
327        span.gen_ai_request_temperature = Some(0.1);
328        span.gen_ai_request_max_tokens = Some(256);
329        span.metadata = serde_json::json!({"run_id": "run-42", "trace_id": "trace-7"});
330
331        backend.start_span(&span);
332        backend.end_span(&span);
333
334        let spans = exporter.get_finished_spans().unwrap();
335        assert_eq!(spans.len(), 1);
336        let exported = &spans[0];
337        assert_eq!(exported.name, "chat gpt-4o-mini");
338        assert_eq!(
339            attr_str(exported, semconv::GEN_AI_PROVIDER_NAME).as_deref(),
340            Some("openai")
341        );
342        assert_eq!(
343            attr_str(exported, semconv::RUN_ID_ATTR).as_deref(),
344            Some("run-42")
345        );
346        assert_eq!(
347            attr_str(exported, semconv::TRACE_ID_ATTR).as_deref(),
348            Some("trace-7")
349        );
350        assert!(exported
351            .attributes
352            .iter()
353            .any(|kv| kv.key.as_str() == semconv::GEN_AI_REQUEST_TEMPERATURE
354                && matches!(kv.value, Value::F64(_))));
355    }
356
357    #[test]
358    fn end_span_emits_usage_finish_reasons_and_error_status() {
359        let (backend, exporter) = backend_with_exporter();
360        let mut span = make_span("t".into(), None, "tool call", crate::SpanKind::Tool);
361        span.gen_ai_tool_name = Some("calculator".into());
362        span.gen_ai_response_model = Some("gpt-4o-mini-2024-07-18".into());
363        span.gen_ai_finish_reason = Some("tool_calls".into());
364        span.tokens = Some(crate::SpanTokenUsage {
365            prompt_tokens: 10,
366            completion_tokens: 4,
367            total_tokens: 14,
368            ..Default::default()
369        });
370        span.status = crate::SpanStatus::Error("TimeoutError: 30s".into());
371
372        backend.start_span(&span);
373        backend.end_span(&span);
374
375        let spans = exporter.get_finished_spans().unwrap();
376        let exported = &spans[0];
377        assert_eq!(exported.name, "execute_tool calculator");
378        assert_eq!(
379            attr_str(exported, semconv::GEN_AI_RESPONSE_MODEL).as_deref(),
380            Some("gpt-4o-mini-2024-07-18")
381        );
382        let reasons = exported
383            .attributes
384            .iter()
385            .find(|kv| kv.key.as_str() == semconv::GEN_AI_RESPONSE_FINISH_REASONS)
386            .map(|kv| match &kv.value {
387                opentelemetry::Value::Array(opentelemetry::Array::String(values)) => values
388                    .iter()
389                    .map(|v| v.as_str().to_string())
390                    .collect::<Vec<_>>(),
391                _ => vec![],
392            })
393            .unwrap();
394        assert_eq!(reasons, vec!["tool_calls".to_string()]);
395        assert!(matches!(
396            exported.status,
397            opentelemetry::trace::Status::Error { .. }
398        ));
399        assert_eq!(
400            attr_str(exported, semconv::ERROR_TYPE).as_deref(),
401            Some("TimeoutError")
402        );
403        assert!(exported.events.iter().any(|e| e.name == "exception"));
404    }
405
406    #[test]
407    fn end_span_emits_cache_and_reasoning_usage_extensions() {
408        // T10: the tracing OTel backend must attribute the same Development
409        // extension usage names as OtelHandler (no drift between surfaces).
410        let (backend, exporter) = backend_with_exporter();
411        let mut span = make_span("t".into(), None, "chat", crate::SpanKind::Llm);
412        span.tokens = Some(crate::SpanTokenUsage {
413            prompt_tokens: 100,
414            completion_tokens: 20,
415            total_tokens: 120,
416            cache_read_input_tokens: Some(80),
417            cache_write_input_tokens: Some(10),
418            reasoning_output_tokens: Some(5),
419        });
420
421        backend.start_span(&span);
422        backend.end_span(&span);
423
424        let spans = exporter.get_finished_spans().unwrap();
425        let exported = &spans[0];
426        assert_eq!(
427            exported
428                .attributes
429                .iter()
430                .find(|kv| kv.key.as_str() == semconv::GEN_AI_USAGE_CACHE_READ)
431                .and_then(|kv| match kv.value {
432                    opentelemetry::Value::I64(v) => Some(v),
433                    _ => None,
434                }),
435            Some(80)
436        );
437        assert_eq!(
438            exported
439                .attributes
440                .iter()
441                .find(|kv| kv.key.as_str() == semconv::GEN_AI_USAGE_CACHE_WRITE)
442                .and_then(|kv| match kv.value {
443                    opentelemetry::Value::I64(v) => Some(v),
444                    _ => None,
445                }),
446            Some(10)
447        );
448        assert_eq!(
449            exported
450                .attributes
451                .iter()
452                .find(|kv| kv.key.as_str() == semconv::GEN_AI_USAGE_REASONING)
453                .and_then(|kv| match kv.value {
454                    opentelemetry::Value::I64(v) => Some(v),
455                    _ => None,
456                }),
457            Some(5)
458        );
459    }
460
461    #[test]
462    fn child_span_links_to_parent_context() {
463        let (backend, exporter) = backend_with_exporter();
464        let root = make_span("root".into(), None, "root", crate::SpanKind::Chain);
465        let mut child = make_span(
466            "child".into(),
467            Some("root".into()),
468            "child",
469            crate::SpanKind::Llm,
470        );
471        // LLM spans are renamed `chat {model}`.
472        child.gen_ai_request_model = Some("mini".into());
473
474        backend.start_span(&root);
475        backend.start_span(&child);
476        backend.end_span(&child);
477        backend.end_span(&root);
478
479        let spans = exporter.get_finished_spans().unwrap();
480        assert_eq!(spans.len(), 2);
481        let root_ctx = spans
482            .iter()
483            .find(|s| s.name == "root")
484            .unwrap()
485            .span_context
486            .clone();
487        let child_data = spans.iter().find(|s| s.name == "chat mini").unwrap();
488        assert_eq!(child_data.parent_span_id, root_ctx.span_id());
489        assert_eq!(child_data.span_context.trace_id(), root_ctx.trace_id());
490    }
491}
lc_callbacks/tracing/backend.rs

lc_callbacks/tracing/
backend.rs
