use chrono::Utc;
use serde::{Deserialize, Serialize};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use uuid::Uuid;
#[cfg(feature = "opentelemetry")]
use opentelemetry::trace::Span as OtelSpan;
#[cfg(feature = "opentelemetry")]
use opentelemetry::trace::Tracer as OtelTracer;
pub type SpanId = String;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum SpanKind {
Llm,
Chain,
Tool,
Retriever,
Agent,
Custom(String),
}
impl std::fmt::Display for SpanKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SpanKind::Llm => write!(f, "llm"),
SpanKind::Chain => write!(f, "chain"),
SpanKind::Tool => write!(f, "tool"),
SpanKind::Retriever => write!(f, "retriever"),
SpanKind::Agent => write!(f, "agent"),
SpanKind::Custom(name) => write!(f, "custom:{}", name),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct SpanTokenUsage {
pub prompt_tokens: usize,
pub completion_tokens: usize,
pub total_tokens: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum SpanStatus {
Ok,
Error(String),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceSpan {
pub id: SpanId,
pub parent_id: Option<SpanId>,
pub name: String,
pub kind: SpanKind,
pub start_time: Option<String>,
pub end_time: Option<String>,
pub tokens: Option<SpanTokenUsage>,
pub cost: Option<f64>,
pub latency_ms: Option<u64>,
pub metadata: serde_json::Value,
pub status: SpanStatus,
}
pub trait TracingBackend: Send + Sync {
fn start_span(&self, span: &TraceSpan);
fn end_span(&self, span: &TraceSpan);
fn flush(&self);
}
pub struct InMemoryTracingBackend {
spans: Mutex<Vec<TraceSpan>>,
}
impl InMemoryTracingBackend {
pub fn new() -> Self {
Self {
spans: Mutex::new(Vec::new()),
}
}
pub fn spans(&self) -> Vec<TraceSpan> {
self.spans.lock().unwrap_or_else(|e| e.into_inner()).clone()
}
pub fn clear(&self) {
self.spans.lock().unwrap_or_else(|e| e.into_inner()).clear();
}
pub fn trace_tree(&self, root_id: &str) -> Option<TraceNode> {
let spans = self.spans.lock().unwrap_or_else(|e| e.into_inner());
let root = spans.iter().find(|s| s.id == root_id)?;
Some(build_tree(root, &spans))
}
}
impl Default for InMemoryTracingBackend {
fn default() -> Self {
Self::new()
}
}
impl TracingBackend for InMemoryTracingBackend {
fn start_span(&self, span: &TraceSpan) {
self.spans
.lock()
.unwrap_or_else(|e| e.into_inner())
.push(span.clone());
}
fn end_span(&self, span: &TraceSpan) {
let mut spans = self.spans.lock().unwrap_or_else(|e| e.into_inner());
if let Some(existing) = spans.iter_mut().find(|s| s.id == span.id) {
*existing = span.clone();
}
}
fn flush(&self) {
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceNode {
pub span: TraceSpan,
pub children: Vec<TraceNode>,
}
fn build_tree(root: &TraceSpan, all_spans: &[TraceSpan]) -> TraceNode {
let children: Vec<TraceNode> = all_spans
.iter()
.filter(|s| s.parent_id.as_deref() == Some(root.id.as_str()))
.map(|child| build_tree(child, all_spans))
.collect();
TraceNode {
span: root.clone(),
children,
}
}
pub struct ConsoleTracingBackend;
impl TracingBackend for ConsoleTracingBackend {
fn start_span(&self, span: &TraceSpan) {
println!("[TRACE START] {} ({})", span.name, span.kind);
}
fn end_span(&self, span: &TraceSpan) {
let latency = span.latency_ms.unwrap_or(0);
let status_str = match &span.status {
SpanStatus::Ok => "OK".to_string(),
SpanStatus::Error(e) => format!("ERROR: {}", e),
};
println!(
"[TRACE END] {} latency={}ms status={}",
span.name, latency, status_str
);
}
fn flush(&self) {}
}
#[cfg(feature = "opentelemetry")]
pub struct OtelTracingBackend {
tracer: opentelemetry::global::BoxedTracer,
spans: Mutex<Vec<opentelemetry::global::BoxedSpan>>,
}
#[cfg(feature = "opentelemetry")]
impl OtelTracingBackend {
pub fn new(tracer: opentelemetry::global::BoxedTracer) -> Self {
Self {
tracer,
spans: Mutex::new(Vec::new()),
}
}
pub fn from_global(name: &str) -> Self {
Self::new(opentelemetry::global::tracer(name.to_string()))
}
}
#[cfg(feature = "opentelemetry")]
impl TracingBackend for OtelTracingBackend {
fn start_span(&self, span: &TraceSpan) {
let otel_span = OtelTracer::start(&self.tracer, span.name.clone());
self.spans
.lock()
.unwrap_or_else(|e| e.into_inner())
.push(otel_span);
}
fn end_span(&self, _span: &TraceSpan) {
if let Some(mut s) = self.spans.lock().unwrap_or_else(|e| e.into_inner()).pop() {
OtelSpan::end(&mut s);
}
}
fn flush(&self) {}
}
#[allow(clippy::missing_const_for_thread_local)]
mod span_stack_tls {
std::thread_local! {
pub(super) static SPAN_STACK: std::cell::RefCell<Vec<super::SpanId>> = const { std::cell::RefCell::new(Vec::new()) };
}
}
tokio::task_local! {
pub(super) static ASYNC_SPAN_STACK: std::cell::RefCell<Vec<super::SpanId>>;
}
pub async fn init_task_span_stack() {
let _ = ASYNC_SPAN_STACK
.scope(std::cell::RefCell::new(Vec::new()), async {})
.await;
}
fn get_current_span_id() -> Option<SpanId> {
if let Ok(id) = ASYNC_SPAN_STACK.try_with(|stack| stack.borrow().last().cloned()) {
return id;
}
span_stack_tls::SPAN_STACK.with(|s| s.borrow().last().cloned())
}
fn push_span_id(id: SpanId) {
let id_clone = id.clone();
if ASYNC_SPAN_STACK
.try_with(|stack| stack.borrow_mut().push(id))
.is_ok()
{
return;
}
span_stack_tls::SPAN_STACK.with(|s| s.borrow_mut().push(id_clone));
}
fn pop_span_id_if_matches(span_id: &str) {
if ASYNC_SPAN_STACK
.try_with(|stack| {
let mut s = stack.borrow_mut();
if s.last().map(|id| id.as_str()) == Some(span_id) {
s.pop();
}
})
.is_ok()
{
return;
}
span_stack_tls::SPAN_STACK.with(|s| {
let mut stack = s.borrow_mut();
if stack.last().map(|id| id.as_str()) == Some(span_id) {
stack.pop();
}
});
}
pub fn clear_span_stack() {
span_stack_tls::SPAN_STACK.with(|s| s.borrow_mut().clear());
let _ = ASYNC_SPAN_STACK.try_with(|stack| stack.borrow_mut().clear());
}
pub struct Tracer {
backend: Arc<dyn TracingBackend>,
}
impl Tracer {
pub fn new(backend: Arc<dyn TracingBackend>) -> Self {
Self { backend }
}
pub fn start(&self, name: &str, kind: SpanKind) -> SpanGuard {
let id = Uuid::now_v7().to_string();
let now = Utc::now().to_rfc3339();
let span = TraceSpan {
id: id.clone(),
parent_id: None,
name: name.to_string(),
kind,
start_time: Some(now),
end_time: None,
tokens: None,
cost: None,
latency_ms: None,
metadata: serde_json::Value::Object(serde_json::Map::new()),
status: SpanStatus::Ok,
};
self.backend.start_span(&span);
push_span_id(id.clone());
SpanGuard {
span,
backend: Arc::clone(&self.backend),
start_instant: Instant::now(),
dropped: false,
}
}
pub fn start_child(&self, name: &str, kind: SpanKind) -> SpanGuard {
let parent_id = get_current_span_id();
let id = Uuid::now_v7().to_string();
let now = Utc::now().to_rfc3339();
let span = TraceSpan {
id: id.clone(),
parent_id,
name: name.to_string(),
kind,
start_time: Some(now),
end_time: None,
tokens: None,
cost: None,
latency_ms: None,
metadata: serde_json::Value::Object(serde_json::Map::new()),
status: SpanStatus::Ok,
};
self.backend.start_span(&span);
push_span_id(id.clone());
SpanGuard {
span,
backend: Arc::clone(&self.backend),
start_instant: Instant::now(),
dropped: false,
}
}
pub fn start_child_with_parent(
&self,
name: &str,
kind: SpanKind,
parent_id: SpanId,
) -> SpanGuard {
let id = Uuid::now_v7().to_string();
let now = Utc::now().to_rfc3339();
let span = TraceSpan {
id: id.clone(),
parent_id: Some(parent_id),
name: name.to_string(),
kind,
start_time: Some(now),
end_time: None,
tokens: None,
cost: None,
latency_ms: None,
metadata: serde_json::Value::Object(serde_json::Map::new()),
status: SpanStatus::Ok,
};
self.backend.start_span(&span);
push_span_id(id.clone());
SpanGuard {
span,
backend: Arc::clone(&self.backend),
start_instant: Instant::now(),
dropped: false,
}
}
pub fn current_span_id(&self) -> Option<SpanId> {
get_current_span_id()
}
pub fn flush(&self) {
self.backend.flush();
}
fn end_span(backend: &Arc<dyn TracingBackend>, span: &TraceSpan) {
backend.end_span(span);
pop_span_id_if_matches(&span.id);
}
}
impl Clone for Tracer {
fn clone(&self) -> Self {
Self {
backend: Arc::clone(&self.backend),
}
}
}
pub struct SpanGuard {
span: TraceSpan,
backend: Arc<dyn TracingBackend>,
start_instant: Instant,
dropped: bool,
}
impl SpanGuard {
pub fn id(&self) -> &str {
&self.span.id
}
pub fn parent_id(&self) -> Option<&str> {
self.span.parent_id.as_deref()
}
pub fn with_tokens(mut self, usage: SpanTokenUsage) -> Self {
self.span.tokens = Some(usage);
self
}
pub fn with_cost(mut self, cost: f64) -> Self {
self.span.cost = Some(cost);
self
}
pub fn with_metadata(mut self, key: &str, value: serde_json::Value) -> Self {
if let Some(obj) = self.span.metadata.as_object_mut() {
obj.insert(key.to_string(), value);
}
self
}
pub fn set_error(&mut self, msg: &str) {
self.span.status = SpanStatus::Error(msg.to_string());
}
pub fn end(mut self) {
if !self.dropped {
self.span.end_time = Some(Utc::now().to_rfc3339());
self.span.latency_ms = Some(self.start_instant.elapsed().as_millis() as u64);
Tracer::end_span(&self.backend, &self.span);
self.dropped = true;
}
}
}
impl Drop for SpanGuard {
fn drop(&mut self) {
if !self.dropped {
self.span.end_time = Some(Utc::now().to_rfc3339());
self.span.latency_ms = Some(self.start_instant.elapsed().as_millis() as u64);
Tracer::end_span(&self.backend, &self.span);
self.dropped = true;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_span_kind_display() {
assert_eq!(format!("{}", SpanKind::Llm), "llm");
assert_eq!(format!("{}", SpanKind::Chain), "chain");
assert_eq!(format!("{}", SpanKind::Tool), "tool");
assert_eq!(format!("{}", SpanKind::Retriever), "retriever");
assert_eq!(format!("{}", SpanKind::Agent), "agent");
assert_eq!(
format!("{}", SpanKind::Custom("embedding".into())),
"custom:embedding"
);
}
#[test]
fn test_span_creation_lifecycle() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let guard = tracer.start("test_span", SpanKind::Chain);
assert!(!guard.id().is_empty());
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
let span = &spans[0];
assert_eq!(span.name, "test_span");
assert_eq!(span.kind, SpanKind::Chain);
assert!(span.start_time.is_some());
assert!(span.end_time.is_some());
assert!(span.latency_ms.is_some());
assert_eq!(span.status, SpanStatus::Ok);
}
#[test]
fn test_parent_child_relationship() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let root = tracer.start("root", SpanKind::Chain);
let root_id = root.id().to_string();
{
let child = tracer.start_child("child", SpanKind::Tool);
assert_eq!(child.parent_id(), Some(root_id.as_str()));
}
}
let spans = backend.spans();
assert_eq!(spans.len(), 2);
let root = spans.iter().find(|s| s.name == "root").unwrap();
let child = spans.iter().find(|s| s.name == "child").unwrap();
assert!(root.parent_id.is_none());
assert_eq!(child.parent_id.as_deref(), Some(root.id.as_str()));
}
#[test]
fn test_start_child_without_parent_creates_root() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
clear_span_stack();
{
let child = tracer.start_child("orphan", SpanKind::Tool);
assert!(child.parent_id().is_none());
}
}
#[test]
fn test_span_guard_with_tokens() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let usage = SpanTokenUsage {
prompt_tokens: 10,
completion_tokens: 20,
total_tokens: 30,
};
let guard = tracer.start("llm_call", SpanKind::Llm).with_tokens(usage);
guard.end();
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
let tokens = spans[0].tokens.as_ref().unwrap();
assert_eq!(tokens.prompt_tokens, 10);
assert_eq!(tokens.completion_tokens, 20);
assert_eq!(tokens.total_tokens, 30);
}
#[test]
fn test_span_guard_with_cost() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
tracer
.start("llm_call", SpanKind::Llm)
.with_cost(0.003)
.end();
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
assert!((spans[0].cost.unwrap() - 0.003).abs() < f64::EPSILON);
}
#[test]
fn test_span_guard_with_metadata() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
tracer
.start("call", SpanKind::Chain)
.with_metadata("model", serde_json::json!("gpt-4"))
.with_metadata("temperature", serde_json::json!(0.7))
.end();
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
let meta = &spans[0].metadata;
assert_eq!(meta["model"], "gpt-4");
assert_eq!(meta["temperature"], 0.7);
}
#[test]
fn test_span_guard_set_error() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let mut guard = tracer.start("call", SpanKind::Tool);
guard.set_error("tool failed");
guard.end();
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
assert_eq!(
spans[0].status,
SpanStatus::Error("tool failed".to_string())
);
}
#[test]
fn test_span_guard_raii_drop() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let _guard = tracer.start("auto_dropped", SpanKind::Chain);
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
assert!(spans[0].end_time.is_some(), "Drop should end the span");
assert!(
spans[0].latency_ms.is_some(),
"Drop should calculate latency"
);
}
#[test]
fn test_in_memory_backend_trace_tree() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
let root_id;
{
let root = tracer.start("root", SpanKind::Agent);
root_id = root.id().to_string();
let child1 = tracer.start_child("child1", SpanKind::Tool);
drop(child1);
let child2 = tracer.start_child("child2", SpanKind::Chain);
let grandchild = tracer.start_child("grandchild", SpanKind::Llm);
drop(grandchild);
drop(child2);
drop(root);
}
let tree = backend.trace_tree(&root_id).unwrap();
assert_eq!(tree.span.name, "root");
assert_eq!(tree.children.len(), 2);
let child1_node = tree
.children
.iter()
.find(|c| c.span.name == "child1")
.unwrap();
let child2_node = tree
.children
.iter()
.find(|c| c.span.name == "child2")
.unwrap();
assert!(child1_node.children.is_empty());
assert_eq!(child2_node.children.len(), 1);
assert_eq!(child2_node.children[0].span.name, "grandchild");
}
#[test]
fn test_in_memory_backend_trace_tree_not_found() {
let backend = InMemoryTracingBackend::new();
assert!(backend.trace_tree("nonexistent").is_none());
}
#[test]
fn test_in_memory_backend_clear() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let _guard = tracer.start("temp", SpanKind::Chain);
}
assert_eq!(backend.spans().len(), 1);
backend.clear();
assert!(backend.spans().is_empty());
}
#[test]
fn test_tracer_current_span_id() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
clear_span_stack();
assert!(tracer.current_span_id().is_none());
let root = tracer.start("root", SpanKind::Chain);
let root_id = root.id().to_string();
assert_eq!(tracer.current_span_id(), Some(root_id.clone()));
let child = tracer.start_child("child", SpanKind::Tool);
let child_id = child.id().to_string();
assert_eq!(tracer.current_span_id(), Some(child_id));
drop(child);
assert_eq!(tracer.current_span_id(), Some(root_id));
drop(root);
assert!(tracer.current_span_id().is_none());
}
#[test]
fn test_start_child_with_explicit_parent() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
clear_span_stack();
let root = tracer.start("root", SpanKind::Agent);
let root_id = root.id().to_string();
let child = tracer.start_child_with_parent("child", SpanKind::Tool, root_id.clone());
assert_eq!(child.parent_id(), Some(root_id.as_str()));
}
#[test]
fn test_span_serialization_roundtrip() {
let span = TraceSpan {
id: "test-id".to_string(),
parent_id: Some("parent-id".to_string()),
name: "test_span".to_string(),
kind: SpanKind::Llm,
start_time: Some("2025-01-01T00:00:00Z".to_string()),
end_time: Some("2025-01-01T00:00:01Z".to_string()),
tokens: Some(SpanTokenUsage {
prompt_tokens: 10,
completion_tokens: 20,
total_tokens: 30,
}),
cost: Some(0.005),
latency_ms: Some(1000),
metadata: serde_json::json!({"model": "gpt-4"}),
status: SpanStatus::Ok,
};
let json = serde_json::to_string(&span).unwrap();
let deserialized: TraceSpan = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.id, span.id);
assert_eq!(deserialized.name, span.name);
assert_eq!(deserialized.kind, span.kind);
assert_eq!(deserialized.tokens, span.tokens);
}
#[test]
fn test_console_backend_does_not_panic() {
let backend = ConsoleTracingBackend;
let span = TraceSpan {
id: "test".to_string(),
parent_id: None,
name: "test_span".to_string(),
kind: SpanKind::Chain,
start_time: Some("2025-01-01T00:00:00Z".to_string()),
end_time: None,
tokens: None,
cost: None,
latency_ms: None,
metadata: serde_json::Value::Null,
status: SpanStatus::Ok,
};
backend.start_span(&span);
backend.flush();
}
#[test]
fn test_tracer_flush() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
tracer.flush();
}
#[test]
fn test_span_guard_end_called_twice_is_safe() {
let backend = Arc::new(InMemoryTracingBackend::new());
let tracer = Tracer::new(backend.clone());
{
let guard = tracer.start("double_end", SpanKind::Chain);
guard.end();
}
let spans = backend.spans();
assert_eq!(spans.len(), 1);
}
}