use std::time::Duration;
use opentelemetry::{global, KeyValue};
use opentelemetry_sdk::{
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
Resource,
};
use crate::TelemetryConfig;
pub struct TracingGuard {
provider: Option<SdkTracerProvider>,
}
impl Drop for TracingGuard {
fn drop(&mut self) {
if let Some(provider) = self.provider.take() {
if let Err(e) = provider.shutdown() {
tracing::warn!("Error shutting down tracer provider: {:?}", e);
}
}
}
}
pub fn init_tracing(
config: &TelemetryConfig,
) -> Result<TracingGuard, Box<dyn std::error::Error + Send + Sync>> {
let provider = if let Some(endpoint) = &config.otlp_endpoint {
tracing::info!(endpoint = %endpoint, "Initializing OTLP tracing");
#[cfg(feature = "otlp")]
{
use opentelemetry_otlp::{SpanExporter, WithExportConfig};
let exporter = SpanExporter::builder()
.with_tonic()
.with_endpoint(endpoint)
.with_timeout(Duration::from_secs(10))
.build()?;
let resource = Resource::builder()
.with_service_name(config.service_name.clone())
.with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION")))
.build();
let provider = SdkTracerProvider::builder()
.with_batch_exporter(exporter)
.with_sampler(Sampler::AlwaysOn)
.with_id_generator(RandomIdGenerator::default())
.with_resource(resource)
.build();
let _ = global::set_tracer_provider(provider.clone());
tracing::info!(
service = %config.service_name,
endpoint = %endpoint,
"OTLP tracing initialized"
);
Some(provider)
}
#[cfg(not(feature = "otlp"))]
{
tracing::warn!("OTLP feature not enabled, tracing will be local only");
None
}
} else {
tracing::debug!("No OTLP endpoint configured, using local tracing only");
None
};
Ok(TracingGuard { provider })
}
#[must_use]
pub fn create_tracer(component: &str) -> opentelemetry::global::BoxedTracer {
global::tracer(component.to_string())
}
pub struct LLMSpan {
pub model_id: String,
pub input_tokens: u32,
pub output_tokens: u32,
pub temperature: f32,
pub ttft_ms: Option<f64>,
pub total_time_ms: Option<f64>,
pub tokens_per_second: Option<f64>,
}
impl LLMSpan {
#[must_use]
pub fn new(model_id: impl Into<String>) -> Self {
Self {
model_id: model_id.into(),
input_tokens: 0,
output_tokens: 0,
temperature: 1.0,
ttft_ms: None,
total_time_ms: None,
tokens_per_second: None,
}
}
pub fn record_tokens(&mut self, input: u32, output: u32) {
self.input_tokens = input;
self.output_tokens = output;
}
pub fn record_timing(&mut self, ttft_ms: f64, total_time_ms: f64) {
self.ttft_ms = Some(ttft_ms);
self.total_time_ms = Some(total_time_ms);
if total_time_ms > 0.0 && self.output_tokens > 0 {
self.tokens_per_second = Some((self.output_tokens as f64 / total_time_ms) * 1000.0);
}
}
#[must_use]
pub fn to_attributes(&self) -> Vec<KeyValue> {
let mut attrs = vec![
KeyValue::new("llm.model_id", self.model_id.clone()),
KeyValue::new("llm.input_tokens", self.input_tokens as i64),
KeyValue::new("llm.output_tokens", self.output_tokens as i64),
KeyValue::new("llm.temperature", self.temperature as f64),
];
if let Some(ttft) = self.ttft_ms {
attrs.push(KeyValue::new("llm.ttft_ms", ttft));
}
if let Some(total) = self.total_time_ms {
attrs.push(KeyValue::new("llm.total_time_ms", total));
}
if let Some(tps) = self.tokens_per_second {
attrs.push(KeyValue::new("llm.tokens_per_second", tps));
}
attrs
}
}
pub struct LLMSpanBuilder {
span: LLMSpan,
}
impl LLMSpanBuilder {
#[must_use]
pub fn new(model_id: impl Into<String>) -> Self {
Self {
span: LLMSpan::new(model_id),
}
}
#[must_use]
pub fn input_tokens(mut self, tokens: u32) -> Self {
self.span.input_tokens = tokens;
self
}
#[must_use]
pub fn output_tokens(mut self, tokens: u32) -> Self {
self.span.output_tokens = tokens;
self
}
#[must_use]
pub fn temperature(mut self, temp: f32) -> Self {
self.span.temperature = temp;
self
}
#[must_use]
pub fn ttft_ms(mut self, ttft: f64) -> Self {
self.span.ttft_ms = Some(ttft);
self
}
#[must_use]
pub fn total_time_ms(mut self, total: f64) -> Self {
self.span.total_time_ms = Some(total);
self
}
#[must_use]
pub fn build(mut self) -> LLMSpan {
if let (Some(total), tokens) = (self.span.total_time_ms, self.span.output_tokens) {
if total > 0.0 && tokens > 0 {
self.span.tokens_per_second = Some((tokens as f64 / total) * 1000.0);
}
}
self.span
}
}
#[derive(Debug, Clone, Default)]
pub struct TracingConfig {
pub enabled: bool,
pub otlp_endpoint: Option<String>,
pub service_name: String,
pub sampling_ratio: f64,
pub propagate_context: bool,
}
impl TracingConfig {
#[must_use]
pub fn new(service_name: impl Into<String>) -> Self {
Self {
enabled: true,
otlp_endpoint: None,
service_name: service_name.into(),
sampling_ratio: 1.0,
propagate_context: true,
}
}
#[must_use]
pub fn with_endpoint(mut self, endpoint: impl Into<String>) -> Self {
self.otlp_endpoint = Some(endpoint.into());
self
}
#[must_use]
pub fn with_jaeger(mut self, host: impl Into<String>, port: u16) -> Self {
self.otlp_endpoint = Some(format!("http://{}:{}", host.into(), port));
self
}
#[must_use]
pub fn with_jaeger_default(self) -> Self {
self.with_jaeger("localhost", 4317)
}
#[must_use]
pub fn with_sampling_ratio(mut self, ratio: f64) -> Self {
self.sampling_ratio = ratio.clamp(0.0, 1.0);
self
}
#[must_use]
pub fn disabled(mut self) -> Self {
self.enabled = false;
self
}
}
#[derive(Debug, Clone)]
pub struct InferenceSpan {
pub model_id: String,
pub request_id: String,
pub input_tokens: u32,
pub output_tokens: u32,
pub temperature: f32,
pub top_p: f32,
pub ttft_ms: Option<f64>,
pub total_time_ms: Option<f64>,
pub batch_size: Option<u32>,
pub streaming: bool,
pub cache_hit_tokens: Option<u32>,
}
impl InferenceSpan {
#[must_use]
pub fn new(model_id: impl Into<String>, request_id: impl Into<String>) -> Self {
Self {
model_id: model_id.into(),
request_id: request_id.into(),
input_tokens: 0,
output_tokens: 0,
temperature: 1.0,
top_p: 1.0,
ttft_ms: None,
total_time_ms: None,
batch_size: None,
streaming: false,
cache_hit_tokens: None,
}
}
#[must_use]
pub fn to_attributes(&self) -> Vec<KeyValue> {
let mut attrs = vec![
KeyValue::new("llm.model_id", self.model_id.clone()),
KeyValue::new("llm.request_id", self.request_id.clone()),
KeyValue::new("llm.input_tokens", self.input_tokens as i64),
KeyValue::new("llm.output_tokens", self.output_tokens as i64),
KeyValue::new("llm.temperature", self.temperature as f64),
KeyValue::new("llm.top_p", self.top_p as f64),
KeyValue::new("llm.streaming", self.streaming),
];
if let Some(ttft) = self.ttft_ms {
attrs.push(KeyValue::new("llm.ttft_ms", ttft));
}
if let Some(total) = self.total_time_ms {
attrs.push(KeyValue::new("llm.total_time_ms", total));
if self.output_tokens > 0 && total > 0.0 {
attrs.push(KeyValue::new(
"llm.tokens_per_second",
(self.output_tokens as f64 / total) * 1000.0,
));
}
}
if let Some(batch) = self.batch_size {
attrs.push(KeyValue::new("llm.batch_size", batch as i64));
}
if let Some(cache_hit) = self.cache_hit_tokens {
attrs.push(KeyValue::new("llm.cache_hit_tokens", cache_hit as i64));
}
attrs
}
}
#[derive(Debug, Clone)]
pub struct RetrievalSpan {
pub query_id: String,
pub num_retrieved: u32,
pub num_reranked: Option<u32>,
pub top_score: Option<f32>,
pub retrieval_time_ms: f64,
pub embedding_time_ms: Option<f64>,
pub hybrid_search: bool,
pub collection: Option<String>,
}
impl RetrievalSpan {
#[must_use]
pub fn new(query_id: impl Into<String>) -> Self {
Self {
query_id: query_id.into(),
num_retrieved: 0,
num_reranked: None,
top_score: None,
retrieval_time_ms: 0.0,
embedding_time_ms: None,
hybrid_search: false,
collection: None,
}
}
#[must_use]
pub fn to_attributes(&self) -> Vec<KeyValue> {
let mut attrs = vec![
KeyValue::new("rag.query_id", self.query_id.clone()),
KeyValue::new("rag.num_retrieved", self.num_retrieved as i64),
KeyValue::new("rag.retrieval_time_ms", self.retrieval_time_ms),
KeyValue::new("rag.hybrid_search", self.hybrid_search),
];
if let Some(reranked) = self.num_reranked {
attrs.push(KeyValue::new("rag.num_reranked", reranked as i64));
}
if let Some(score) = self.top_score {
attrs.push(KeyValue::new("rag.top_score", score as f64));
}
if let Some(embed_time) = self.embedding_time_ms {
attrs.push(KeyValue::new("rag.embedding_time_ms", embed_time));
}
if let Some(ref collection) = self.collection {
attrs.push(KeyValue::new("rag.collection", collection.clone()));
}
attrs
}
}
#[derive(Debug, Clone)]
pub struct ToolSpan {
pub tool_name: String,
pub agent_id: String,
pub success: bool,
pub execution_time_ms: f64,
pub error: Option<String>,
pub params_summary: Option<String>,
pub risk_level: Option<String>,
}
impl ToolSpan {
#[must_use]
pub fn new(tool_name: impl Into<String>, agent_id: impl Into<String>) -> Self {
Self {
tool_name: tool_name.into(),
agent_id: agent_id.into(),
success: false,
execution_time_ms: 0.0,
error: None,
params_summary: None,
risk_level: None,
}
}
pub fn mark_success(&mut self, execution_time_ms: f64) {
self.success = true;
self.execution_time_ms = execution_time_ms;
}
pub fn mark_failure(&mut self, error: impl Into<String>, execution_time_ms: f64) {
self.success = false;
self.error = Some(error.into());
self.execution_time_ms = execution_time_ms;
}
#[must_use]
pub fn to_attributes(&self) -> Vec<KeyValue> {
let mut attrs = vec![
KeyValue::new("agent.tool_name", self.tool_name.clone()),
KeyValue::new("agent.agent_id", self.agent_id.clone()),
KeyValue::new("agent.tool_success", self.success),
KeyValue::new("agent.execution_time_ms", self.execution_time_ms),
];
if let Some(ref error) = self.error {
attrs.push(KeyValue::new("agent.tool_error", error.clone()));
}
if let Some(ref params) = self.params_summary {
attrs.push(KeyValue::new("agent.params_summary", params.clone()));
}
if let Some(ref risk) = self.risk_level {
attrs.push(KeyValue::new("agent.risk_level", risk.clone()));
}
attrs
}
}
#[derive(Debug, Clone)]
pub struct AgentSpan {
pub agent_id: String,
pub agent_name: String,
pub objective: String,
pub steps_executed: u32,
pub tool_calls: u32,
pub total_tokens: u32,
pub total_time_ms: f64,
pub success: bool,
pub planning_strategy: Option<String>,
}
impl AgentSpan {
#[must_use]
pub fn new(
agent_id: impl Into<String>,
agent_name: impl Into<String>,
objective: impl Into<String>,
) -> Self {
Self {
agent_id: agent_id.into(),
agent_name: agent_name.into(),
objective: objective.into(),
steps_executed: 0,
tool_calls: 0,
total_tokens: 0,
total_time_ms: 0.0,
success: false,
planning_strategy: None,
}
}
#[must_use]
pub fn to_attributes(&self) -> Vec<KeyValue> {
let mut attrs = vec![
KeyValue::new("agent.id", self.agent_id.clone()),
KeyValue::new("agent.name", self.agent_name.clone()),
KeyValue::new("agent.objective", self.objective.clone()),
KeyValue::new("agent.steps_executed", self.steps_executed as i64),
KeyValue::new("agent.tool_calls", self.tool_calls as i64),
KeyValue::new("agent.total_tokens", self.total_tokens as i64),
KeyValue::new("agent.total_time_ms", self.total_time_ms),
KeyValue::new("agent.success", self.success),
];
if let Some(ref strategy) = self.planning_strategy {
attrs.push(KeyValue::new("agent.planning_strategy", strategy.clone()));
}
attrs
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_llm_span_builder() {
let span = LLMSpanBuilder::new("test-model")
.input_tokens(100)
.output_tokens(50)
.temperature(0.7)
.ttft_ms(25.0)
.total_time_ms(500.0)
.build();
assert_eq!(span.model_id, "test-model");
assert_eq!(span.input_tokens, 100);
assert_eq!(span.output_tokens, 50);
assert_eq!(span.temperature, 0.7);
assert!(span.tokens_per_second.is_some());
let tps = span.tokens_per_second.unwrap();
assert!((tps - 100.0).abs() < 0.1);
}
#[test]
fn test_llm_span_attributes() {
let span = LLMSpan::new("gpt-4");
let attrs = span.to_attributes();
assert!(attrs.iter().any(|kv| kv.key.as_str() == "llm.model_id"));
}
#[test]
fn test_tracing_config() {
let config = TracingConfig::new("infernum")
.with_endpoint("http://localhost:4317")
.with_sampling_ratio(0.5);
assert!(config.enabled);
assert_eq!(
config.otlp_endpoint,
Some("http://localhost:4317".to_string())
);
assert_eq!(config.sampling_ratio, 0.5);
}
#[test]
fn test_tracing_config_jaeger() {
let config = TracingConfig::new("infernum").with_jaeger("jaeger.local", 4317);
assert_eq!(
config.otlp_endpoint,
Some("http://jaeger.local:4317".to_string())
);
}
#[test]
fn test_tracing_config_jaeger_default() {
let config = TracingConfig::new("infernum").with_jaeger_default();
assert_eq!(
config.otlp_endpoint,
Some("http://localhost:4317".to_string())
);
}
#[test]
fn test_inference_span() {
let mut span = InferenceSpan::new("llama-3.2", "req-123");
span.input_tokens = 100;
span.output_tokens = 50;
span.temperature = 0.8;
span.streaming = true;
span.ttft_ms = Some(150.0);
span.total_time_ms = Some(500.0);
let attrs = span.to_attributes();
assert!(attrs.iter().any(|kv| kv.key.as_str() == "llm.model_id"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "llm.request_id"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "llm.streaming"));
assert!(attrs
.iter()
.any(|kv| kv.key.as_str() == "llm.tokens_per_second"));
}
#[test]
fn test_retrieval_span() {
let mut span = RetrievalSpan::new("query-456");
span.num_retrieved = 10;
span.num_reranked = Some(5);
span.top_score = Some(0.92);
span.retrieval_time_ms = 45.0;
span.collection = Some("documents".to_string());
let attrs = span.to_attributes();
assert!(attrs.iter().any(|kv| kv.key.as_str() == "rag.query_id"));
assert!(attrs
.iter()
.any(|kv| kv.key.as_str() == "rag.num_retrieved"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "rag.top_score"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "rag.collection"));
}
#[test]
fn test_tool_span_success() {
let mut span = ToolSpan::new("calculator", "agent-789");
span.mark_success(25.0);
span.risk_level = Some("low".to_string());
let attrs = span.to_attributes();
assert!(span.success);
assert_eq!(span.execution_time_ms, 25.0);
assert!(attrs.iter().any(|kv| kv.key.as_str() == "agent.tool_name"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "agent.risk_level"));
}
#[test]
fn test_tool_span_failure() {
let mut span = ToolSpan::new("web_search", "agent-789");
span.mark_failure("Connection timeout", 5000.0);
assert!(!span.success);
assert_eq!(span.error, Some("Connection timeout".to_string()));
assert_eq!(span.execution_time_ms, 5000.0);
}
#[test]
fn test_agent_span() {
let mut span = AgentSpan::new("agent-001", "research-assistant", "Find latest papers");
span.steps_executed = 5;
span.tool_calls = 3;
span.total_tokens = 2500;
span.total_time_ms = 12000.0;
span.success = true;
span.planning_strategy = Some("ReAct".to_string());
let attrs = span.to_attributes();
assert!(attrs.iter().any(|kv| kv.key.as_str() == "agent.id"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "agent.name"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "agent.objective"));
assert!(attrs
.iter()
.any(|kv| kv.key.as_str() == "agent.planning_strategy"));
assert!(attrs.iter().any(|kv| kv.key.as_str() == "agent.success"));
}
}