aether_telemetry/
telemetry_runtime.rs

1use crate::AgentTraceContext;
2use crate::content_capture::ContentCaptureSettings;
3use crate::error::{TelemetryInitError, TelemetryShutdownError};
4use crate::gen_ai_metrics::GenAiMetrics;
5use crate::genai_constants;
6use crate::otel_observer::OtelInstrumentation;
7use crate::span_guard::{ErrorKind, SpanGuard};
8use crate::trace_context::{extract_trace_context, inject_trace_context};
9use aether_core::events::{
10    AgentObserver, DynObserverFactory, McpRequestInstrumentation, ObserverFactory, TraceContext,
11};
12use opentelemetry::metrics::MeterProvider as _;
13use opentelemetry::trace::{SpanBuilder, SpanKind, TraceId, TraceState, TracerProvider as _};
14use opentelemetry::{Context, KeyValue};
15use opentelemetry_otlp::{MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig};
16use opentelemetry_sdk::Resource;
17use opentelemetry_sdk::metrics::SdkMeterProvider;
18use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader;
19use opentelemetry_sdk::runtime::Tokio;
20use opentelemetry_sdk::trace::span_processor_with_async_runtime::BatchSpanProcessor;
21use opentelemetry_sdk::trace::{IdGenerator, RandomIdGenerator, Sampler, SdkTracerProvider};
22use std::collections::HashMap;
23use std::str::FromStr;
24
25pub struct TelemetryRuntime {
26    tracer_provider: SdkTracerProvider,
27    meter_provider: SdkMeterProvider,
28    instrumentation: OtelInstrumentation,
29}
30
31struct OtelObserverFactory {
32    instrumentation: OtelInstrumentation,
33}
34
35struct OtelMcpRequestInstrumentation {
36    span: SpanGuard,
37}
38
39/// Fully-resolved telemetry configuration. Every field is required: defaults
40/// and settings-layer merging live in `TelemetrySettings`, so a value here is
41/// always deliberate.
42pub struct TelemetryConfig {
43    /// Base URL for the OTLP/HTTP collector, required whenever a signal needs an exporter.
44    pub endpoint: Option<String>,
45    /// Exact OTLP/HTTP trace export URL. When set, this takes precedence over the
46    /// trace URL derived from `endpoint`.
47    pub traces_endpoint: Option<String>,
48    /// Exact OTLP/HTTP metric export URL. When set, this takes precedence over the
49    /// metric URL derived from `endpoint`.
50    pub metrics_endpoint: Option<String>,
51    pub headers: HashMap<String, String>,
52    pub service_name: String,
53    pub service_version: String,
54    pub sample_ratio: f64,
55    pub content: ContentCaptureSettings,
56    pub trace_context: Option<AgentTraceContext>,
57    pub traces_enabled: bool,
58    pub metrics_enabled: bool,
59}
60
61impl TelemetryRuntime {
62    pub fn new(config: &TelemetryConfig) -> Result<Self, TelemetryInitError> {
63        if !(0.0..=1.0).contains(&config.sample_ratio) {
64            return Err(TelemetryInitError::InvalidSampleRatio(config.sample_ratio));
65        }
66
67        let trace_root = resolve_trace_root(config.trace_context.as_ref())?;
68        let http_client = build_http_client(&config.headers)?;
69        let resource = Resource::builder()
70            .with_service_name(config.service_name.clone())
71            .with_attribute(KeyValue::new(genai_constants::SERVICE_VERSION, config.service_version.clone()))
72            .build();
73        let scope = genai_constants::genai_instrumentation_scope(config.service_version.clone());
74        let tracer_provider =
75            build_tracer_provider(config, resource.clone(), http_client.clone(), trace_root.trace_id)?;
76        let meter_provider = build_meter_provider(config, resource, http_client)?;
77        let instrumentation = OtelInstrumentation {
78            tracer: tracer_provider.tracer_with_scope(scope.clone()),
79            metrics: GenAiMetrics::new(&meter_provider.meter_with_scope(scope)),
80            content: config.content,
81            root_parent: trace_root.parent,
82            agent_name: None,
83        };
84
85        Ok(Self { tracer_provider, meter_provider, instrumentation })
86    }
87
88    pub fn observer_factory(&self) -> DynObserverFactory {
89        std::sync::Arc::new(OtelObserverFactory { instrumentation: self.instrumentation.clone() })
90    }
91
92    /// Flushes and shuts down, logging any failure. For callers whose only
93    /// recovery is logging.
94    pub fn shutdown_or_log(&self) {
95        if let Err(error) = self.shutdown() {
96            tracing::error!("Failed to shutdown telemetry: {error}");
97        }
98    }
99
100    /// Flushes and shuts down every signal. Both providers are always attempted;
101    /// providers for disabled signals have no exporters.
102    pub fn shutdown(&self) -> Result<(), TelemetryShutdownError> {
103        let traces = self.tracer_provider.shutdown().map_err(TelemetryShutdownError::Trace);
104        let metrics = self.meter_provider.shutdown().map_err(TelemetryShutdownError::Metric);
105        traces.and(metrics)
106    }
107}
108
109impl ObserverFactory for OtelObserverFactory {
110    fn agent(&self, agent_name: Option<&str>, parent: Option<&TraceContext>) -> Box<dyn AgentObserver> {
111        let mut instrumentation = self.instrumentation.clone();
112        instrumentation.agent_name = agent_name.map(str::to_string);
113        if let Some(parent) = parent.and_then(extract_trace_context) {
114            instrumentation.root_parent = Some(parent);
115        }
116        Box::new(crate::OtelObserver::new(instrumentation))
117    }
118
119    fn tool_call_request(&self, tool_name: &str, parent: Option<&TraceContext>) -> Box<dyn McpRequestInstrumentation> {
120        let remote_parent = parent.and_then(extract_trace_context);
121        let attributes = vec![
122            KeyValue::new(genai_constants::GEN_AI_OPERATION_NAME, "execute_tool"),
123            KeyValue::new(genai_constants::GEN_AI_TOOL_NAME, tool_name.to_string()),
124            KeyValue::new(genai_constants::MCP_METHOD_NAME, TOOLS_CALL_METHOD),
125            KeyValue::new(genai_constants::MCP_TOOL_NAME, tool_name.to_string()),
126        ];
127        let builder = SpanBuilder::from_name(format!("{TOOLS_CALL_METHOD} {tool_name}"))
128            .with_kind(SpanKind::Server)
129            .with_attributes(attributes);
130        let context = self
131            .instrumentation
132            .start_span(builder, remote_parent.as_ref().or(self.instrumentation.root_parent.as_ref()));
133        Box::new(OtelMcpRequestInstrumentation { span: SpanGuard::new(context, "MCP request cancelled") })
134    }
135}
136
137impl McpRequestInstrumentation for OtelMcpRequestInstrumentation {
138    fn trace_context(&self) -> Option<TraceContext> {
139        inject_trace_context(self.span.context())
140    }
141
142    fn finish(mut self: Box<Self>, error: Option<&str>) {
143        match error {
144            Some(error) => self.span.end_error(Some(ErrorKind::McpError), error),
145            None => self.span.end_ok(),
146        }
147    }
148}
149
150impl TelemetryConfig {
151    fn signal_endpoint(&self, signal: &str) -> Result<String, TelemetryInitError> {
152        let signal_specific_endpoint = match signal {
153            "traces" => self.traces_endpoint.as_deref(),
154            "metrics" => self.metrics_endpoint.as_deref(),
155            _ => None,
156        };
157        if let Some(endpoint) = signal_specific_endpoint.filter(|endpoint| !endpoint.is_empty()) {
158            return Ok(endpoint.to_string());
159        }
160
161        Ok(format!("{}/v1/{signal}", self.required_endpoint()?.trim_end_matches('/')))
162    }
163
164    fn required_endpoint(&self) -> Result<&str, TelemetryInitError> {
165        self.endpoint.as_deref().filter(|endpoint| !endpoint.is_empty()).ok_or(TelemetryInitError::MissingOtlpEndpoint)
166    }
167}
168
169const TOOLS_CALL_METHOD: &str = "tools/call";
170
171#[derive(Default)]
172struct TraceRoot {
173    parent: Option<Context>,
174    trace_id: Option<TraceId>,
175}
176
177#[derive(Debug)]
178struct TraceIdGenerator {
179    trace_id: TraceId,
180    random: RandomIdGenerator,
181}
182
183impl IdGenerator for TraceIdGenerator {
184    fn new_trace_id(&self) -> TraceId {
185        self.trace_id
186    }
187
188    fn new_span_id(&self) -> opentelemetry::trace::SpanId {
189        self.random.new_span_id()
190    }
191}
192
193fn resolve_trace_root(trace_context: Option<&AgentTraceContext>) -> Result<TraceRoot, TelemetryInitError> {
194    let Some(trace_context) = trace_context else {
195        return Ok(TraceRoot::default());
196    };
197
198    match trace_context {
199        AgentTraceContext::Parent(trace_context) => {
200            if let Some(tracestate) = &trace_context.tracestate {
201                TraceState::from_str(tracestate).map_err(|_| TelemetryInitError::InvalidTraceContext("tracestate"))?;
202            }
203
204            let parent =
205                extract_trace_context(trace_context).ok_or(TelemetryInitError::InvalidTraceContext("traceparent"))?;
206
207            Ok(TraceRoot { parent: Some(parent), trace_id: None })
208        }
209        AgentTraceContext::Root { trace_id } => {
210            let trace_id = parse_trace_id(trace_id)?;
211            Ok(TraceRoot { parent: None, trace_id: Some(trace_id) })
212        }
213    }
214}
215
216fn parse_trace_id(value: &str) -> Result<TraceId, TelemetryInitError> {
217    let valid_hex =
218        value.len() == 32 && value.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
219    if !valid_hex {
220        return Err(TelemetryInitError::InvalidTraceContext("traceId"));
221    }
222
223    let trace_id = TraceId::from_hex(value).map_err(|_| TelemetryInitError::InvalidTraceContext("traceId"))?;
224    if trace_id == TraceId::INVALID {
225        return Err(TelemetryInitError::InvalidTraceContext("traceId"));
226    }
227
228    Ok(trace_id)
229}
230
231fn build_tracer_provider(
232    config: &TelemetryConfig,
233    resource: Resource,
234    http_client: reqwest::Client,
235    trace_id: Option<TraceId>,
236) -> Result<SdkTracerProvider, TelemetryInitError> {
237    let builder = SdkTracerProvider::builder().with_resource(resource);
238    let builder = match trace_id {
239        Some(trace_id) => {
240            builder.with_id_generator(TraceIdGenerator { trace_id, random: RandomIdGenerator::default() })
241        }
242        None => builder,
243    };
244    if !config.traces_enabled {
245        return Ok(builder.with_sampler(Sampler::AlwaysOff).build());
246    }
247
248    let endpoint = config.signal_endpoint("traces")?;
249    let exporter = SpanExporter::builder()
250        .with_http()
251        .with_endpoint(endpoint)
252        .with_http_client(http_client)
253        .build()
254        .map_err(TelemetryInitError::TraceExporter)?;
255
256    Ok(builder
257        .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(config.sample_ratio))))
258        .with_span_processor(BatchSpanProcessor::builder(exporter, Tokio).build())
259        .build())
260}
261
262fn build_meter_provider(
263    config: &TelemetryConfig,
264    resource: Resource,
265    http_client: reqwest::Client,
266) -> Result<SdkMeterProvider, TelemetryInitError> {
267    let builder = SdkMeterProvider::builder().with_resource(resource);
268    if !config.metrics_enabled {
269        return Ok(builder.build());
270    }
271
272    let endpoint = config.signal_endpoint("metrics")?;
273    let exporter = MetricExporter::builder()
274        .with_http()
275        .with_endpoint(endpoint)
276        .with_http_client(http_client)
277        .build()
278        .map_err(TelemetryInitError::MetricExporter)?;
279
280    Ok(builder.with_reader(PeriodicReader::builder(exporter, Tokio).build()).build())
281}
282
283fn build_http_client(headers: &HashMap<String, String>) -> Result<reqwest::Client, TelemetryInitError> {
284    let mut parsed = reqwest::header::HeaderMap::new();
285    for (key, value) in headers {
286        let name = reqwest::header::HeaderName::from_bytes(key.as_bytes())
287            .map_err(|_| TelemetryInitError::InvalidHeaderName(key.clone()))?;
288        let value = reqwest::header::HeaderValue::from_str(value)
289            .map_err(|_| TelemetryInitError::InvalidHeaderValue(key.clone()))?;
290        parsed.insert(name, value);
291    }
292    reqwest::Client::builder().default_headers(parsed).build().map_err(TelemetryInitError::HttpClient)
293}
aether_telemetry/telemetry_runtime.rs

aether_telemetry/
telemetry_runtime.rs
