use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Instant;
use opentelemetry::trace::{SpanKind, SpanRef, Status, TraceContextExt, Tracer};
use opentelemetry::{Context as OtelContext, InstrumentationScope, KeyValue, global};
use tower::Service;
use tracing::Instrument;
use crate::shared::observability::domain::{DetailLevel, MetricsLeversConfig};
use camel_api::metrics::MetricsCollector;
use camel_api::{Body, BoxProcessor, CIRCUIT_OPEN, CamelError, Exchange, SpanKindHint};
pub(crate) struct SpanEndGuard(pub(crate) OtelContext);
impl Drop for SpanEndGuard {
fn drop(&mut self) {
self.0.span().end();
}
}
fn body_type_name(body: &Body) -> &'static str {
match body {
Body::Empty => "empty",
Body::Bytes(_) => "bytes",
Body::Text(_) => "text",
Body::Json(_) => "json",
Body::Xml(_) => "xml",
Body::Stream(_) => "stream",
_ => "unknown",
}
}
pub struct TracingProcessor {
inner: BoxProcessor,
route_id: String,
step_id: String,
span_name: String,
step_index: usize,
detail_level: DetailLevel,
metrics: Option<Arc<dyn MetricsCollector>>,
span_kind: SpanKind,
spans_enabled: bool,
metric_levers: MetricsLeversConfig,
}
pub(crate) fn step_id_for(index: usize) -> String {
format!("step-{index}")
}
impl TracingProcessor {
pub fn new(
inner: BoxProcessor,
route_id: String,
step_index: usize,
detail_level: DetailLevel,
metrics: Option<Arc<dyn MetricsCollector>>,
label: Option<Arc<str>>,
kind_hint: SpanKindHint,
) -> Self {
let step_id = step_id_for(step_index);
let span_name = format!("{route_id}:{}", label.as_deref().unwrap_or(&step_id));
let span_kind = match kind_hint {
SpanKindHint::Internal => SpanKind::Internal,
SpanKindHint::Producer => SpanKind::Producer,
SpanKindHint::Consumer => SpanKind::Consumer,
SpanKindHint::Client => SpanKind::Client,
SpanKindHint::Server => SpanKind::Server,
_ => SpanKind::Internal,
};
Self {
inner,
route_id,
step_id,
span_name,
step_index,
detail_level,
metrics,
span_kind,
spans_enabled: true,
metric_levers: MetricsLeversConfig::default(),
}
}
pub fn with_spans_enabled(mut self, enabled: bool) -> Self {
self.spans_enabled = enabled;
self
}
pub fn with_metric_levers(mut self, levers: MetricsLeversConfig) -> Self {
self.metric_levers = levers;
self
}
fn call_metrics_only(
&mut self,
exchange: Exchange,
start: Instant,
) -> Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>> {
let fresh = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, fresh);
let metrics = self.metrics.clone();
let route_id = self.route_id.clone();
let levers = self.metric_levers.clone();
Box::pin(async move {
let result = inner.call(exchange).await;
record_step_metrics(
metrics.as_ref(),
&route_id,
&levers,
start.elapsed(),
&result,
);
result
})
}
}
fn record_step_metrics(
metrics: Option<&Arc<dyn MetricsCollector>>,
route_id: &str,
levers: &MetricsLeversConfig,
duration: std::time::Duration,
result: &Result<Exchange, CamelError>,
) {
let Some(metrics) = metrics else { return };
if levers.durations_enabled() {
metrics.record_exchange_duration(route_id, duration);
}
if levers.exchanges_enabled() {
metrics.increment_exchanges(route_id);
}
if let Err(e) = result {
let error_class = e.classify();
if error_class != CIRCUIT_OPEN {
metrics.increment_errors(route_id, error_class);
}
}
}
impl Service<Exchange> for TracingProcessor {
type Response = Exchange;
type Error = CamelError;
type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut exchange: Exchange) -> Self::Future {
let start = Instant::now();
if !self.spans_enabled {
return self.call_metrics_only(exchange, start);
}
let span_name = self.span_name.clone();
let span_kind = self.span_kind.clone();
let tracer = global::tracer_with_scope(
InstrumentationScope::builder("camel-core")
.with_version(env!("CARGO_PKG_VERSION"))
.build(),
);
let parent_cx = exchange.otel_context.clone();
let mut attributes =
step_span_attributes(&self.route_id, self.step_index, exchange.correlation_id());
if self.detail_level >= DetailLevel::Medium {
attributes.push(KeyValue::new(
"headers_count",
exchange.input.headers.len() as i64,
));
attributes.push(KeyValue::new(
"body_type",
body_type_name(&exchange.input.body),
));
attributes.push(KeyValue::new("has_error", exchange.has_error()));
}
let span = tracer
.span_builder(span_name)
.with_kind(span_kind)
.with_attributes(attributes.iter().cloned())
.start_with_context(&tracer, &parent_cx);
let cx = parent_cx.with_span(span);
exchange.otel_context = cx.clone();
let tracing_span = tracing::info_span!(
target: "camel_tracer",
"step",
correlation_id = %exchange.correlation_id(),
route_id = %self.route_id,
step_id = %self.step_id,
step_index = self.step_index,
duration_ms = tracing::field::Empty,
status = tracing::field::Empty,
headers_count = tracing::field::Empty,
body_type = tracing::field::Empty,
has_error = tracing::field::Empty,
output_body_type = tracing::field::Empty,
header_0 = tracing::field::Empty,
header_1 = tracing::field::Empty,
header_2 = tracing::field::Empty,
error = tracing::field::Empty,
error_type = tracing::field::Empty,
);
if self.detail_level >= DetailLevel::Medium {
tracing_span.record("headers_count", exchange.input.headers.len() as u64);
tracing_span.record("body_type", body_type_name(&exchange.input.body));
tracing_span.record("has_error", exchange.has_error());
}
if self.detail_level >= DetailLevel::Full {
let headers: Vec<_> = exchange.input.headers.iter().take(3).collect();
if let Some((k, v)) = headers.first() {
tracing_span.record("header_0", format!("{k}={v:?}"));
}
if let Some((k, v)) = headers.get(1) {
tracing_span.record("header_1", format!("{k}={v:?}"));
}
if let Some((k, v)) = headers.get(2) {
tracing_span.record("header_2", format!("{k}={v:?}"));
}
}
let fresh = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, fresh);
let detail_level = self.detail_level.clone();
let metrics = self.metrics.clone();
let route_id = self.route_id.clone();
let levers = self.metric_levers.clone();
Box::pin(
async move {
let _guard = SpanEndGuard(cx.clone());
let result = inner.call(exchange).await;
let duration = start.elapsed();
let duration_ms = duration.as_millis() as u64;
tracing::Span::current().record("duration_ms", duration_ms);
record_step_metrics(metrics.as_ref(), &route_id, &levers, duration, &result);
match result {
Ok(mut ex) => {
tracing::Span::current().record("status", "success");
cx.span().set_status(Status::Ok);
if detail_level >= DetailLevel::Medium {
tracing::Span::current()
.record("output_body_type", body_type_name(&ex.input.body));
cx.span().set_attribute(KeyValue::new(
"output_body_type",
body_type_name(&ex.input.body),
));
}
ex.otel_context = parent_cx.clone();
Ok(ex)
}
Err(e) => {
record_exception(&cx.span(), &e);
let error_class = e.classify();
tracing::Span::current().record("status", "error");
tracing::Span::current().record("error", e.to_string());
tracing::Span::current().record("error_type", error_class);
Err(e)
}
}
}
.instrument(tracing_span),
)
}
}
impl Clone for TracingProcessor {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
route_id: self.route_id.clone(),
step_id: self.step_id.clone(),
span_name: self.span_name.clone(),
step_index: self.step_index,
detail_level: self.detail_level.clone(),
metrics: self.metrics.clone(),
span_kind: self.span_kind.clone(),
spans_enabled: self.spans_enabled,
metric_levers: self.metric_levers.clone(),
}
}
}
pub(crate) fn capped_correlation_id(id: &str) -> &str {
const CAP: usize = 128;
if id.len() > CAP {
"<oversized:correlation_id>"
} else {
id
}
}
pub(crate) fn step_span_attributes(
route_id: &str,
step_index: usize,
correlation_id: &str,
) -> Vec<KeyValue> {
vec![
KeyValue::new("messaging.system", "camel"),
KeyValue::new(
"correlation_id",
capped_correlation_id(correlation_id).to_string(),
),
KeyValue::new("route_id", route_id.to_string()),
KeyValue::new("step_index", step_index as i64),
]
}
pub(crate) fn record_exception(span: &SpanRef<'_>, e: &CamelError) {
let error_class = e.classify();
span.set_status(Status::error(e.to_string()));
span.add_event(
"exception",
vec![
KeyValue::new("exception.type", error_class.to_string()),
KeyValue::new("exception.message", e.to_string()),
],
);
}
#[cfg(test)]
#[path = "tracer_tests.rs"]
mod tests;