use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use tokio_util::sync::CancellationToken;
use tower::Service;
use camel_api::metrics::MetricsCollector;
use camel_api::{
BoxProcessor, CamelError, Exchange, IdentityProcessor, Message, NoOpMetrics,
ORIGINAL_MESSAGE_EXTENSION, PipelineOutcome,
};
use camel_api::error_handler::{BoundaryKind, RetryOutcome, StepDisposition};
use camel_processor::{
CircuitBreakerDecision, CircuitBreakerGate, RouteErrorHandler, invoke_processor,
};
use opentelemetry::trace::{SpanKind, Status, TraceContextExt, Tracer};
use opentelemetry::{Context as OtelContext, InstrumentationScope, KeyValue, global};
use tracing::Instrument;
use crate::lifecycle::adapters::body_coercing::wrap_if_needed;
use crate::lifecycle::adapters::step_compilers::CompiledStep;
use crate::shared::observability::adapters::TracingProcessor;
use crate::shared::observability::adapters::tracer::{
SpanEndGuard, capped_correlation_id, record_exception, step_id_for, step_span_attributes,
};
use crate::shared::observability::domain::{DetailLevel, MetricsLeversConfig};
pub(crate) use super::outcome_composition::{
BodyCoercingSegment, BoxProcessorSegment, StopSegment, compose_outcome_segment,
};
tokio::task_local! {
pub(crate) static CANCEL_TOKEN: CancellationToken;
}
#[derive(Clone)]
pub struct PipelineRuntimeCtx {
pub metrics: Arc<dyn MetricsCollector>,
pub route_id: Arc<str>,
}
impl PipelineRuntimeCtx {
pub fn compile_time() -> Self {
Self {
metrics: Arc::new(NoOpMetrics),
route_id: Arc::from(""),
}
}
}
#[derive(Clone)]
struct SharedSnapshot(Arc<[CompiledStep]>);
#[allow(dead_code)]
const _: () = {
fn assert_send<T: Send>() {}
fn assert_sync<T: Sync>() {}
fn _check() {
assert_send::<CompiledStep>();
assert_sync::<CompiledStep>();
}
};
pub fn compose_pipeline(processors: Vec<CompiledStep>, ctx: PipelineRuntimeCtx) -> BoxProcessor {
if processors.is_empty() {
return BoxProcessor::new(IdentityProcessor);
}
BoxProcessor::new(SequentialPipeline {
steps: SharedSnapshot(processors.into()),
handler: None,
ctx,
})
}
pub fn compose_pipeline_with_handler(
processors: Vec<CompiledStep>,
handler: Option<Arc<dyn RouteErrorHandler>>,
ctx: PipelineRuntimeCtx,
) -> BoxProcessor {
if processors.is_empty() {
return BoxProcessor::new(IdentityProcessor);
}
BoxProcessor::new(SequentialPipeline {
steps: SharedSnapshot(processors.into()),
handler,
ctx,
})
}
#[derive(Clone, Debug)]
pub struct TracerPipelineGating {
pub pipeline_enabled: bool,
pub spans_enabled: bool,
pub levers: MetricsLeversConfig,
}
impl TracerPipelineGating {
pub fn traced() -> Self {
Self {
pipeline_enabled: true,
spans_enabled: true,
levers: MetricsLeversConfig::default(),
}
}
pub fn off() -> Self {
Self {
pipeline_enabled: false,
spans_enabled: false,
levers: MetricsLeversConfig::default(),
}
}
}
impl From<bool> for TracerPipelineGating {
fn from(trace_enabled: bool) -> Self {
if trace_enabled {
Self::traced()
} else {
Self::off()
}
}
}
pub fn compose_traced_pipeline(
processors: Vec<CompiledStep>,
route_id: &str,
gating: impl Into<TracerPipelineGating>,
detail_level: DetailLevel,
metrics: Option<Arc<dyn MetricsCollector>>,
handler: Option<Arc<dyn RouteErrorHandler>>,
ctx: PipelineRuntimeCtx,
) -> BoxProcessor {
let gating = gating.into();
if !gating.pipeline_enabled {
return compose_pipeline_with_handler(processors, handler, ctx);
}
let wrapped: Vec<CompiledStep> = processors
.into_iter()
.enumerate()
.map(|(idx, step)| {
let (p, c, lc, lbl, kh) = match step {
CompiledStep::Process {
processor,
body_contract,
lifecycle,
label,
kind_hint,
} => (processor, body_contract, lifecycle, label, kind_hint),
CompiledStep::Stop => return CompiledStep::Stop,
CompiledStep::Segment { .. } => return step,
};
let traced = BoxProcessor::new(
TracingProcessor::new(
p,
route_id.to_string(),
idx,
detail_level.clone(),
metrics.clone(),
lbl.clone(),
kh,
)
.with_spans_enabled(gating.spans_enabled)
.with_metric_levers(gating.levers.clone()),
);
CompiledStep::Process {
processor: traced,
body_contract: c,
lifecycle: lc,
label: lbl,
kind_hint: kh,
}
})
.collect();
if !gating.spans_enabled {
return BoxProcessor::new(SequentialPipeline {
steps: SharedSnapshot(wrapped.into()),
handler,
ctx,
});
}
BoxProcessor::new(TracedPipeline {
steps: SharedSnapshot(wrapped.into()),
route_id: route_id.to_string(),
handler,
ctx,
})
}
pub fn compose_pipeline_with_contracts(
processors: Vec<CompiledStep>,
handler: Option<Arc<dyn RouteErrorHandler>>,
ctx: PipelineRuntimeCtx,
) -> BoxProcessor {
let wrapped: Vec<CompiledStep> = processors
.into_iter()
.map(|step| match step {
CompiledStep::Process {
processor,
body_contract,
lifecycle,
label,
kind_hint,
} => {
let coerced = wrap_if_needed(processor, body_contract);
CompiledStep::Process {
processor: coerced,
body_contract: None,
lifecycle,
label,
kind_hint,
}
}
CompiledStep::Stop => CompiledStep::Stop,
CompiledStep::Segment { .. } => step,
})
.collect();
compose_pipeline_with_handler(wrapped, handler, ctx)
}
pub(crate) fn compose_traced_pipeline_with_contracts(
processors: Vec<CompiledStep>,
route_id: &str,
gating: impl Into<TracerPipelineGating>,
detail_level: DetailLevel,
metrics: Option<Arc<dyn MetricsCollector>>,
handler: Option<Arc<dyn RouteErrorHandler>>,
ctx: PipelineRuntimeCtx,
) -> BoxProcessor {
let gating = gating.into();
if !gating.pipeline_enabled {
return compose_pipeline_with_contracts(processors, handler, ctx);
}
let coerced: Vec<CompiledStep> = processors
.into_iter()
.map(|step| match step {
CompiledStep::Process {
processor,
body_contract,
lifecycle,
label,
kind_hint,
} => {
let processor = wrap_if_needed(processor, body_contract);
CompiledStep::Process {
processor,
body_contract: None,
lifecycle,
label,
kind_hint,
}
}
CompiledStep::Stop => CompiledStep::Stop,
CompiledStep::Segment { .. } => step,
})
.collect();
compose_traced_pipeline(
coerced,
route_id,
gating,
detail_level,
metrics,
handler,
ctx,
)
}
#[derive(Clone)]
struct SequentialPipeline {
steps: SharedSnapshot,
handler: Option<Arc<dyn RouteErrorHandler>>,
ctx: PipelineRuntimeCtx,
}
impl Service<Exchange> for SequentialPipeline {
type Response = Exchange;
type Error = CamelError;
type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
match self.steps.0.first() {
Some(CompiledStep::Process { processor, .. }) => {
let mut proc = processor.clone();
match proc.poll_ready(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(_)) if self.handler.is_some() => Poll::Ready(Ok(())),
Poll::Ready(other) => Poll::Ready(other),
}
}
Some(CompiledStep::Stop) => Poll::Ready(Ok(())),
Some(CompiledStep::Segment { .. }) => Poll::Ready(Ok(())),
None => Poll::Ready(Ok(())),
}
}
fn call(&mut self, exchange: Exchange) -> Self::Future {
let steps = self.steps.clone();
let handler = self.handler.clone();
let ctx = self.ctx.clone();
Box::pin(async move {
run_steps(steps, exchange, handler, false, &ctx.route_id, &ctx)
.await
.into_tower_result()
})
}
}
#[derive(Clone)]
struct TracedPipeline {
steps: SharedSnapshot,
route_id: String,
handler: Option<Arc<dyn RouteErrorHandler>>,
ctx: PipelineRuntimeCtx,
}
impl Service<Exchange> for TracedPipeline {
type Response = Exchange;
type Error = CamelError;
type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
match self.steps.0.first() {
Some(CompiledStep::Process { processor, .. }) => {
let mut proc = processor.clone();
match proc.poll_ready(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Err(_)) if self.handler.is_some() => Poll::Ready(Ok(())),
Poll::Ready(other) => Poll::Ready(other),
}
}
Some(CompiledStep::Stop) => Poll::Ready(Ok(())),
Some(CompiledStep::Segment { .. }) => Poll::Ready(Ok(())),
None => Poll::Ready(Ok(())),
}
}
fn call(&mut self, exchange: Exchange) -> Self::Future {
let steps = self.steps.clone();
let route_id = self.route_id.clone();
let handler = self.handler.clone();
let ctx = self.ctx.clone();
Box::pin(async move {
let tracer = global::tracer_with_scope(
InstrumentationScope::builder("camel-core")
.with_version(env!("CARGO_PKG_VERSION"))
.build(),
);
let entry_cx = exchange.otel_context.clone();
let root_span = tracer
.span_builder(route_id.clone())
.with_kind(SpanKind::Internal)
.with_attributes([
KeyValue::new("messaging.system", "camel"),
KeyValue::new("route_id", route_id.clone()),
KeyValue::new(
"correlation_id",
capped_correlation_id(exchange.correlation_id()).to_string(),
),
])
.start_with_context(&tracer, &entry_cx);
let root_cx = entry_cx.with_span(root_span);
let _root_guard = SpanEndGuard(root_cx.clone());
let mut exchange = exchange;
exchange.otel_context = root_cx.clone();
let outcome = run_steps(steps, exchange, handler, true, &route_id, &ctx).await;
finish_span_outcome(outcome, &root_cx, entry_cx).into_tower_result()
})
}
}
async fn run_steps(
steps: SharedSnapshot,
exchange: Exchange,
handler: Option<Arc<dyn RouteErrorHandler>>,
trace: bool,
route_id: &str,
ctx: &PipelineRuntimeCtx,
) -> PipelineOutcome {
use camel_api::error_handler::RetryableStep;
let mut ex = exchange;
let len = steps.0.len();
for i in 0..len {
let cancelled = CANCEL_TOKEN.try_with(|t| t.is_cancelled()).unwrap_or(false);
if cancelled {
return PipelineOutcome::Failed(CamelError::ConsumerStopping);
}
let mut retryable: OwnedRetryable = match &steps.0[i] {
CompiledStep::Stop => return PipelineOutcome::Stopped(ex),
CompiledStep::Process { processor, .. } => OwnedRetryable::Processor(processor.clone()),
CompiledStep::Segment { segment, label, .. } => {
if trace {
OwnedRetryable::TracedSegment(TracedSegmentStep {
segment: segment.clone(),
route_id: route_id.to_string(),
index: i,
label: label.clone(),
})
} else {
OwnedRetryable::Segment(segment.clone())
}
}
};
let original = handler.as_ref().map(|_| ex.clone());
let outcome = if trace {
invoke_with_span(&mut retryable, ex, i).await
} else {
retryable.invoke(ex).await
};
match outcome {
PipelineOutcome::Completed(next) => {
if camel_api::is_camel_stop(&next) {
return PipelineOutcome::Stopped(next);
}
ex = next;
}
PipelineOutcome::Stopped(stopped_ex) => {
return PipelineOutcome::Stopped(stopped_ex);
}
PipelineOutcome::Failed(err) => {
let (Some(handler), Some(original)) = (handler.as_ref(), original) else {
return PipelineOutcome::Failed(err);
};
let policy = handler.match_policy(&err);
match handler
.retry_step(policy, &mut retryable, original, err)
.await
{
RetryOutcome::Recovered(exchange) => {
ctx.metrics.record_counter(
"pipeline_disposition",
1.0,
&[("disposition", "recovered"), ("route_id", &ctx.route_id)],
);
ex = exchange;
}
RetryOutcome::Stopped(stopped_ex) => {
ctx.metrics.record_counter(
"pipeline_disposition",
1.0,
&[("disposition", "stopped"), ("route_id", &ctx.route_id)],
);
return PipelineOutcome::Stopped(stopped_ex);
}
RetryOutcome::Exhausted {
exchange,
error,
policy,
} => {
let disposition = if trace {
handler
.handle_step(policy, exchange, error)
.instrument(tracing::debug_span!("error_handler", step_index = i))
.await
} else {
handler.handle_step(policy, exchange, error).await
};
match disposition {
Ok(StepDisposition::Propagate(e)) => {
ctx.metrics.record_counter(
"pipeline_disposition",
1.0,
&[("disposition", "propagated"), ("route_id", &ctx.route_id)],
);
return PipelineOutcome::Failed(e);
}
Ok(StepDisposition::Handled(done)) => {
ctx.metrics.record_counter(
"pipeline_disposition",
1.0,
&[("disposition", "handled"), ("route_id", &ctx.route_id)],
);
return PipelineOutcome::Completed(done);
}
Ok(StepDisposition::Continued(next)) => {
ctx.metrics.record_counter(
"pipeline_disposition",
1.0,
&[("disposition", "continued"), ("route_id", &ctx.route_id)],
);
ex = next;
}
Err(e) => {
ctx.metrics.record_counter(
"pipeline_disposition",
1.0,
&[
("disposition", "handler_error"),
("route_id", &ctx.route_id),
],
);
return PipelineOutcome::Failed(e);
}
_ => {
return PipelineOutcome::Failed(CamelError::ProcessorError(
"unknown step disposition".to_string(),
));
}
}
}
_ => {
return PipelineOutcome::Failed(CamelError::ProcessorError(
"unknown retry outcome".to_string(),
));
}
}
}
}
}
PipelineOutcome::Completed(ex)
}
enum OwnedRetryable {
Processor(camel_api::BoxProcessor),
Segment(camel_api::OutcomeSegment),
TracedSegment(TracedSegmentStep),
}
impl camel_api::error_handler::RetryableStep for OwnedRetryable {
fn invoke<'a>(
&'a mut self,
exchange: Exchange,
) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
match self {
OwnedRetryable::Processor(p) => p.invoke(exchange),
OwnedRetryable::Segment(s) => s.invoke(exchange),
OwnedRetryable::TracedSegment(s) => s.invoke(exchange),
}
}
}
fn segment_span(
tracer: &global::BoxedTracer,
route_id: &str,
index: usize,
label: Option<Arc<str>>,
entry_cx: &OtelContext,
correlation_id: &str,
) -> global::BoxedSpan {
tracer
.span_builder(format!(
"{route_id}:{}",
label.as_deref().unwrap_or(&step_id_for(index))
))
.with_kind(SpanKind::Internal)
.with_attributes(step_span_attributes(route_id, index, correlation_id))
.start_with_context(tracer, entry_cx)
}
struct TracedSegmentStep {
segment: camel_api::OutcomeSegment,
route_id: String,
index: usize,
label: Option<Arc<str>>,
}
fn finish_span_outcome(
outcome: PipelineOutcome,
span_cx: &OtelContext,
entry_cx: OtelContext,
) -> PipelineOutcome {
match outcome {
PipelineOutcome::Completed(mut ex) => {
span_cx.span().set_status(Status::Ok);
ex.otel_context = entry_cx;
PipelineOutcome::Completed(ex)
}
PipelineOutcome::Stopped(mut ex) => {
span_cx.span().set_status(Status::Ok);
ex.otel_context = entry_cx;
PipelineOutcome::Stopped(ex)
}
PipelineOutcome::Failed(e) => {
record_exception(&span_cx.span(), &e);
PipelineOutcome::Failed(e)
}
}
}
impl camel_api::error_handler::RetryableStep for TracedSegmentStep {
fn invoke<'a>(
&'a mut self,
mut exchange: Exchange,
) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
Box::pin(async move {
let tracer = global::tracer_with_scope(
InstrumentationScope::builder("camel-core")
.with_version(env!("CARGO_PKG_VERSION"))
.build(),
);
let entry_cx = exchange.otel_context.clone();
let span = segment_span(
&tracer,
&self.route_id,
self.index,
self.label.clone(),
&entry_cx,
exchange.correlation_id(),
);
let cx = entry_cx.with_span(span);
let _guard = SpanEndGuard(cx.clone());
exchange.otel_context = cx.clone();
finish_span_outcome(self.segment.run(exchange).await, &cx, entry_cx)
})
}
}
async fn invoke_with_span(
retryable: &mut dyn camel_api::error_handler::RetryableStep,
exchange: Exchange,
idx: usize,
) -> PipelineOutcome {
retryable
.invoke(exchange)
.instrument(tracing::debug_span!("pipeline_step", index = idx))
.await
}
#[derive(Clone)]
pub struct RouteChannelService {
handler: Arc<dyn RouteErrorHandler>,
security: Option<BoxProcessor>,
cb_gate: Option<CircuitBreakerGate>,
pipeline: BoxProcessor,
use_original_message: bool,
}
impl RouteChannelService {
pub fn new(
handler: Arc<dyn RouteErrorHandler>,
security: Option<BoxProcessor>,
cb_gate: Option<CircuitBreakerGate>,
pipeline: BoxProcessor,
use_original_message: bool,
) -> Self {
Self {
handler,
security,
cb_gate,
pipeline,
use_original_message,
}
}
}
impl Service<Exchange> for RouteChannelService {
type Response = Exchange;
type Error = CamelError;
type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), CamelError>> {
if let Some(ref mut sec) = self.security {
match sec.clone().poll_ready(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Err(_)) | Poll::Ready(Ok(())) => {}
}
}
match self.pipeline.clone().poll_ready(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Err(_)) | Poll::Ready(Ok(())) => {}
}
Poll::Ready(Ok(()))
}
fn call(&mut self, exchange: Exchange) -> Self::Future {
let handler = self.handler.clone();
let security = self.security.clone();
let cb_gate = self.cb_gate.clone();
let mut pipeline = self.pipeline.clone();
let use_original_message = self.use_original_message;
Box::pin(async move {
let mut ex = exchange;
if use_original_message {
let original: Arc<Message> = Arc::new(ex.input.clone());
ex.set_extension(ORIGINAL_MESSAGE_EXTENSION, original);
}
if let Some(mut sec) = security {
let original = ex.clone();
match invoke_processor(&mut sec, ex).await {
Ok(next) => ex = next,
Err(err) => {
return handler
.handle_boundary(BoundaryKind::Security, original, err)
.await;
}
}
}
if let Some(ref cb) = cb_gate {
match cb.before_call() {
CircuitBreakerDecision::Allow => { }
CircuitBreakerDecision::Fallback(mut fb) => {
let original = ex.clone();
match invoke_processor(&mut fb, ex).await {
Ok(result) => return Ok(result),
Err(err) => {
return handler
.handle_boundary(BoundaryKind::CircuitBreaker, original, err)
.await;
}
}
}
CircuitBreakerDecision::Reject(err) => {
let original = ex.clone();
return handler
.handle_boundary(BoundaryKind::CircuitBreaker, original, err)
.await;
}
}
}
let result = invoke_processor(&mut pipeline, ex).await;
if let Some(ref cb) = cb_gate {
cb.after_result(&result);
}
result
})
}
}
#[cfg(test)]
#[path = "route_compiler_tests.rs"]
mod tests;