Skip to main content

camel_processor/
error_handler.rs

1use std::future::Future;
2use std::pin::Pin;
3use std::task::{Context, Poll};
4
5use tower::{Layer, Service, ServiceExt};
6
7use camel_api::error_handler::{
8    BoundaryKind, ExceptionDisposition, ExceptionPolicy, HEADER_REDELIVERED,
9    HEADER_REDELIVERY_COUNTER, HEADER_REDELIVERY_MAX_COUNTER, PolicyId, RetryOutcome,
10    StepDisposition,
11};
12use camel_api::{BoxProcessor, CamelError, Exchange, PipelineOutcome, SyncBoxProcessor, Value};
13
14async fn execute_on_steps(
15    original: Exchange,
16    original_err: CamelError,
17    on_steps: &SyncBoxProcessor,
18    disposition: ExceptionDisposition,
19    handler: Option<BoxProcessor>,
20) -> Result<Exchange, CamelError> {
21    let snapshot = original.clone();
22    let mut ex = original;
23    ex.set_error(original_err.clone());
24    let mut pipeline = on_steps.clone_inner();
25    let step_result = async {
26        let svc = pipeline.ready().await?;
27        svc.call(ex).await
28    }
29    .await;
30
31    match step_result {
32        Ok(mut ex) => {
33            if disposition == ExceptionDisposition::Handled {
34                ex.handle_error();
35                Ok(ex)
36            } else {
37                // Propagate or Continued — steps execute for side-effects (e.g. logging) but
38                // the modified exchange is discarded and the original error propagated.
39                Err(original_err)
40            }
41        }
42        Err(_) => {
43            // log-policy: handler-owned
44            tracing::warn!(error = %original_err, "on_steps pipeline failed, falling back to handler/DLC");
45            let mut ex = snapshot;
46            ex.set_error(original_err.clone());
47            forward_or_propagate(ex, handler, original_err).await
48        }
49    }
50}
51
52/// Invoke a processor: readiness check + call, unified into a single Result.
53///
54/// Readiness errors and call errors are both returned as `Err(CamelError)`,
55/// allowing the pipeline's recovery loop to handle them uniformly.
56pub async fn invoke_processor(
57    svc: &mut BoxProcessor,
58    ex: Exchange,
59) -> Result<Exchange, CamelError> {
60    match svc.ready().await {
61        Ok(ready) => ready.call(ex).await,
62        Err(err) => Err(err),
63    }
64}
65
66/// Route-level error handler owning ALL error handling logic.
67///
68/// Single owner of DLC, retry, onException policies. Called from
69/// `RouteChannelService` (boundary errors) and `run_steps` (step errors).
70#[async_trait::async_trait]
71pub trait RouteErrorHandler: Send + Sync {
72    /// Match a policy for the given error. Called once before retry.
73    fn match_policy(&self, err: &CamelError) -> Option<PolicyId>;
74
75    /// Phase 1: Retry the failed step.
76    async fn retry_step(
77        &self,
78        policy: Option<PolicyId>,
79        step: &mut dyn camel_api::error_handler::RetryableStep,
80        original: Exchange,
81        error: CamelError,
82    ) -> RetryOutcome;
83
84    /// Phase 2: Determine step disposition after retry exhaustion.
85    async fn handle_step(
86        &self,
87        policy: Option<PolicyId>,
88        exchange: Exchange,
89        error: CamelError,
90    ) -> Result<StepDisposition, CamelError>;
91
92    /// Handle boundary (infrastructure) errors.
93    async fn handle_boundary(
94        &self,
95        kind: BoundaryKind,
96        exchange: Exchange,
97        error: CamelError,
98    ) -> Result<Exchange, CamelError>;
99}
100
101/// Default implementation of RouteErrorHandler.
102/// Owns DLC producer exclusively. Encapsulates retry/onException/DLC logic.
103///
104/// Uses `SyncBoxProcessor` internally so the handler is `Send + Sync` as required
105/// by the `RouteErrorHandler` trait.
106pub struct DefaultRouteErrorHandler {
107    pub(crate) dlc_producer: Option<SyncBoxProcessor>,
108    pub(crate) policies: Vec<(ExceptionPolicy, Option<SyncBoxProcessor>)>,
109    /// When true, restore the original Message (body and headers, pre-route, pre-mutation)
110    /// before forwarding to the DLC/handler. The original message is stashed by
111    /// `RouteChannelService::call` as the `ORIGINAL_MESSAGE_EXTENSION` extension.
112    pub use_original_message: bool,
113}
114
115impl DefaultRouteErrorHandler {
116    pub fn new(
117        dlc_producer: Option<BoxProcessor>,
118        policies: Vec<(ExceptionPolicy, Option<BoxProcessor>)>,
119    ) -> Self {
120        Self {
121            dlc_producer: dlc_producer.map(SyncBoxProcessor::new),
122            policies: policies
123                .into_iter()
124                .map(|(p, prod)| (p, prod.map(SyncBoxProcessor::new)))
125                .collect(),
126            use_original_message: false,
127        }
128    }
129
130    /// Enable restoration of the original message before forwarding to DLC.
131    pub fn with_use_original_message(mut self, enabled: bool) -> Self {
132        self.use_original_message = enabled;
133        self
134    }
135
136    /// Restore the original message from the `CamelOriginalMessage` extension if
137    /// `use_original_message` is enabled. Called immediately before `send_to_handler`
138    /// dispatch in both `handle_step` and `handle_boundary`.
139    fn restore_original_message_if_enabled(&self, exchange: &mut Exchange) {
140        if self.use_original_message
141            && let Some(orig) =
142                exchange.get_extension::<camel_api::Message>(camel_api::ORIGINAL_MESSAGE_EXTENSION)
143        {
144            exchange.input = orig.clone();
145        }
146    }
147
148    /// Resolve (disposition, producer) for a matched policy.
149    /// Shared by handle_step and handle_boundary.
150    fn resolve_producer(
151        &self,
152        policy: Option<PolicyId>,
153    ) -> (ExceptionDisposition, Option<BoxProcessor>) {
154        match policy {
155            Some(PolicyId(idx)) => match self.policies.get(idx) {
156                Some((p, prod)) => (
157                    p.disposition,
158                    prod.as_ref()
159                        .map(|p| p.clone_inner())
160                        .or_else(|| self.dlc_producer.as_ref().map(|p| p.clone_inner())),
161                ),
162                None => (
163                    ExceptionDisposition::Propagate,
164                    self.dlc_producer.as_ref().map(|p| p.clone_inner()),
165                ),
166            },
167            None => (
168                ExceptionDisposition::Propagate,
169                self.dlc_producer.as_ref().map(|p| p.clone_inner()),
170            ),
171        }
172    }
173}
174
175#[async_trait::async_trait]
176impl RouteErrorHandler for DefaultRouteErrorHandler {
177    fn match_policy(&self, err: &CamelError) -> Option<PolicyId> {
178        self.policies
179            .iter()
180            .position(|(p, _)| (p.matches)(err))
181            .map(PolicyId)
182    }
183
184    async fn retry_step(
185        &self,
186        policy: Option<PolicyId>,
187        step: &mut dyn camel_api::error_handler::RetryableStep,
188        original: Exchange,
189        error: CamelError,
190    ) -> RetryOutcome {
191        let Some(PolicyId(idx)) = policy else {
192            return RetryOutcome::Exhausted {
193                exchange: original,
194                error,
195                policy: None,
196            };
197        };
198        let Some((policy_def, _)) = self.policies.get(idx) else {
199            return RetryOutcome::Exhausted {
200                exchange: original,
201                error,
202                policy,
203            };
204        };
205        let Some(ref backoff) = policy_def.retry else {
206            return RetryOutcome::Exhausted {
207                exchange: original,
208                error,
209                policy,
210            };
211        };
212
213        for attempt in 0..backoff.max_attempts {
214            let delay = backoff.delay_for(attempt);
215            tokio::time::sleep(delay).await;
216
217            let mut ex = original.clone();
218            ex.input.set_header(HEADER_REDELIVERED, Value::Bool(true));
219            ex.input.set_header(
220                HEADER_REDELIVERY_COUNTER,
221                Value::Number((attempt + 1).into()),
222            );
223            ex.input.set_header(
224                HEADER_REDELIVERY_MAX_COUNTER,
225                Value::Number(backoff.max_attempts.into()),
226            );
227
228            match step.invoke(ex).await {
229                PipelineOutcome::Completed(exchange) => {
230                    return RetryOutcome::Recovered(exchange);
231                }
232                PipelineOutcome::Stopped(stopped_ex) => {
233                    return RetryOutcome::Stopped(stopped_ex);
234                }
235                PipelineOutcome::Failed(retry_err) => {
236                    if attempt + 1 == backoff.max_attempts {
237                        let mut final_ex = original;
238                        final_ex
239                            .input
240                            .set_header(HEADER_REDELIVERED, Value::Bool(true));
241                        final_ex.input.set_header(
242                            HEADER_REDELIVERY_COUNTER,
243                            Value::Number(backoff.max_attempts.into()),
244                        );
245                        final_ex.input.set_header(
246                            HEADER_REDELIVERY_MAX_COUNTER,
247                            Value::Number(backoff.max_attempts.into()),
248                        );
249                        return RetryOutcome::Exhausted {
250                            exchange: final_ex,
251                            error: retry_err,
252                            policy,
253                        };
254                    }
255                }
256            }
257        }
258
259        RetryOutcome::Exhausted {
260            exchange: original,
261            error,
262            policy,
263        }
264    }
265
266    async fn handle_step(
267        &self,
268        policy: Option<PolicyId>,
269        mut exchange: Exchange,
270        error: CamelError,
271    ) -> Result<StepDisposition, CamelError> {
272        let (disposition, producer) = self.resolve_producer(policy);
273
274        // rc-fu1of: a non-matching policy with no DLC silently propagates —
275        // the operator sees "handler never ran" with no signal why. Emit a
276        // diagnostic naming the error kind so `kind:` vocabulary gaps are
277        // discoverable. debug! (not warn!): non-matching is an expected,
278        // selective-policy path, and this fires per failed exchange.
279        if policy.is_none() && producer.is_none() {
280            tracing::debug!(
281                kind = %error.variant_name(),
282                "no on_exceptions policy matched and no dead-letter channel configured; propagating error"
283            );
284        }
285
286        // Run on_steps if present (using the SAME policy identified by PolicyId).
287        // Skip on_steps for Propagate disposition to prevent double side-effects:
288        // on_steps results would be discarded (snapshot restored), and the DLC
289        // handler fires next — causing duplicate message production.
290        if !matches!(disposition, ExceptionDisposition::Propagate)
291            && let Some(PolicyId(idx)) = policy
292            && let Some((p, _)) = self.policies.get(idx)
293            && let Some(ref steps) = p.on_steps
294        {
295            let snapshot = exchange.clone();
296            exchange.set_error(error.clone());
297            let mut step_pipeline = steps.clone_inner();
298            let step_result = async {
299                let svc = step_pipeline.ready().await?;
300                svc.call(exchange).await
301            }
302            .await;
303            match step_result {
304                Ok(mut ex) => match disposition {
305                    ExceptionDisposition::Handled => {
306                        ex.handle_error();
307                        return Ok(StepDisposition::Handled(ex));
308                    }
309                    ExceptionDisposition::Continued => {
310                        ex.clear_error();
311                        return Ok(StepDisposition::Continued(ex));
312                    }
313                    // Propagate and any future variant restore the snapshot
314                    // and fall through to the DLC/handler path.
315                    _ => {
316                        exchange = snapshot;
317                    }
318                },
319                Err(_) => {
320                    exchange = snapshot;
321                }
322            }
323        }
324
325        // No on_steps, on_steps failed, or Propagate — forward to DLC/handler.
326        // BIND the returned exchange (must use handler output).
327        self.restore_original_message_if_enabled(&mut exchange);
328        exchange.set_error(error.clone());
329        match send_to_handler(exchange, producer).await {
330            Ok(handler_ex) => match disposition {
331                ExceptionDisposition::Handled => {
332                    let mut ex = handler_ex;
333                    ex.clear_error();
334                    Ok(StepDisposition::Handled(ex))
335                }
336                ExceptionDisposition::Continued => {
337                    let mut ex = handler_ex;
338                    ex.clear_error();
339                    Ok(StepDisposition::Continued(ex))
340                }
341                // Propagate and any future variant forward the error.
342                _ => Ok(StepDisposition::Propagate(error)),
343            },
344            // Delegate failure (rc-ntpof): the ORIGINAL step error
345            // propagates — a failed delegate must never surface as a
346            // successful Handled/Continued disposition.
347            Err(_delegate) => Ok(StepDisposition::Propagate(error)),
348        }
349    }
350
351    async fn handle_boundary(
352        &self,
353        boundary_kind: BoundaryKind,
354        mut exchange: Exchange,
355        error: CamelError,
356    ) -> Result<Exchange, CamelError> {
357        // Boundary errors: match policy, run on_steps, forward to DLC.
358        // Disposition mapping:
359        //   Handled → clear error, return Ok(exchange)
360        //   Propagate | Continued → forward to DLC, return Ok(exchange_with_error)
361        //   (Continued at boundary = Propagate — no next step to continue to)
362        let policy = self.match_policy(&error);
363        let (disposition, producer) = self.resolve_producer(policy);
364
365        // rc-fu1of: a non-matching policy with no DLC silently propagates —
366        // the operator sees "handler never ran" with no signal why. Emit a
367        // diagnostic naming the error kind so `kind:` vocabulary gaps are
368        // discoverable. debug! (not warn!): non-matching is an expected,
369        // selective-policy path, and this fires per failed exchange.
370        // (Parity with handle_step; boundary adds which gate raised the error.)
371        if policy.is_none() && producer.is_none() {
372            tracing::debug!(
373                boundary = ?boundary_kind,
374                kind = %error.variant_name(),
375                "no on_exceptions policy matched and no dead-letter channel configured; propagating error"
376            );
377        }
378
379        // Run on_steps if present (shared logic with handle_step).
380        // Skip on_steps for Propagate/Continued disposition to prevent double
381        // side-effects: on_steps results would be discarded (snapshot restored),
382        // and the DLC handler fires next — causing duplicate message production.
383        // At boundary level, Continued maps to Propagate semantics.
384        if !matches!(
385            disposition,
386            ExceptionDisposition::Propagate | ExceptionDisposition::Continued
387        ) && let Some(PolicyId(idx)) = policy
388            && let Some((p, _)) = self.policies.get(idx)
389            && let Some(ref steps) = p.on_steps
390        {
391            let snapshot = exchange.clone();
392            exchange.set_error(error.clone());
393            let mut step_pipeline = steps.clone_inner();
394            let step_result = async {
395                let svc = step_pipeline.ready().await?;
396                svc.call(exchange).await
397            }
398            .await;
399            match step_result {
400                Ok(mut ex) => match disposition {
401                    ExceptionDisposition::Handled => {
402                        ex.handle_error();
403                        return Ok(ex);
404                    }
405                    // Propagate | Continued and any future variant restore the snapshot.
406                    _ => {
407                        exchange = snapshot;
408                    }
409                },
410                Err(_) => {
411                    exchange = snapshot;
412                }
413            }
414        }
415
416        // Forward to DLC/handler — BIND returned exchange
417        self.restore_original_message_if_enabled(&mut exchange);
418        exchange.set_error(error.clone());
419        match send_to_handler(exchange, producer).await {
420            Ok(handler_ex) => match disposition {
421                ExceptionDisposition::Handled => {
422                    let mut ex = handler_ex;
423                    ex.clear_error();
424                    Ok(ex)
425                }
426                // Propagate | Continued and any future variant forward the error.
427                _ => {
428                    let mut ex = handler_ex;
429                    ex.set_error(error);
430                    Ok(ex)
431                }
432            },
433            // Delegate failure (rc-ntpof): the ORIGINAL boundary error
434            // propagates — the delegate error is only logged and recorded
435            // on the span, never replaces the original.
436            Err(_delegate) => Err(error),
437        }
438    }
439}
440
441/// Tower Layer that wraps a pipeline with error handling behaviour.
442///
443/// Constructed with already-resolved producers; URI resolution happens in `camel-core`.
444pub struct ErrorHandlerLayer {
445    /// Resolved DLC producer (None = log only).
446    dlc_producer: Option<BoxProcessor>,
447    /// Policies with their resolved `handled_by` producers.
448    policies: Vec<(ExceptionPolicy, Option<BoxProcessor>)>,
449}
450
451impl ErrorHandlerLayer {
452    /// Create the layer with pre-resolved producers.
453    pub fn new(
454        dlc_producer: Option<BoxProcessor>,
455        policies: Vec<(ExceptionPolicy, Option<BoxProcessor>)>,
456    ) -> Self {
457        Self {
458            dlc_producer,
459            policies,
460        }
461    }
462}
463
464impl<S> Layer<S> for ErrorHandlerLayer
465where
466    S: Service<Exchange, Response = Exchange, Error = CamelError> + Send + Clone + 'static,
467    S::Future: Send + 'static,
468{
469    type Service = ErrorHandlerService<S>;
470
471    fn layer(&self, inner: S) -> Self::Service {
472        ErrorHandlerService {
473            inner,
474            dlc_producer: self.dlc_producer.clone(),
475            policies: self
476                .policies
477                .iter()
478                .map(|(p, prod)| (p.clone(), prod.clone()))
479                .collect(),
480        }
481    }
482}
483
484/// Tower Service that absorbs pipeline errors by retrying and/or forwarding to a DLC.
485///
486/// Pipeline errors are absorbed: the returned `Ok` exchange will have
487/// `has_error() == true` if the pipeline ultimately failed. A DELEGATE
488/// failure (DLC/handler readiness or call error) is the one exception: it
489/// surfaces as `Err(original_error)` so a broken delegate can never mask
490/// the original failure as success (rc-ntpof).
491pub struct ErrorHandlerService<S> {
492    inner: S,
493    dlc_producer: Option<BoxProcessor>,
494    policies: Vec<(ExceptionPolicy, Option<BoxProcessor>)>,
495}
496
497impl<S: Clone> Clone for ErrorHandlerService<S> {
498    fn clone(&self) -> Self {
499        Self {
500            inner: self.inner.clone(),
501            dlc_producer: self.dlc_producer.clone(),
502            policies: self
503                .policies
504                .iter()
505                .map(|(p, prod)| (p.clone(), prod.clone()))
506                .collect(),
507        }
508    }
509}
510
511impl<S> ErrorHandlerService<S>
512where
513    S: Service<Exchange, Response = Exchange, Error = CamelError> + Send + Clone + 'static,
514    S::Future: Send + 'static,
515{
516    /// Create the service directly (used in unit tests; in production use the Layer).
517    pub fn new(
518        inner: S,
519        dlc_producer: Option<BoxProcessor>,
520        policies: Vec<(ExceptionPolicy, Option<BoxProcessor>)>,
521    ) -> Self {
522        Self {
523            inner,
524            dlc_producer,
525            policies,
526        }
527    }
528}
529
530impl<S> Service<Exchange> for ErrorHandlerService<S>
531where
532    S: Service<Exchange, Response = Exchange, Error = CamelError> + Send + Clone + 'static,
533    S::Future: Send + 'static,
534{
535    type Response = Exchange;
536    type Error = CamelError;
537    type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
538
539    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
540        // Preserve backpressure (Pending) but never leak readiness errors upward.
541        // Readiness errors are deferred to call(), where they go through the same
542        // retry/onException/DLC path as call() errors. This is safe because call()
543        // re-checks readiness on a fresh inner clone via inner.ready().await.
544        match self.inner.poll_ready(cx) {
545            Poll::Pending => Poll::Pending,
546            Poll::Ready(Err(_)) => Poll::Ready(Ok(())),
547            Poll::Ready(Ok(())) => Poll::Ready(Ok(())),
548        }
549    }
550
551    fn call(&mut self, exchange: Exchange) -> Self::Future {
552        let mut inner = self.inner.clone();
553        let dlc = self.dlc_producer.clone();
554        let policies: Vec<(ExceptionPolicy, Option<BoxProcessor>)> = self
555            .policies
556            .iter()
557            .map(|(p, prod)| (p.clone(), prod.clone()))
558            .collect();
559
560        Box::pin(async move {
561            let original = exchange.clone();
562            let result = match inner.ready().await {
563                Ok(svc) => svc.call(exchange).await,
564                Err(e) => Err(e), // readiness error — enters retry/DLC/onException path below
565            };
566
567            let err = match result {
568                Ok(ex) => return Ok(ex),
569                Err(e) => e,
570            };
571
572            // Find the first matching policy.
573            let matched = policies.into_iter().find(|(p, _)| (p.matches)(&err));
574
575            if let Some((policy, policy_producer)) = matched {
576                // Retry if configured.
577                if let Some(ref backoff) = policy.retry {
578                    for attempt in 0..backoff.max_attempts {
579                        let delay = backoff.delay_for(attempt);
580                        tokio::time::sleep(delay).await;
581
582                        // Set redelivery headers
583                        let mut ex = original.clone();
584                        ex.input.set_header(HEADER_REDELIVERED, Value::Bool(true));
585                        ex.input.set_header(
586                            HEADER_REDELIVERY_COUNTER,
587                            Value::Number((attempt + 1).into()),
588                        );
589                        ex.input.set_header(
590                            HEADER_REDELIVERY_MAX_COUNTER,
591                            Value::Number(backoff.max_attempts.into()),
592                        );
593
594                        let result = match inner.ready().await {
595                            Ok(svc) => svc.call(ex).await,
596                            Err(e) => Err(e), // readiness error — enters retry exhaustion path
597                        };
598                        match result {
599                            Ok(ex) => return Ok(ex),
600                            Err(retry_err) => {
601                                if attempt + 1 == backoff.max_attempts {
602                                    // Retries exhausted — send to handler.
603                                    let mut original = original.clone();
604                                    original
605                                        .input
606                                        .set_header(HEADER_REDELIVERED, Value::Bool(true));
607                                    original.input.set_header(
608                                        HEADER_REDELIVERY_COUNTER,
609                                        Value::Number(backoff.max_attempts.into()),
610                                    );
611                                    original.input.set_header(
612                                        HEADER_REDELIVERY_MAX_COUNTER,
613                                        Value::Number(backoff.max_attempts.into()),
614                                    );
615                                    if let Some(ref steps) = policy.on_steps {
616                                        let handler = policy_producer.clone().or(dlc.clone());
617                                        return execute_on_steps(
618                                            original,
619                                            retry_err,
620                                            steps,
621                                            policy.disposition,
622                                            handler,
623                                        )
624                                        .await;
625                                    }
626                                    original.set_error(retry_err.clone());
627                                    let handler = policy_producer.or(dlc);
628                                    return forward_or_propagate(original, handler, retry_err)
629                                        .await;
630                                }
631                            }
632                        }
633                    }
634                }
635                // No retry configured (or 0 attempts) — send to policy handler or DLC.
636                if let Some(ref steps) = policy.on_steps {
637                    let handler = policy_producer.or(dlc);
638                    return execute_on_steps(original, err, steps, policy.disposition, handler)
639                        .await;
640                }
641                let mut ex = original.clone();
642                ex.set_error(err.clone());
643                let handler = policy_producer.or(dlc);
644                forward_or_propagate(ex, handler, err).await
645            } else {
646                // No matching policy — forward directly to DLC.
647                let mut ex = original;
648                ex.set_error(err.clone());
649                forward_or_propagate(ex, dlc, err).await
650            }
651        })
652    }
653}
654
655/// Record the delegate error on the current span, if one is active and
656/// declares an `error` field. The span carries the DELEGATE error while the
657/// system-broken log structures BOTH errors (rc-ntpof).
658///
659/// Shared with the `do_try` arms (bd rc-zgbqq): the catch-failure envelope
660/// records the catch error on the active span the same way.
661pub(crate) fn record_span_error(delegate_err: &CamelError) {
662    let span = tracing::Span::current();
663    if !span.is_none() {
664        span.record("error", tracing::field::display(delegate_err));
665    }
666}
667
668/// Forward a failed exchange to the DLC/handler delegate, mapping the
669/// result so the ORIGINAL error always wins (rc-ntpof).
670///
671/// A DELEGATE failure (readiness or call error) is logged and recorded on
672/// the span inside `send_to_handler`; here it is discarded and
673/// `original_error` propagated instead — a failed delegate must never
674/// surface as a successful exchange nor replace the original error.
675async fn forward_or_propagate(
676    exchange: Exchange,
677    producer: Option<BoxProcessor>,
678    original_error: CamelError,
679) -> Result<Exchange, CamelError> {
680    match send_to_handler(exchange, producer).await {
681        Ok(ex) => Ok(ex),
682        Err(_delegate) => Err(original_error),
683    }
684}
685
686async fn send_to_handler(
687    exchange: Exchange,
688    producer: Option<BoxProcessor>,
689) -> Result<Exchange, CamelError> {
690    match producer {
691        None => {
692            // log-policy: system-broken
693            tracing::error!(
694                error = ?exchange.error,
695                "Exchange failed with no error handler configured"
696            );
697            Ok(exchange)
698        }
699        Some(mut prod) => match prod.ready().await {
700            Err(e) => {
701                // BOTH the original exchange error and the delegate
702                // failure are structured; the span records the DELEGATE
703                // error (rc-ntpof).
704                // log-policy: system-broken
705                tracing::error!(
706                    original_error = ?exchange.error,
707                    delegate_error = %e,
708                    "DLC/handler not ready"
709                );
710                record_span_error(&e);
711                Err(e)
712            }
713            Ok(svc) => match svc.call(exchange.clone()).await {
714                Ok(ex) => Ok(ex),
715                Err(e) => {
716                    // BOTH the original exchange error and the delegate
717                    // failure are structured; the span records the
718                    // DELEGATE error (rc-ntpof).
719                    // log-policy: system-broken
720                    tracing::error!(
721                        original_error = ?exchange.error,
722                        delegate_error = %e,
723                        "DLC/handler call failed"
724                    );
725                    record_span_error(&e);
726                    // The delegate error is returned to the caller so no
727                    // path can mistake a failed delegate for success.
728                    Err(e)
729                }
730            },
731        },
732    }
733}
734
735#[cfg(test)]
736mod tests {
737    use super::*;
738    use camel_api::{
739        BoxProcessor, BoxProcessorExt, CamelError, Exchange, Message, OutcomePipeline,
740        OutcomeSegment, PipelineOutcome, RetryableStep, SyncBoxProcessor, Value,
741        error_handler::RedeliveryPolicy,
742    };
743    use std::future::Future;
744    use std::pin::Pin;
745    use std::sync::{
746        Arc,
747        atomic::{AtomicU32, Ordering},
748    };
749    use std::time::Duration;
750    use tower::ServiceExt;
751
752    fn make_exchange() -> Exchange {
753        Exchange::new(Message::new("test"))
754    }
755
756    fn failing_processor() -> BoxProcessor {
757        BoxProcessor::from_fn(|_ex| {
758            Box::pin(async { Err(CamelError::ProcessorError("boom".into())) })
759        })
760    }
761
762    fn ok_processor() -> BoxProcessor {
763        BoxProcessor::from_fn(|ex| Box::pin(async move { Ok(ex) }))
764    }
765
766    fn fail_n_times(n: u32) -> BoxProcessor {
767        let count = Arc::new(AtomicU32::new(0));
768        BoxProcessor::from_fn(move |ex| {
769            let count = Arc::clone(&count);
770            Box::pin(async move {
771                let c = count.fetch_add(1, Ordering::SeqCst);
772                if c < n {
773                    Err(CamelError::ProcessorError(format!("attempt {c}")))
774                } else {
775                    Ok(ex)
776                }
777            })
778        })
779    }
780
781    #[tokio::test]
782    async fn test_ok_passthrough() {
783        let svc = ErrorHandlerService::new(ok_processor(), None, vec![]);
784        let result = svc.oneshot(make_exchange()).await;
785        assert!(result.is_ok());
786        assert!(!result.unwrap().has_error());
787    }
788
789    #[tokio::test]
790    async fn test_error_goes_to_dlc() {
791        let received = Arc::new(std::sync::Mutex::new(Vec::<Exchange>::new()));
792        let received_clone = Arc::clone(&received);
793        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
794            let r = Arc::clone(&received_clone);
795            Box::pin(async move {
796                r.lock().unwrap().push(ex.clone());
797                Ok(ex)
798            })
799        });
800
801        let svc = ErrorHandlerService::new(failing_processor(), Some(dlc), vec![]);
802        let result = svc.oneshot(make_exchange()).await;
803        assert!(result.is_ok());
804        let ex = result.unwrap();
805        assert!(ex.has_error());
806        assert_eq!(received.lock().unwrap().len(), 1);
807    }
808
809    #[tokio::test]
810    async fn test_retry_recovers() {
811        let inner = fail_n_times(2);
812        let policy = ExceptionPolicy {
813            matches: Arc::new(|_| true),
814            retry: Some(RedeliveryPolicy {
815                max_attempts: 3,
816                initial_delay: Duration::from_millis(1),
817                multiplier: 1.0,
818                max_delay: Duration::from_millis(10),
819                jitter_factor: 0.0,
820            }),
821            handled_by: None,
822            on_steps: None,
823            disposition: ExceptionDisposition::Propagate,
824        };
825        let svc = ErrorHandlerService::new(inner, None, vec![(policy, None)]);
826        let result = svc.oneshot(make_exchange()).await;
827        assert!(result.is_ok());
828        assert!(!result.unwrap().has_error());
829    }
830
831    #[tokio::test]
832    async fn test_retry_exhausted_goes_to_dlc() {
833        let inner = fail_n_times(10);
834        let received = Arc::new(std::sync::Mutex::new(0u32));
835        let received_clone = Arc::clone(&received);
836        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
837            let r = Arc::clone(&received_clone);
838            Box::pin(async move {
839                *r.lock().unwrap() += 1;
840                Ok(ex)
841            })
842        });
843        let policy = ExceptionPolicy {
844            matches: Arc::new(|_| true),
845            retry: Some(RedeliveryPolicy {
846                max_attempts: 2,
847                initial_delay: Duration::from_millis(1),
848                multiplier: 1.0,
849                max_delay: Duration::from_millis(10),
850                jitter_factor: 0.0,
851            }),
852            handled_by: None,
853            on_steps: None,
854            disposition: ExceptionDisposition::Propagate,
855        };
856        let svc = ErrorHandlerService::new(inner, Some(dlc), vec![(policy, None)]);
857        let result = svc.oneshot(make_exchange()).await;
858        assert!(result.is_ok());
859        assert!(result.unwrap().has_error());
860        assert_eq!(*received.lock().unwrap(), 1);
861    }
862
863    #[test]
864    fn test_poll_ready_delegates_to_inner() {
865        use std::sync::atomic::AtomicBool;
866
867        /// A service that returns `Pending` on the first `poll_ready`, then `Ready`.
868        #[derive(Clone)]
869        struct DelayedReadyService {
870            ready: Arc<AtomicBool>,
871        }
872
873        impl Service<Exchange> for DelayedReadyService {
874            type Response = Exchange;
875            type Error = CamelError;
876            type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
877
878            fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
879                if self.ready.fetch_or(true, Ordering::SeqCst) {
880                    // Already marked ready (second+ call) → Ready
881                    Poll::Ready(Ok(()))
882                } else {
883                    // First call → Pending, schedule a wake
884                    cx.waker().wake_by_ref();
885                    Poll::Pending
886                }
887            }
888
889            fn call(&mut self, ex: Exchange) -> Self::Future {
890                Box::pin(async move { Ok(ex) })
891            }
892        }
893
894        let waker = futures::task::noop_waker();
895        let mut cx = Context::from_waker(&waker);
896
897        let inner = DelayedReadyService {
898            ready: Arc::new(AtomicBool::new(false)),
899        };
900        let mut svc = ErrorHandlerService::new(inner, None, vec![]);
901
902        // First poll_ready: inner returns Pending, so ErrorHandlerService must too.
903        let first = Pin::new(&mut svc).poll_ready(&mut cx);
904        assert!(first.is_pending(), "expected Pending on first poll_ready");
905
906        // Second poll_ready: inner returns Ready, so ErrorHandlerService must too.
907        let second = Pin::new(&mut svc).poll_ready(&mut cx);
908        assert!(second.is_ready(), "expected Ready on second poll_ready");
909    }
910
911    #[tokio::test]
912    async fn test_no_matching_policy_uses_dlc() {
913        let received = Arc::new(std::sync::Mutex::new(0u32));
914        let received_clone = Arc::clone(&received);
915        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
916            let r = Arc::clone(&received_clone);
917            Box::pin(async move {
918                *r.lock().unwrap() += 1;
919                Ok(ex)
920            })
921        });
922        let policy = ExceptionPolicy::new(|e| matches!(e, CamelError::Io(_)));
923        let svc = ErrorHandlerService::new(failing_processor(), Some(dlc), vec![(policy, None)]);
924        let result = svc.oneshot(make_exchange()).await;
925        assert!(result.is_ok());
926        assert_eq!(*received.lock().unwrap(), 1);
927    }
928
929    #[tokio::test]
930    async fn test_redelivery_headers_are_set() {
931        use camel_api::error_handler::{
932            HEADER_REDELIVERED, HEADER_REDELIVERY_COUNTER, HEADER_REDELIVERY_MAX_COUNTER,
933            RedeliveryPolicy,
934        };
935
936        let inner = fail_n_times(10);
937        let received = Arc::new(std::sync::Mutex::new(None));
938        let received_clone = Arc::clone(&received);
939        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
940            let r = Arc::clone(&received_clone);
941            Box::pin(async move {
942                *r.lock().unwrap() = Some(ex.clone());
943                Ok(ex)
944            })
945        });
946
947        let policy = ExceptionPolicy {
948            matches: Arc::new(|_| true),
949            retry: Some(RedeliveryPolicy {
950                max_attempts: 2,
951                initial_delay: Duration::from_millis(1),
952                multiplier: 1.0,
953                max_delay: Duration::from_millis(10),
954                jitter_factor: 0.0,
955            }),
956            handled_by: None,
957            on_steps: None,
958            disposition: ExceptionDisposition::Propagate,
959        };
960
961        let svc = ErrorHandlerService::new(inner, Some(dlc), vec![(policy, None)]);
962        let _ = svc.oneshot(make_exchange()).await.unwrap();
963
964        let ex = received.lock().unwrap().take().unwrap();
965        assert_eq!(
966            ex.input.header(HEADER_REDELIVERED),
967            Some(&Value::Bool(true))
968        );
969        assert_eq!(
970            ex.input.header(HEADER_REDELIVERY_COUNTER),
971            Some(&Value::Number(2.into()))
972        );
973        assert_eq!(
974            ex.input.header(HEADER_REDELIVERY_MAX_COUNTER),
975            Some(&Value::Number(2.into()))
976        );
977    }
978
979    #[tokio::test]
980    async fn test_jitter_produces_varying_delays_in_retry_flow() {
981        use std::time::Instant;
982
983        let inner = fail_n_times(10);
984        let received = Arc::new(std::sync::Mutex::new(None));
985        let received_clone = Arc::clone(&received);
986        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
987            let r = Arc::clone(&received_clone);
988            Box::pin(async move {
989                *r.lock().unwrap() = Some(ex.clone());
990                Ok(ex)
991            })
992        });
993
994        let policy = ExceptionPolicy {
995            matches: Arc::new(|_| true),
996            retry: Some(RedeliveryPolicy {
997                max_attempts: 5,
998                initial_delay: Duration::from_millis(20),
999                multiplier: 1.0,
1000                max_delay: Duration::from_millis(100),
1001                jitter_factor: 0.5,
1002            }),
1003            handled_by: None,
1004            on_steps: None,
1005            disposition: ExceptionDisposition::Propagate,
1006        };
1007
1008        let start = Instant::now();
1009        let svc = ErrorHandlerService::new(inner, Some(dlc), vec![(policy, None)]);
1010        let _ = svc.oneshot(make_exchange()).await.unwrap();
1011        let elapsed = start.elapsed();
1012
1013        assert!(
1014            received.lock().unwrap().is_some(),
1015            "DLC should have received exchange"
1016        );
1017
1018        assert!(
1019            elapsed >= Duration::from_millis(50),
1020            "5 retries with 20ms base delay should take at least 50ms (with jitter low bound)"
1021        );
1022
1023        assert!(
1024            elapsed <= Duration::from_millis(500),
1025            "5 retries with 20ms base delay + 50% jitter should not exceed 500ms"
1026        );
1027    }
1028
1029    #[tokio::test]
1030    async fn test_on_steps_handled_true_consumes_error() {
1031        use tower::ServiceExt;
1032
1033        let steps_pipeline = BoxProcessor::new(tower::service_fn(|mut ex: Exchange| {
1034            ex.input.body = camel_api::Body::Bytes("handled".into());
1035            async move { Ok(ex) }
1036        }));
1037        let policy = ExceptionPolicy {
1038            matches: Arc::new(|_| true),
1039            retry: None,
1040            handled_by: None,
1041            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1042            disposition: ExceptionDisposition::Handled,
1043        };
1044        let inner = tower::service_fn(|_ex: Exchange| async {
1045            Err::<Exchange, CamelError>(CamelError::RouteError("fail".to_string()))
1046        });
1047        let mut svc = ErrorHandlerService::new(inner, None, vec![(policy, None)]);
1048        let ex = Exchange::default();
1049        let result = svc.ready().await.unwrap().call(ex).await.unwrap();
1050        assert!(result.error.is_none(), "handled:true should clear error");
1051        assert!(matches!(result.input.body, camel_api::Body::Bytes(_)));
1052    }
1053
1054    #[tokio::test]
1055    async fn test_on_steps_handled_false_propagates_error() {
1056        use tower::ServiceExt;
1057
1058        let steps_pipeline = BoxProcessor::new(tower::service_fn(|mut ex: Exchange| {
1059            ex.input.body = camel_api::Body::Bytes("handled".into());
1060            async move { Ok(ex) }
1061        }));
1062        let policy = ExceptionPolicy {
1063            matches: Arc::new(|_| true),
1064            retry: None,
1065            handled_by: None,
1066            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1067            disposition: ExceptionDisposition::Propagate,
1068        };
1069        let inner = tower::service_fn(|_ex: Exchange| async {
1070            Err::<Exchange, CamelError>(CamelError::RouteError("fail".to_string()))
1071        });
1072        let mut svc = ErrorHandlerService::new(inner, None, vec![(policy, None)]);
1073        let ex = Exchange::default();
1074        let result = svc.ready().await.unwrap().call(ex).await;
1075        assert!(result.is_err(), "handled:false should propagate error");
1076    }
1077
1078    // --- Readiness error capture tests ---
1079    //
1080    // ErrorHandlerService must capture readiness errors (poll_ready returning Err)
1081    // and route them through retry/onException/DLC instead of propagating raw.
1082
1083    /// A service whose `poll_ready` always returns `Ready(Err(...))` but whose
1084    /// `call` returns `Ok`. This simulates a permanently-not-ready endpoint.
1085    #[derive(Clone)]
1086    struct ReadinessFailService {
1087        error: CamelError,
1088    }
1089
1090    impl ReadinessFailService {
1091        fn new(error: CamelError) -> Self {
1092            Self { error }
1093        }
1094    }
1095
1096    impl Service<Exchange> for ReadinessFailService {
1097        type Response = Exchange;
1098        type Error = CamelError;
1099        type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
1100
1101        fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
1102            Poll::Ready(Err(self.error.clone()))
1103        }
1104
1105        fn call(&mut self, ex: Exchange) -> Self::Future {
1106            // call() should never be reached if poll_ready returns Err,
1107            // but Tower's ready().await on a clone will re-encounter the readiness error.
1108            Box::pin(async move { Ok(ex) })
1109        }
1110    }
1111
1112    #[tokio::test]
1113    async fn test_readiness_error_goes_to_dlc() {
1114        let readiness_err = CamelError::ProcessorError("readiness-fail".into());
1115        let inner = ReadinessFailService {
1116            error: readiness_err,
1117        };
1118
1119        let received = Arc::new(std::sync::Mutex::new(Vec::<Exchange>::new()));
1120        let received_clone = Arc::clone(&received);
1121        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
1122            let r = Arc::clone(&received_clone);
1123            Box::pin(async move {
1124                r.lock().unwrap().push(ex.clone());
1125                Ok(ex)
1126            })
1127        });
1128
1129        let svc = ErrorHandlerService::new(inner, Some(dlc), vec![]);
1130        let result = svc.oneshot(make_exchange()).await;
1131
1132        // The error must be absorbed (Ok), not propagated raw (Err).
1133        assert!(
1134            result.is_ok(),
1135            "readiness error should be captured and sent to DLC, got: {:?}",
1136            result
1137        );
1138        let ex = result.unwrap();
1139        assert!(ex.has_error(), "exchange should carry the readiness error");
1140        assert_eq!(
1141            received.lock().unwrap().len(),
1142            1,
1143            "DLC should have received the exchange exactly once"
1144        );
1145    }
1146
1147    #[tokio::test]
1148    async fn test_readiness_error_goes_to_matching_policy() {
1149        let readiness_err = CamelError::ProcessorError("readiness-fail".into());
1150        let inner = ReadinessFailService {
1151            error: readiness_err,
1152        };
1153
1154        let steps_pipeline = BoxProcessor::new(tower::service_fn(|mut ex: Exchange| {
1155            ex.input.body = camel_api::Body::Bytes("handled-readiness".into());
1156            async move { Ok(ex) }
1157        }));
1158        let policy = ExceptionPolicy {
1159            matches: Arc::new(|_| true),
1160            retry: None,
1161            handled_by: None,
1162            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1163            disposition: ExceptionDisposition::Handled,
1164        };
1165
1166        let svc = ErrorHandlerService::new(inner, None, vec![(policy, None)]);
1167        let result = svc.oneshot(make_exchange()).await;
1168
1169        // The error must be absorbed and routed to the on_steps handler.
1170        assert!(
1171            result.is_ok(),
1172            "readiness error should be captured by policy, got: {:?}",
1173            result
1174        );
1175        let ex = result.unwrap();
1176        assert!(ex.error.is_none(), "handled:true should clear error");
1177        assert!(
1178            matches!(ex.input.body, camel_api::Body::Bytes(_)),
1179            "on_steps should have modified the body"
1180        );
1181    }
1182
1183    #[test]
1184    fn test_poll_ready_converts_readiness_error_to_ok() {
1185        let readiness_err = CamelError::ProcessorError("readiness-fail".into());
1186        let inner = ReadinessFailService {
1187            error: readiness_err,
1188        };
1189        let mut svc = ErrorHandlerService::new(inner, None, vec![]);
1190
1191        let waker = futures::task::noop_waker();
1192        let mut cx = Context::from_waker(&waker);
1193
1194        // poll_ready must NOT propagate the readiness error — convert to Ok.
1195        let poll = Pin::new(&mut svc).poll_ready(&mut cx);
1196        match poll {
1197            Poll::Ready(Ok(())) => { /* correct */ }
1198            Poll::Ready(Err(e)) => panic!("poll_ready leaked readiness error: {:?}", e),
1199            Poll::Pending => panic!("poll_ready should be Ready for readiness errors"),
1200        }
1201    }
1202
1203    // --- invoke_processor tests ---
1204
1205    #[tokio::test]
1206    async fn test_invoke_processor_returns_ok_on_success() {
1207        let mut svc = ok_processor();
1208        let ex = make_exchange();
1209        let result = invoke_processor(&mut svc, ex).await;
1210        assert!(result.is_ok());
1211    }
1212
1213    #[tokio::test]
1214    async fn test_invoke_processor_captures_readiness_error() {
1215        let mut failing_ready: BoxProcessor = BoxProcessor::new(ReadinessFailService::new(
1216            CamelError::ProcessorError("not ready".into()),
1217        ));
1218        let ex = make_exchange();
1219        let result = invoke_processor(&mut failing_ready, ex).await;
1220        assert!(result.is_err());
1221    }
1222
1223    #[tokio::test]
1224    async fn test_on_steps_handled_true_clears_exception_properties() {
1225        use tower::ServiceExt;
1226
1227        let steps_pipeline = BoxProcessor::new(tower::service_fn(|mut ex: Exchange| {
1228            ex.input.body = camel_api::Body::Bytes("handled".into());
1229            async move { Ok(ex) }
1230        }));
1231        let policy = ExceptionPolicy {
1232            matches: Arc::new(|_| true),
1233            retry: None,
1234            handled_by: None,
1235            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1236            disposition: ExceptionDisposition::Handled,
1237        };
1238        let inner = tower::service_fn(|_ex: Exchange| async {
1239            Err::<Exchange, CamelError>(CamelError::RouteError("fail".to_string()))
1240        });
1241        let mut svc = ErrorHandlerService::new(inner, None, vec![(policy, None)]);
1242        let ex = Exchange::default();
1243        let result = svc.ready().await.unwrap().call(ex).await.unwrap();
1244        assert!(result.error.is_none(), "handled:true should clear error");
1245        assert!(
1246            !result
1247                .properties
1248                .contains_key(camel_api::exchange::PROPERTY_EXCEPTION_MESSAGE),
1249            "handled:true should clear exception properties"
1250        );
1251        assert!(
1252            !result
1253                .properties
1254                .contains_key(camel_api::exchange::PROPERTY_EXCEPTION_KIND),
1255            "handled:true should clear exception kind property"
1256        );
1257        assert!(
1258            !result
1259                .properties
1260                .contains_key(camel_api::exchange::PROPERTY_EXCEPTION_CAUGHT),
1261            "handled:true should clear exception caught property"
1262        );
1263    }
1264
1265    // ── DefaultRouteErrorHandler tests ──
1266
1267    #[test]
1268    fn test_match_policy_returns_id_for_matching_error() {
1269        let handler = DefaultRouteErrorHandler::new(
1270            None,
1271            vec![(
1272                ExceptionPolicy::new(|e| matches!(e, CamelError::ProcessorError(_))),
1273                None,
1274            )],
1275        );
1276        let id = handler.match_policy(&CamelError::ProcessorError("test".into()));
1277        assert_eq!(id, Some(PolicyId(0)));
1278    }
1279
1280    #[test]
1281    fn test_match_policy_returns_none_for_unmatched() {
1282        let handler = DefaultRouteErrorHandler::new(None, vec![]);
1283        let id = handler.match_policy(&CamelError::ProcessorError("test".into()));
1284        assert_eq!(id, None);
1285    }
1286
1287    // ── retry_step tests ──
1288
1289    #[tokio::test]
1290    async fn test_retry_step_succeeds_on_second_attempt() {
1291        let mut policy = ExceptionPolicy::new(|_| true);
1292        policy.retry = Some(RedeliveryPolicy::new(3));
1293        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1294        let mut step = fail_n_times(1); // fails once, then succeeds
1295        let ex = make_exchange();
1296        let outcome = handler
1297            .retry_step(
1298                Some(PolicyId(0)),
1299                &mut step,
1300                ex,
1301                CamelError::ProcessorError("attempt 0".into()),
1302            )
1303            .await;
1304        assert!(matches!(outcome, RetryOutcome::Recovered(_)));
1305    }
1306
1307    #[tokio::test]
1308    async fn test_retry_step_exhausted_when_all_fail() {
1309        let mut policy = ExceptionPolicy::new(|_| true);
1310        policy.retry = Some(RedeliveryPolicy::new(3));
1311        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1312        let mut step = failing_processor();
1313        let ex = make_exchange();
1314        let outcome = handler
1315            .retry_step(
1316                Some(PolicyId(0)),
1317                &mut step,
1318                ex,
1319                CamelError::ProcessorError("boom".into()),
1320            )
1321            .await;
1322        assert!(matches!(outcome, RetryOutcome::Exhausted { .. }));
1323    }
1324
1325    #[tokio::test]
1326    async fn test_retry_step_no_policy_returns_exhausted_immediately() {
1327        let handler = DefaultRouteErrorHandler::new(None, vec![]);
1328        let mut step = ok_processor();
1329        let ex = make_exchange();
1330        let outcome = handler
1331            .retry_step(
1332                None,
1333                &mut step,
1334                ex,
1335                CamelError::ProcessorError("boom".into()),
1336            )
1337            .await;
1338        assert!(matches!(
1339            outcome,
1340            RetryOutcome::Exhausted { policy: None, .. }
1341        ));
1342    }
1343
1344    // ── handle_step tests ──
1345
1346    #[tokio::test]
1347    async fn test_handle_step_propagate_sends_to_dlc() {
1348        let dlc = BoxProcessor::from_fn(|ex| Box::pin(async move { Ok(ex) }));
1349        let handler = DefaultRouteErrorHandler::new(Some(dlc), vec![]);
1350        let ex = make_exchange();
1351        let result = handler
1352            .handle_step(None, ex, CamelError::ProcessorError("boom".into()))
1353            .await;
1354        assert!(matches!(result, Ok(StepDisposition::Propagate(_))));
1355    }
1356
1357    #[tokio::test]
1358    async fn test_handle_step_handled_uses_handler_output() {
1359        let handler_producer = BoxProcessor::from_fn(|mut ex| {
1360            Box::pin(async move {
1361                ex.input.set_header("processed_by", Value::Bool(true));
1362                Ok(ex)
1363            })
1364        });
1365        let policy = ExceptionPolicy {
1366            matches: std::sync::Arc::new(|_| true),
1367            retry: None,
1368            handled_by: None,
1369            on_steps: None,
1370            disposition: ExceptionDisposition::Handled,
1371        };
1372        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, Some(handler_producer))]);
1373        let mut ex = make_exchange();
1374        ex.set_error(CamelError::ProcessorError("boom".into()));
1375        let result = handler
1376            .handle_step(
1377                Some(PolicyId(0)),
1378                ex,
1379                CamelError::ProcessorError("boom".into()),
1380            )
1381            .await;
1382        match result {
1383            Ok(StepDisposition::Handled(ex)) => {
1384                assert!(!ex.has_error(), "error should be cleared");
1385                assert_eq!(
1386                    ex.input.header("processed_by"),
1387                    Some(&Value::Bool(true)),
1388                    "should use handler's output exchange"
1389                );
1390            }
1391            other => panic!("expected Handled, got {:?}", other.is_ok()),
1392        }
1393    }
1394
1395    #[tokio::test]
1396    async fn test_handle_step_continued_clears_error() {
1397        let policy = ExceptionPolicy {
1398            matches: std::sync::Arc::new(|_| true),
1399            retry: None,
1400            handled_by: None,
1401            on_steps: None,
1402            disposition: ExceptionDisposition::Continued,
1403        };
1404        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1405        let mut ex = make_exchange();
1406        ex.set_error(CamelError::ProcessorError("boom".into()));
1407        let result = handler
1408            .handle_step(
1409                Some(PolicyId(0)),
1410                ex,
1411                CamelError::ProcessorError("boom".into()),
1412            )
1413            .await;
1414        match result {
1415            Ok(StepDisposition::Continued(ex)) => assert!(!ex.has_error()),
1416            other => panic!("expected Continued, got {:?}", other.is_ok()),
1417        }
1418    }
1419
1420    #[tokio::test]
1421    async fn test_handle_step_with_on_steps_handled() {
1422        let steps_pipeline = BoxProcessor::new(tower::service_fn(|mut ex: Exchange| {
1423            ex.input.body = camel_api::Body::Bytes("on_steps_ran".into());
1424            async move { Ok(ex) }
1425        }));
1426        let policy = ExceptionPolicy {
1427            matches: std::sync::Arc::new(|_| true),
1428            retry: None,
1429            handled_by: None,
1430            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1431            disposition: ExceptionDisposition::Handled,
1432        };
1433        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1434        let mut ex = make_exchange();
1435        ex.set_error(CamelError::ProcessorError("boom".into()));
1436        let result = handler
1437            .handle_step(
1438                Some(PolicyId(0)),
1439                ex,
1440                CamelError::ProcessorError("boom".into()),
1441            )
1442            .await;
1443        match result {
1444            Ok(StepDisposition::Handled(ex)) => {
1445                assert!(!ex.has_error(), "error should be cleared");
1446                assert!(
1447                    matches!(ex.input.body, camel_api::Body::Bytes(_)),
1448                    "on_steps should have modified the body"
1449                );
1450            }
1451            other => panic!("expected Handled, got: {}", other.is_ok()),
1452        }
1453    }
1454
1455    #[tokio::test]
1456    async fn test_handle_step_dlc_failure_propagates() {
1457        let failing_dlc = BoxProcessor::from_fn(|_| {
1458            Box::pin(async { Err(CamelError::ProcessorError("dlc broken".into())) })
1459        });
1460        let handler = DefaultRouteErrorHandler::new(Some(failing_dlc), vec![]);
1461        let ex = make_exchange();
1462        let result = handler
1463            .handle_step(None, ex, CamelError::ProcessorError("original".into()))
1464            .await;
1465        assert!(
1466            matches!(result, Ok(StepDisposition::Propagate(_))),
1467            "DLC failure should still return Propagate with original error"
1468        );
1469    }
1470
1471    // ── handle_boundary tests ──
1472
1473    #[tokio::test]
1474    async fn test_handle_boundary_security_error_goes_to_dlc() {
1475        let dlc_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0));
1476        let count_clone = dlc_count.clone();
1477        let dlc = BoxProcessor::from_fn(move |ex| {
1478            let c = count_clone.clone();
1479            Box::pin(async move {
1480                c.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1481                Ok(ex)
1482            })
1483        });
1484        let handler = DefaultRouteErrorHandler::new(Some(dlc), vec![]);
1485        let ex = make_exchange();
1486        let result = handler
1487            .handle_boundary(
1488                BoundaryKind::Security,
1489                ex,
1490                CamelError::Unauthorized("denied".into()),
1491            )
1492            .await;
1493        assert!(result.is_ok());
1494        assert_eq!(dlc_count.load(std::sync::atomic::Ordering::SeqCst), 1);
1495    }
1496
1497    #[tokio::test]
1498    async fn test_handle_boundary_handled_clears_error() {
1499        let policy = ExceptionPolicy {
1500            matches: std::sync::Arc::new(|_| true),
1501            retry: None,
1502            handled_by: None,
1503            on_steps: None,
1504            disposition: ExceptionDisposition::Handled,
1505        };
1506        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1507        let ex = make_exchange();
1508        let result = handler
1509            .handle_boundary(
1510                BoundaryKind::Security,
1511                ex,
1512                CamelError::Unauthorized("denied".into()),
1513            )
1514            .await;
1515        assert!(result.is_ok());
1516        assert!(
1517            !result.unwrap().has_error(),
1518            "Handled disposition should clear error"
1519        );
1520    }
1521
1522    #[tokio::test]
1523    async fn test_handle_boundary_propagate_preserves_error() {
1524        let policy = ExceptionPolicy {
1525            matches: std::sync::Arc::new(|_| true),
1526            retry: None,
1527            handled_by: None,
1528            on_steps: None,
1529            disposition: ExceptionDisposition::Propagate,
1530        };
1531        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1532        let ex = make_exchange();
1533        let result = handler
1534            .handle_boundary(
1535                BoundaryKind::CircuitBreaker,
1536                ex,
1537                CamelError::CircuitOpen("open".into()),
1538            )
1539            .await;
1540        assert!(result.is_ok(), "boundary errors always return Ok");
1541        assert!(
1542            result.unwrap().has_error(),
1543            "Propagate disposition should preserve error"
1544        );
1545    }
1546
1547    #[tokio::test]
1548    async fn test_handle_boundary_continued_preserves_error_like_propagate() {
1549        let dlc_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0));
1550        let count_clone = dlc_count.clone();
1551        let dlc = BoxProcessor::from_fn(move |ex| {
1552            let c = count_clone.clone();
1553            Box::pin(async move {
1554                c.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1555                Ok(ex)
1556            })
1557        });
1558        let policy = ExceptionPolicy {
1559            matches: std::sync::Arc::new(|_| true),
1560            retry: None,
1561            handled_by: None,
1562            on_steps: None,
1563            disposition: ExceptionDisposition::Continued,
1564        };
1565        let handler = DefaultRouteErrorHandler::new(Some(dlc), vec![(policy, None)]);
1566        let ex = make_exchange();
1567        let result = handler
1568            .handle_boundary(
1569                BoundaryKind::Security,
1570                ex,
1571                CamelError::Unauthorized("denied".into()),
1572            )
1573            .await;
1574        assert!(result.is_ok(), "boundary errors always return Ok");
1575        assert!(
1576            result.unwrap().has_error(),
1577            "Continued at boundary should preserve error"
1578        );
1579        assert_eq!(
1580            dlc_count.load(std::sync::atomic::Ordering::SeqCst),
1581            1,
1582            "DLC should be called"
1583        );
1584    }
1585
1586    #[tokio::test]
1587    async fn test_handle_boundary_with_on_steps_handled() {
1588        let steps_pipeline = BoxProcessor::new(tower::service_fn(|mut ex: Exchange| {
1589            ex.input.body = camel_api::Body::Bytes("on_steps_ran".into());
1590            async move { Ok(ex) }
1591        }));
1592        let policy = ExceptionPolicy {
1593            matches: std::sync::Arc::new(|_| true),
1594            retry: None,
1595            handled_by: None,
1596            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1597            disposition: ExceptionDisposition::Handled,
1598        };
1599        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1600        let ex = make_exchange();
1601        let result = handler
1602            .handle_boundary(
1603                BoundaryKind::Security,
1604                ex,
1605                CamelError::Unauthorized("denied".into()),
1606            )
1607            .await;
1608        assert!(result.is_ok(), "boundary errors always return Ok");
1609        let ex = result.unwrap();
1610        assert!(!ex.has_error(), "Handled disposition should clear error");
1611        assert!(
1612            matches!(ex.input.body, camel_api::Body::Bytes(_)),
1613            "on_steps should have modified the body"
1614        );
1615    }
1616
1617    #[tokio::test]
1618    async fn retry_step_segment_stop_maps_to_retry_outcome_stopped() {
1619        use std::sync::Arc;
1620        use std::sync::atomic::{AtomicUsize, Ordering};
1621
1622        #[derive(Clone)]
1623        struct StoppingSegment {
1624            n: Arc<AtomicUsize>,
1625        }
1626        impl OutcomePipeline for StoppingSegment {
1627            fn clone_box(&self) -> Box<dyn OutcomePipeline> {
1628                Box::new(self.clone())
1629            }
1630            fn run<'a>(
1631                &'a mut self,
1632                ex: Exchange,
1633            ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
1634                let n = self.n.clone();
1635                Box::pin(async move {
1636                    n.fetch_add(1, Ordering::SeqCst);
1637                    PipelineOutcome::Stopped(ex)
1638                })
1639            }
1640        }
1641
1642        let call_count = Arc::new(AtomicUsize::new(0));
1643        let seg = OutcomeSegment::new(Box::new(StoppingSegment {
1644            n: call_count.clone(),
1645        }));
1646        let mut retryable: Box<dyn RetryableStep> = Box::new(seg);
1647
1648        let mut policy = ExceptionPolicy::new(|_e: &CamelError| true);
1649        policy.retry = Some(RedeliveryPolicy::new(3));
1650        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, None)]);
1651
1652        let original = Exchange::new(Message::new("retry-me"));
1653        let err = CamelError::ProcessorError("trigger retry".into());
1654        let outcome = handler
1655            .retry_step(Some(PolicyId(0)), retryable.as_mut(), original, err)
1656            .await;
1657
1658        assert!(
1659            matches!(outcome, RetryOutcome::Stopped(_)),
1660            "Segment Stop must map to RetryOutcome::Stopped, got {:?}",
1661            outcome
1662        );
1663        assert_eq!(
1664            call_count.load(Ordering::SeqCst),
1665            1,
1666            "Stop must short-circuit retry — only 1 invoke expected, got {}",
1667            call_count.load(Ordering::SeqCst)
1668        );
1669    }
1670
1671    #[tokio::test]
1672    async fn retry_step_new_signature_works_with_dlc_producer() {
1673        use std::sync::Arc;
1674        use std::sync::atomic::{AtomicUsize, Ordering};
1675
1676        #[derive(Clone)]
1677        struct CountingProducer {
1678            count: Arc<AtomicUsize>,
1679            succeed_on: usize,
1680        }
1681        impl tower::Service<Exchange> for CountingProducer {
1682            type Response = Exchange;
1683            type Error = CamelError;
1684            type Future = Pin<Box<dyn Future<Output = Result<Exchange, CamelError>> + Send>>;
1685            fn poll_ready(
1686                &mut self,
1687                _cx: &mut std::task::Context<'_>,
1688            ) -> std::task::Poll<Result<(), Self::Error>> {
1689                std::task::Poll::Ready(Ok(()))
1690            }
1691            fn call(&mut self, ex: Exchange) -> Self::Future {
1692                let n = self.count.fetch_add(1, Ordering::SeqCst);
1693                let succeed_on = self.succeed_on;
1694                Box::pin(async move {
1695                    if n >= succeed_on {
1696                        Ok(ex)
1697                    } else {
1698                        Err(CamelError::ProcessorError("retry".into()))
1699                    }
1700                })
1701            }
1702        }
1703
1704        let count = Arc::new(AtomicUsize::new(0));
1705        let producer = CountingProducer {
1706            count: count.clone(),
1707            succeed_on: 2,
1708        };
1709        let sync_bp = SyncBoxProcessor::new(BoxProcessor::new(producer));
1710        let bp1 = sync_bp.clone_inner();
1711        let bp2 = sync_bp.clone_inner();
1712        let mut retryable1: Box<dyn RetryableStep> = Box::new(bp1);
1713        let mut retryable2: Box<dyn RetryableStep> = Box::new(bp2);
1714
1715        let ex = Exchange::new(Message::new("dlc"));
1716        let outcome1 = retryable1.invoke(ex.clone()).await;
1717        let outcome2 = retryable2.invoke(ex).await;
1718        assert!(matches!(outcome1, PipelineOutcome::Failed(_)));
1719        assert!(matches!(outcome2, PipelineOutcome::Failed(_)));
1720        assert_eq!(
1721            count.load(Ordering::SeqCst),
1722            2,
1723            "DLC producer must be invoked exactly twice through SyncBoxProcessor"
1724        );
1725        drop(retryable1);
1726        drop(retryable2);
1727        drop(sync_bp);
1728    }
1729
1730    // ── use_original_message tests ──
1731
1732    #[tokio::test]
1733    async fn test_use_original_message_restores_body_before_dlc() {
1734        // Verifies that when the extension is set, handle_step restores the
1735        // original message body before the DLC sees it.
1736        let dlc_received = Arc::new(std::sync::Mutex::new(None::<Exchange>));
1737        let dlc_received_clone = Arc::clone(&dlc_received);
1738        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
1739            let r = Arc::clone(&dlc_received_clone);
1740            Box::pin(async move {
1741                *r.lock().unwrap() = Some(ex.clone());
1742                Ok(ex)
1743            })
1744        });
1745
1746        let mut handler = DefaultRouteErrorHandler::new(Some(dlc), vec![]);
1747        handler.use_original_message = true;
1748
1749        // Build exchange with original body, stash it, then mutate.
1750        let mut ex = make_exchange();
1751        ex.input = Message::new("original-body");
1752
1753        // Simulate RouteChannelService stashing the original message.
1754        let original: Arc<Message> = Arc::new(ex.input.clone());
1755        ex.set_extension(camel_api::ORIGINAL_MESSAGE_EXTENSION, original);
1756
1757        // Mutate the body (simulating a pipeline step that transforms then fails).
1758        ex.input.body = camel_api::Body::Bytes("mutated-body".into());
1759
1760        // Call handle_step — the restore should fire before send_to_handler.
1761        let result = handler
1762            .handle_step(None, ex, CamelError::ProcessorError("boom".into()))
1763            .await;
1764        assert!(matches!(result, Ok(StepDisposition::Propagate(_))));
1765
1766        // The DLC must have received the exchange with the ORIGINAL body.
1767        let received = dlc_received
1768            .lock()
1769            .unwrap()
1770            .take()
1771            .expect("DLC should have been called");
1772        let received_text = match &received.input.body {
1773            camel_api::Body::Text(s) => s.clone(),
1774            camel_api::Body::Bytes(b) => String::from_utf8_lossy(b).to_string(),
1775            camel_api::Body::Json(v) => v.to_string(),
1776            _ => String::new(),
1777        };
1778        assert_eq!(
1779            received_text, "original-body",
1780            "DLC should receive original message body, not mutated version"
1781        );
1782    }
1783
1784    #[tokio::test]
1785    async fn test_use_original_message_handle_boundary_restores_before_dlc() {
1786        let dlc_received = Arc::new(std::sync::Mutex::new(None::<Exchange>));
1787        let dlc_received_clone = Arc::clone(&dlc_received);
1788        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
1789            let r = Arc::clone(&dlc_received_clone);
1790            Box::pin(async move {
1791                *r.lock().unwrap() = Some(ex.clone());
1792                Ok(ex)
1793            })
1794        });
1795
1796        let mut handler = DefaultRouteErrorHandler::new(Some(dlc), vec![]);
1797        handler.use_original_message = true;
1798
1799        let mut ex = make_exchange();
1800        ex.input = Message::new("orig-boundary");
1801
1802        let original: Arc<Message> = Arc::new(ex.input.clone());
1803        ex.set_extension(camel_api::ORIGINAL_MESSAGE_EXTENSION, original);
1804
1805        // Mutate body before boundary error.
1806        ex.input.body = camel_api::Body::Bytes("mutated-boundary".into());
1807
1808        let result = handler
1809            .handle_boundary(
1810                BoundaryKind::Security,
1811                ex,
1812                CamelError::Unauthorized("denied".into()),
1813            )
1814            .await;
1815        assert!(result.is_ok());
1816
1817        let received = dlc_received
1818            .lock()
1819            .unwrap()
1820            .take()
1821            .expect("DLC should have been called");
1822        let received_text = match &received.input.body {
1823            camel_api::Body::Text(s) => s.clone(),
1824            camel_api::Body::Bytes(b) => String::from_utf8_lossy(b).to_string(),
1825            camel_api::Body::Json(v) => v.to_string(),
1826            _ => String::new(),
1827        };
1828        assert_eq!(
1829            received_text, "orig-boundary",
1830            "handle_boundary should restore original message before sending to DLC"
1831        );
1832    }
1833
1834    #[tokio::test]
1835    async fn test_use_original_message_false_does_not_restore() {
1836        // When use_original_message is false (default), the mutation should PASS THROUGH.
1837        let dlc_received = Arc::new(std::sync::Mutex::new(None::<Exchange>));
1838        let dlc_received_clone = Arc::clone(&dlc_received);
1839        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
1840            let r = Arc::clone(&dlc_received_clone);
1841            Box::pin(async move {
1842                *r.lock().unwrap() = Some(ex.clone());
1843                Ok(ex)
1844            })
1845        });
1846
1847        let handler = DefaultRouteErrorHandler::new(Some(dlc), vec![]);
1848        // use_original_message defaults to false
1849
1850        let mut ex = make_exchange();
1851        ex.input = Message::new("original-body");
1852
1853        // Stash still set but flag is false — must be ignored.
1854        let original: Arc<Message> = Arc::new(ex.input.clone());
1855        ex.set_extension(camel_api::ORIGINAL_MESSAGE_EXTENSION, original);
1856
1857        // Mutate the body.
1858        ex.input.body = camel_api::Body::Bytes("mutated-body".into());
1859
1860        let result = handler
1861            .handle_step(None, ex, CamelError::ProcessorError("boom".into()))
1862            .await;
1863        assert!(matches!(result, Ok(StepDisposition::Propagate(_))));
1864
1865        let received = dlc_received
1866            .lock()
1867            .unwrap()
1868            .take()
1869            .expect("DLC should have been called");
1870        let received_text = match &received.input.body {
1871            camel_api::Body::Text(s) => s.clone(),
1872            camel_api::Body::Bytes(b) => String::from_utf8_lossy(b).to_string(),
1873            camel_api::Body::Json(v) => v.to_string(),
1874            _ => String::new(),
1875        };
1876        assert_eq!(
1877            received_text, "mutated-body",
1878            "When use_original_message=false, DLC should see the mutated body"
1879        );
1880    }
1881
1882    // ── D-M17: Propagate must skip on_steps (prevents double side-effects) ──
1883
1884    #[tokio::test]
1885    async fn propagate_skips_on_steps_in_handle_step() {
1886        let on_steps_called = Arc::new(AtomicU32::new(0));
1887        let on_steps_called_clone = on_steps_called.clone();
1888        let steps_pipeline = BoxProcessor::new(tower::service_fn(move |mut ex: Exchange| {
1889            let c = on_steps_called_clone.clone();
1890            Box::pin(async move {
1891                c.fetch_add(1, Ordering::SeqCst);
1892                ex.input.body = camel_api::Body::Bytes("on_steps_ran".into());
1893                Ok(ex)
1894            })
1895        }));
1896        let dlc_called = Arc::new(AtomicU32::new(0));
1897        let dlc_called_clone = dlc_called.clone();
1898        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
1899            let c = dlc_called_clone.clone();
1900            Box::pin(async move {
1901                c.fetch_add(1, Ordering::SeqCst);
1902                Ok(ex)
1903            })
1904        });
1905        let policy = ExceptionPolicy {
1906            matches: std::sync::Arc::new(|_| true),
1907            retry: None,
1908            handled_by: None,
1909            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1910            disposition: ExceptionDisposition::Propagate,
1911        };
1912        let handler = DefaultRouteErrorHandler::new(Some(dlc), vec![(policy, None)]);
1913        let mut ex = make_exchange();
1914        ex.set_error(CamelError::ProcessorError("boom".into()));
1915        let result = handler
1916            .handle_step(
1917                Some(PolicyId(0)),
1918                ex,
1919                CamelError::ProcessorError("boom".into()),
1920            )
1921            .await;
1922        assert!(
1923            matches!(result, Ok(StepDisposition::Propagate(_))),
1924            "Propagate disposition should return Propagate"
1925        );
1926        assert_eq!(
1927            on_steps_called.load(Ordering::SeqCst),
1928            0,
1929            "on_steps must NOT be called when disposition is Propagate (double side-effect bug)"
1930        );
1931        assert_eq!(
1932            dlc_called.load(Ordering::SeqCst),
1933            1,
1934            "DLC should still be called when disposition is Propagate"
1935        );
1936    }
1937
1938    #[tokio::test]
1939    async fn propagate_skips_on_steps_in_handle_boundary() {
1940        let on_steps_called = Arc::new(AtomicU32::new(0));
1941        let on_steps_called_clone = on_steps_called.clone();
1942        let steps_pipeline = BoxProcessor::new(tower::service_fn(move |mut ex: Exchange| {
1943            let c = on_steps_called_clone.clone();
1944            Box::pin(async move {
1945                c.fetch_add(1, Ordering::SeqCst);
1946                ex.input.body = camel_api::Body::Bytes("on_steps_ran".into());
1947                Ok(ex)
1948            })
1949        }));
1950        let dlc_called = Arc::new(AtomicU32::new(0));
1951        let dlc_called_clone = dlc_called.clone();
1952        let dlc = BoxProcessor::from_fn(move |ex: Exchange| {
1953            let c = dlc_called_clone.clone();
1954            Box::pin(async move {
1955                c.fetch_add(1, Ordering::SeqCst);
1956                Ok(ex)
1957            })
1958        });
1959        let policy = ExceptionPolicy {
1960            matches: std::sync::Arc::new(|_| true),
1961            retry: None,
1962            handled_by: None,
1963            on_steps: Some(SyncBoxProcessor::new(steps_pipeline)),
1964            disposition: ExceptionDisposition::Propagate,
1965        };
1966        let handler = DefaultRouteErrorHandler::new(Some(dlc), vec![(policy, None)]);
1967        let ex = make_exchange();
1968        let result = handler
1969            .handle_boundary(
1970                BoundaryKind::CircuitBreaker,
1971                ex,
1972                CamelError::CircuitOpen("open".into()),
1973            )
1974            .await;
1975        assert!(result.is_ok(), "boundary errors always return Ok");
1976        assert!(
1977            result.unwrap().has_error(),
1978            "Propagate disposition should preserve error"
1979        );
1980        assert_eq!(
1981            on_steps_called.load(Ordering::SeqCst),
1982            0,
1983            "on_steps must NOT be called when disposition is Propagate (double side-effect bug)"
1984        );
1985        assert_eq!(
1986            dlc_called.load(Ordering::SeqCst),
1987            1,
1988            "DLC should still be called when disposition is Propagate"
1989        );
1990    }
1991
1992    // ── Delegate-failure contract tests (rc-ntpof) ──
1993    //
1994    // A failing delegate (DLC/handler producer) must NEVER surface as a
1995    // successful exchange or disposition: the ORIGINAL error always
1996    // propagates, every path.
1997
1998    /// Delegate producer whose `call` always fails with a distinct error.
1999    fn failing_delegate() -> BoxProcessor {
2000        BoxProcessor::from_fn(|_| {
2001            Box::pin(async { Err(CamelError::ProcessorError("delegate-broken".into())) })
2002        })
2003    }
2004
2005    /// Handler with one match-any policy (Handled disposition) and the
2006    /// given `handled_by` producer.
2007    fn handled_policy_with(producer: Option<BoxProcessor>) -> DefaultRouteErrorHandler {
2008        let policy = ExceptionPolicy {
2009            matches: std::sync::Arc::new(|_| true),
2010            retry: None,
2011            handled_by: None,
2012            on_steps: None,
2013            disposition: ExceptionDisposition::Handled,
2014        };
2015        DefaultRouteErrorHandler::new(None, vec![(policy, producer)])
2016    }
2017
2018    fn assert_propagates_original(result: &Result<StepDisposition, CamelError>) {
2019        assert!(
2020            matches!(
2021                result,
2022                Ok(StepDisposition::Propagate(CamelError::ProcessorError(m)))
2023                    if m == "original-err"
2024            ),
2025            "delegate failure must propagate the ORIGINAL error, got: {:?}",
2026            result
2027                .as_ref()
2028                .map(|d| d_variant_name(d))
2029                .unwrap_or_default()
2030        );
2031    }
2032
2033    /// Test-only: short label of a StepDisposition for panic messages.
2034    fn d_variant_name(d: &StepDisposition) -> &'static str {
2035        match d {
2036            StepDisposition::Propagate(_) => "Propagate",
2037            StepDisposition::Handled(_) => "Handled",
2038            StepDisposition::Continued(_) => "Continued",
2039            _ => "other",
2040        }
2041    }
2042
2043    #[tokio::test]
2044    async fn delegate_call_failure_with_handled_propagates_original() {
2045        // Prevent callsite-interest poisoning of the system-broken error!
2046        // callsites before this test executes them (see ensure_global_registry).
2047        ensure_global_registry();
2048        let handler = handled_policy_with(Some(failing_delegate()));
2049        let result = handler
2050            .handle_step(
2051                Some(PolicyId(0)),
2052                make_exchange(),
2053                CamelError::ProcessorError("original-err".into()),
2054            )
2055            .await;
2056        assert_propagates_original(&result);
2057    }
2058
2059    #[test]
2060    fn delegate_failure_emits_system_broken_log_and_span_error() {
2061        let handler = handled_policy_with(Some(failing_delegate()));
2062        let mut ex = make_exchange();
2063        ex.set_error(CamelError::ProcessorError("original-err".into()));
2064        let (result, captured, span_records) = capture_debugs_with_span_records(|| {
2065            // Declared `error` field so send_to_handler's span record lands.
2066            let span = tracing::info_span!("delegate_failure_test", error = tracing::field::Empty);
2067            let _guard = span.enter();
2068            tokio::runtime::Builder::new_current_thread()
2069                .enable_all()
2070                .build()
2071                .expect("current-thread runtime")
2072                .block_on(handler.handle_step(
2073                    Some(PolicyId(0)),
2074                    ex,
2075                    CamelError::ProcessorError("original-err".into()),
2076                ))
2077        });
2078        assert!(
2079            matches!(result, Ok(StepDisposition::Propagate(_))),
2080            "delegate failure must propagate, got: {}",
2081            result
2082                .as_ref()
2083                .map(|d| d_variant_name(d))
2084                .unwrap_or("Err(..)")
2085        );
2086        // (i) system-broken record structuring BOTH errors.
2087        assert!(
2088            captured.iter().any(|line| {
2089                line.contains("DLC/handler call failed")
2090                    && line.contains("original-err")
2091                    && line.contains("delegate-broken")
2092            }),
2093            "expected system-broken log structuring BOTH errors, captured: {captured:?}"
2094        );
2095        // (ii) the span recorded the `error` field with the DELEGATE error.
2096        assert!(
2097            span_records.iter().any(|line| {
2098                line.contains("error=")
2099                    && line.contains("delegate-broken")
2100                    && !line.contains("original-err")
2101            }),
2102            "expected span error record carrying the DELEGATE error, span records: {span_records:?}"
2103        );
2104    }
2105
2106    #[tokio::test]
2107    async fn retry_exhausted_then_delegate_failure_propagates_original() {
2108        // Prevent callsite-interest poisoning of the system-broken error!
2109        // callsites before this test executes them (see ensure_global_registry).
2110        ensure_global_registry();
2111        let policy = ExceptionPolicy {
2112            matches: std::sync::Arc::new(|_| true),
2113            retry: Some(RedeliveryPolicy {
2114                max_attempts: 1,
2115                initial_delay: Duration::from_millis(1),
2116                multiplier: 1.0,
2117                max_delay: Duration::from_millis(10),
2118                jitter_factor: 0.0,
2119            }),
2120            handled_by: None,
2121            on_steps: None,
2122            disposition: ExceptionDisposition::Handled,
2123        };
2124        let svc = ErrorHandlerService::new(
2125            failing_processor(),
2126            Some(failing_delegate()),
2127            vec![(policy, None)],
2128        );
2129        let result = svc.oneshot(make_exchange()).await;
2130        assert!(
2131            matches!(&result, Err(CamelError::ProcessorError(m)) if m == "boom"),
2132            "retry-exhausted ORIGINAL error must surface as Err, got: {:?}",
2133            result.map(|ex| ex.has_error())
2134        );
2135    }
2136
2137    #[tokio::test]
2138    async fn on_steps_fallback_delegate_failure_propagates_original() {
2139        // Prevent callsite-interest poisoning of the system-broken error!
2140        // callsites before this test executes them (see ensure_global_registry).
2141        ensure_global_registry();
2142        // on_steps pipeline itself fails → handle_step's on_steps Err arm
2143        // falls through to the send_to_handler forward → delegate also
2144        // fails → the ORIGINAL step error must propagate.
2145        let policy = ExceptionPolicy {
2146            matches: std::sync::Arc::new(|_| true),
2147            retry: None,
2148            handled_by: None,
2149            on_steps: Some(SyncBoxProcessor::new(failing_processor())),
2150            disposition: ExceptionDisposition::Handled,
2151        };
2152        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, Some(failing_delegate()))]);
2153        let result = handler
2154            .handle_step(
2155                Some(PolicyId(0)),
2156                make_exchange(),
2157                CamelError::ProcessorError("original-err".into()),
2158            )
2159            .await;
2160        assert_propagates_original(&result);
2161    }
2162
2163    #[tokio::test]
2164    async fn no_match_dlc_delegate_failure_propagates_original() {
2165        // Prevent callsite-interest poisoning of the system-broken error!
2166        // callsites before this test executes them (see ensure_global_registry).
2167        ensure_global_registry();
2168        // No policy matches → the no-match DLC forward fires → DLC fails →
2169        // the ORIGINAL error surfaces as Err (never an Ok exchange masking
2170        // the failure).
2171        let policy = ExceptionPolicy::new(|e| matches!(e, CamelError::Io(_)));
2172        let svc = ErrorHandlerService::new(
2173            failing_processor(),
2174            Some(failing_delegate()),
2175            vec![(policy, None)],
2176        );
2177        let result = svc.oneshot(make_exchange()).await;
2178        assert!(
2179            matches!(&result, Err(CamelError::ProcessorError(m)) if m == "boom"),
2180            "no-match DLC delegate failure must surface the ORIGINAL error as Err, got: {:?}",
2181            result.map(|ex| ex.has_error())
2182        );
2183    }
2184
2185    #[tokio::test]
2186    async fn delegate_ready_failure_propagates_original() {
2187        // Prevent callsite-interest poisoning of the system-broken error!
2188        // callsites before this test executes them (see ensure_global_registry).
2189        ensure_global_registry();
2190        let not_ready: BoxProcessor = BoxProcessor::new(ReadinessFailService::new(
2191            CamelError::ProcessorError("delegate-not-ready".into()),
2192        ));
2193        let handler = handled_policy_with(Some(not_ready));
2194        let result = handler
2195            .handle_step(
2196                Some(PolicyId(0)),
2197                make_exchange(),
2198                CamelError::ProcessorError("original-err".into()),
2199            )
2200            .await;
2201        assert_propagates_original(&result);
2202    }
2203
2204    #[tokio::test]
2205    async fn delegate_failure_with_continued_propagates_original() {
2206        // Prevent callsite-interest poisoning of the system-broken error!
2207        // callsites before this test executes them (see ensure_global_registry).
2208        ensure_global_registry();
2209        let policy = ExceptionPolicy {
2210            matches: std::sync::Arc::new(|_| true),
2211            retry: None,
2212            handled_by: None,
2213            on_steps: None,
2214            disposition: ExceptionDisposition::Continued,
2215        };
2216        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, Some(failing_delegate()))]);
2217        let result = handler
2218            .handle_step(
2219                Some(PolicyId(0)),
2220                make_exchange(),
2221                CamelError::ProcessorError("original-err".into()),
2222            )
2223            .await;
2224        assert_propagates_original(&result);
2225    }
2226
2227    #[tokio::test]
2228    async fn boundary_delegate_failure_returns_original_err() {
2229        // Prevent callsite-interest poisoning of the system-broken error!
2230        // callsites before this test executes them (see ensure_global_registry).
2231        ensure_global_registry();
2232        let handler = DefaultRouteErrorHandler::new(Some(failing_delegate()), vec![]);
2233        let result = handler
2234            .handle_boundary(
2235                BoundaryKind::Security,
2236                make_exchange(),
2237                CamelError::Unauthorized("denied".into()),
2238            )
2239            .await;
2240        assert!(
2241            matches!(&result, Err(CamelError::Unauthorized(m)) if m == "denied"),
2242            "boundary delegate failure must return the ORIGINAL boundary error, got: {:?}",
2243            result
2244        );
2245    }
2246
2247    #[tokio::test]
2248    async fn tap_policy_without_handled_propagates() {
2249        // Tap semantics: delegate + default disposition (Propagate) — the
2250        // delegate fires for side-effects, and the ORIGINAL error still
2251        // propagates.
2252        let delegate_hits = Arc::new(AtomicU32::new(0));
2253        let hits = Arc::clone(&delegate_hits);
2254        let tap_delegate = BoxProcessor::from_fn(move |ex: Exchange| {
2255            let h = Arc::clone(&hits);
2256            Box::pin(async move {
2257                h.fetch_add(1, Ordering::SeqCst);
2258                Ok(ex)
2259            })
2260        });
2261        let policy = ExceptionPolicy {
2262            matches: std::sync::Arc::new(|_| true),
2263            retry: None,
2264            handled_by: None,
2265            on_steps: None,
2266            disposition: ExceptionDisposition::Propagate,
2267        };
2268        let handler = DefaultRouteErrorHandler::new(None, vec![(policy, Some(tap_delegate))]);
2269        let result = handler
2270            .handle_step(
2271                Some(PolicyId(0)),
2272                make_exchange(),
2273                CamelError::ProcessorError("original-err".into()),
2274            )
2275            .await;
2276        assert_propagates_original(&result);
2277        assert_eq!(
2278            delegate_hits.load(Ordering::SeqCst),
2279            1,
2280            "delegate must have received the exchange exactly once"
2281        );
2282    }
2283
2284    #[test]
2285    fn test_handle_step_no_match_emits_silent_propagate_diagnostic() {
2286        // rc-xtiem sweep item [1]: parity with the handle_boundary
2287        // diagnostic test — proves the ORIGINAL rc-fu1of site still fires.
2288        let handler = DefaultRouteErrorHandler::new(None, vec![]);
2289        let (result, captured) = capture_debugs(|| {
2290            tokio::runtime::Builder::new_current_thread()
2291                .enable_all()
2292                .build()
2293                .expect("current-thread runtime")
2294                .block_on(handler.handle_step(
2295                    None,
2296                    make_exchange(),
2297                    CamelError::Unauthorized("denied".into()),
2298                ))
2299        });
2300        assert!(result.is_ok(), "step handler always returns Ok");
2301        assert!(
2302            captured.iter().any(|line| {
2303                line.contains(
2304                    "no on_exceptions policy matched and no dead-letter channel configured; \
2305                     propagating error",
2306                )
2307            }),
2308            "expected silent-propagate diagnostic, captured: {captured:?}"
2309        );
2310        assert!(
2311            captured
2312                .iter()
2313                .any(|line| line.contains("kind=Unauthorized")),
2314            "diagnostic should name the error kind, captured: {captured:?}"
2315        );
2316    }
2317
2318    #[test]
2319    fn test_handle_boundary_no_match_emits_silent_propagate_diagnostic() {
2320        let handler = DefaultRouteErrorHandler::new(None, vec![]);
2321        let (result, captured) = capture_debugs(|| {
2322            tokio::runtime::Builder::new_current_thread()
2323                .enable_all()
2324                .build()
2325                .expect("current-thread runtime")
2326                .block_on(handler.handle_boundary(
2327                    BoundaryKind::Security,
2328                    make_exchange(),
2329                    CamelError::Unauthorized("denied".into()),
2330                ))
2331        });
2332        assert!(result.is_ok(), "boundary handler always returns Ok");
2333        assert!(
2334            captured.iter().any(|line| {
2335                line.contains(
2336                    "no on_exceptions policy matched and no dead-letter channel configured; \
2337                     propagating error",
2338                )
2339            }),
2340            "expected silent-propagate diagnostic, captured: {captured:?}"
2341        );
2342        assert!(
2343            captured.iter().any(
2344                |line| line.contains("boundary=Security") && line.contains("kind=Unauthorized")
2345            ),
2346            "diagnostic should name the boundary gate and error kind, captured: {captured:?}"
2347        );
2348    }
2349
2350    // Log-capture helpers moved to the shared `crate::test_log_capture`
2351    // module (bd rc-zgbqq) so the do_try tests reuse the exact same
2352    // capture behavior. `capture_debugs` stays local as a thin wrapper.
2353    use crate::test_log_capture::{capture_debugs_with_span_records, ensure_global_registry};
2354
2355    /// Event-only capture: discards span records (existing contract).
2356    fn capture_debugs<T>(f: impl FnOnce() -> T) -> (T, Vec<String>) {
2357        let (out, events, _span_records) = capture_debugs_with_span_records(f);
2358        (out, events)
2359    }
2360}