camel_core/lifecycle/adapters/route_controller.rs
1//! Default implementation of RouteController.
2//!
3//! This module provides [`DefaultRouteController`], which manages route lifecycle
4//! including starting, stopping, suspending, and resuming routes.
5
6use std::collections::HashMap;
7use std::sync::atomic::AtomicU64;
8use std::sync::{Arc, Weak};
9use std::time::Duration;
10
11use tokio::sync::mpsc;
12use tokio_util::sync::CancellationToken;
13use tower::{Layer, ServiceExt};
14use tracing::{debug, info, warn};
15
16use super::route_controller_trait::{
17 BindExposureAcks, bind_key_from_uri, enforce_bind_exposure_gate,
18};
19use camel_api::error_handler::ErrorHandlerConfig;
20use camel_api::metrics::MetricsCollector;
21#[allow(unused_imports)]
22use camel_api::{
23 BoxProcessor, CamelError, Exchange, FunctionInvoker, IdentityProcessor, InFlightClaim,
24 NoOpMetrics, NoopPlatformService, PlatformService, ProducerContext, RouteController,
25 RuntimeHandle, StepLifecycle,
26};
27use camel_component_api::{Consumer, ConsumerContext, consumer::ExchangeEnvelope};
28use camel_processor::aggregator::AggregatorService;
29pub use camel_processor::aggregator::SharedLanguageRegistry;
30use camel_processor::aggregator::{AggregateEmission, AggregationReceipt};
31
32use crate::health_registry::HealthCheckRegistry;
33use crate::intercept::InterceptRules;
34use crate::lifecycle::CohortActivationGate;
35use crate::lifecycle::adapters::controller_component_context::ControllerComponentContext;
36use crate::lifecycle::adapters::route_compiler::TracerPipelineGating;
37use crate::lifecycle::adapters::route_compiler_ext::{
38 RouteCompilerExt, build_eh_config_pipeline, transport_from_uri,
39};
40use crate::lifecycle::adapters::route_helpers::{
41 AggregateSplitInfo, CrashNotification, ManagedRoute, assert_no_mixed_top_level_splits,
42 handle_is_running, inferred_lifecycle_label, is_pending, send_reply_or_b_prime,
43};
44#[cfg(test)]
45pub(super) use crate::lifecycle::adapters::route_helpers::{
46 emit_start_route_event, set_start_route_event_hook,
47};
48use crate::lifecycle::adapters::route_registry::RouteRegistry;
49use crate::lifecycle::adapters::route_runtime_state;
50use crate::lifecycle::adapters::step_compilers::CompiledStep;
51use crate::lifecycle::application::route_definition::{BuilderStep, RouteDefinition};
52pub(crate) use crate::lifecycle::domain::CompiledPipeline;
53use crate::shared::components::domain::Registry;
54use crate::shared::observability::domain::{DetailLevel, TracerConfig};
55use camel_bean::BeanRegistry;
56
57/// Default implementation of [`RouteController`].
58///
59/// Manages route lifecycle with support for:
60/// - Starting/stopping individual routes
61/// - Suspending and resuming routes
62/// - Auto-startup with startup ordering
63/// - Graceful shutdown
64pub struct DefaultRouteController {
65 /// Routes indexed by route ID.
66 pub(super) routes: RouteRegistry,
67 /// Reference to the component registry for resolving endpoints.
68 pub(super) registry: Arc<std::sync::Mutex<Registry>>,
69 /// Shared language registry for resolving declarative language expressions.
70 pub(super) languages: SharedLanguageRegistry,
71 /// Bean registry for bean method invocation.
72 pub(super) beans: Arc<std::sync::Mutex<BeanRegistry>>,
73 /// Runtime handle injected into ProducerContext for command/query operations.
74 pub(super) runtime: Option<Weak<dyn RuntimeHandle>>,
75 /// Optional global error handler applied to all routes without a per-route handler.
76 pub(super) global_error_handler: Option<ErrorHandlerConfig>,
77 /// Optional crash notifier for supervision.
78 pub(super) crash_notifier: Option<mpsc::Sender<CrashNotification>>,
79 /// Whether tracing is enabled for route pipelines.
80 pub(super) tracer_gating: TracerPipelineGating,
81 /// Detail level for tracing when enabled.
82 pub(super) tracer_detail_level: DetailLevel,
83 /// Metrics collector for tracing processor.
84 pub(super) tracer_metrics: Option<Arc<dyn MetricsCollector>>,
85 pub(super) platform_service: Arc<dyn PlatformService>,
86 pub(super) function_invoker: Option<Arc<dyn FunctionInvoker>>,
87 pub(super) health_registry: Option<Arc<HealthCheckRegistry>>,
88 /// Shared idempotent repository registry. Defaults to an empty registry;
89 /// the CamelContext builder installs a populated handle that includes the
90 /// built-in `"memory"` repository.
91 pub(super) idempotent_repositories: crate::SharedIdempotentRegistry,
92 pub(super) claim_check_repositories: crate::SharedClaimCheckRegistry,
93 pub(super) cache_repositories: crate::SharedCacheRegistry,
94 /// F2 staging: prepared-but-not-inserted ManagedRoutes keyed by route_id.
95 /// `prepare_*` writes here; `insert_prepared_route` drains via `remove()`.
96 /// On insert-failure error paths, the caller (`reload_actions.rs`) must
97 /// explicitly drain to avoid orphan CancellationToken/SharedPipeline leaks.
98 pub(super) prepared_staging: HashMap<String, ManagedRoute>,
99 /// Source endpoint URI to route_id index (one-to-many).
100 pub(super) endpoint_index: super::endpoint_index::EndpointIndex,
101 /// Operator acknowledgements for per-bind public exposure (ADR-0061).
102 /// Empty by default → the gate fails closed on non-loopback binds.
103 pub(super) bind_acks: super::route_controller_trait::BindExposureAcks,
104 /// Route send-point interception rules captured by step compilation.
105 pub(super) intercept: InterceptRules,
106 /// Intercept-rules freeze. Trips on `add_route` success and on the
107 /// `MarkStarted` actor command; never reset (stop/restart included),
108 /// because compiled pipelines capture the rules at compile time.
109 pub(super) frozen: bool,
110 /// Startup-cohort activation barrier (rc-jxkj). Cloned into the
111 /// `RouteControllerHandle` at spawn; reset/activate act on this shared
112 /// gate directly, never through the actor.
113 pub(super) cohort: Arc<CohortActivationGate>,
114 /// Context-global accepted-not-completed counter (drainclaim): the
115 /// SAME `Arc` the owning `CamelContext` exposes through
116 /// `total_in_flight()` (installed by the builder). Standalone
117 /// controllers keep an isolated zero counter — claims flow, but only
118 /// the constructing scope can read them.
119 pub(super) in_flight_total: Arc<AtomicU64>,
120}
121
122impl DefaultRouteController {
123 /// Open the startup-cohort activation barrier (rc-jxkj).
124 ///
125 /// The CamelContext lifecycle opens this automatically once the startup
126 /// cohort completes. Consumers that drive a bare `DefaultRouteController`
127 /// (outside a full context) must call this before dispatching
128 /// (typically after starting routes), or pipeline dispatch parks
129 /// every envelope until the caller's call timeout surfaces as a
130 /// failure.
131 pub fn activate_cohort(&self) {
132 self.cohort.open();
133 }
134
135 pub(super) fn health_registry(&self) -> Arc<HealthCheckRegistry> {
136 self.health_registry.clone().unwrap_or_else(|| {
137 debug!("health_registry not configured — creating isolated fallback");
138 Arc::new(HealthCheckRegistry::new(Duration::from_secs(5)))
139 })
140 }
141
142 /// Create a new `DefaultRouteController` with the given registry.
143 pub fn new(
144 registry: Arc<std::sync::Mutex<Registry>>,
145 platform_service: Arc<dyn PlatformService>,
146 ) -> Self {
147 Self::with_beans_and_platform_service(
148 registry,
149 Arc::new(std::sync::Mutex::new(BeanRegistry::new())),
150 platform_service,
151 )
152 }
153
154 /// Create a new `DefaultRouteController` with shared bean registry.
155 pub fn with_beans(
156 registry: Arc<std::sync::Mutex<Registry>>,
157 beans: Arc<std::sync::Mutex<BeanRegistry>>,
158 ) -> Self {
159 Self::with_beans_and_platform_service(
160 registry,
161 beans,
162 Arc::new(NoopPlatformService::default()),
163 )
164 }
165
166 fn with_beans_and_platform_service(
167 registry: Arc<std::sync::Mutex<Registry>>,
168 beans: Arc<std::sync::Mutex<BeanRegistry>>,
169 platform_service: Arc<dyn PlatformService>,
170 ) -> Self {
171 Self {
172 routes: RouteRegistry::new(),
173 registry,
174 languages: Arc::new(std::sync::Mutex::new(HashMap::new())),
175 beans,
176 runtime: None,
177 global_error_handler: None,
178 crash_notifier: None,
179 tracer_gating: TracerPipelineGating::off(),
180 tracer_detail_level: DetailLevel::Minimal,
181 tracer_metrics: None,
182 platform_service,
183 function_invoker: None,
184 health_registry: None,
185 idempotent_repositories: Arc::new(crate::IdempotentRegistry::new()),
186 claim_check_repositories: Arc::new(crate::ClaimCheckRegistry::new()),
187 cache_repositories: Arc::new(crate::CacheRegistry::new()),
188 prepared_staging: HashMap::new(),
189 endpoint_index: super::endpoint_index::EndpointIndex::new(),
190 bind_acks: Default::default(),
191 intercept: InterceptRules::default(),
192 frozen: false,
193 cohort: Arc::new(CohortActivationGate::new_closed()),
194 in_flight_total: Arc::new(AtomicU64::new(0)),
195 }
196 }
197
198 /// Create a new `DefaultRouteController` with shared language registry.
199 pub fn with_languages(
200 registry: Arc<std::sync::Mutex<Registry>>,
201 languages: SharedLanguageRegistry,
202 platform_service: Arc<dyn PlatformService>,
203 ) -> Self {
204 Self {
205 routes: RouteRegistry::new(),
206 registry,
207 languages,
208 beans: Arc::new(std::sync::Mutex::new(BeanRegistry::new())),
209 runtime: None,
210 global_error_handler: None,
211 crash_notifier: None,
212 tracer_gating: TracerPipelineGating::off(),
213 tracer_detail_level: DetailLevel::Minimal,
214 tracer_metrics: None,
215 platform_service,
216 function_invoker: None,
217 health_registry: None,
218 idempotent_repositories: Arc::new(crate::IdempotentRegistry::new()),
219 claim_check_repositories: Arc::new(crate::ClaimCheckRegistry::new()),
220 cache_repositories: Arc::new(crate::CacheRegistry::new()),
221 prepared_staging: HashMap::new(),
222 endpoint_index: super::endpoint_index::EndpointIndex::new(),
223 bind_acks: Default::default(),
224 intercept: InterceptRules::default(),
225 frozen: false,
226 cohort: Arc::new(CohortActivationGate::new_closed()),
227 in_flight_total: Arc::new(AtomicU64::new(0)),
228 }
229 }
230
231 pub fn with_languages_and_beans(
232 registry: Arc<std::sync::Mutex<Registry>>,
233 languages: SharedLanguageRegistry,
234 platform_service: Arc<dyn PlatformService>,
235 beans: Arc<std::sync::Mutex<BeanRegistry>>,
236 ) -> Self {
237 Self {
238 routes: RouteRegistry::new(),
239 registry,
240 languages,
241 beans,
242 runtime: None,
243 global_error_handler: None,
244 crash_notifier: None,
245 tracer_gating: TracerPipelineGating::off(),
246 tracer_detail_level: DetailLevel::Minimal,
247 tracer_metrics: None,
248 platform_service,
249 function_invoker: None,
250 health_registry: None,
251 idempotent_repositories: Arc::new(crate::IdempotentRegistry::new()),
252 claim_check_repositories: Arc::new(crate::ClaimCheckRegistry::new()),
253 cache_repositories: Arc::new(crate::CacheRegistry::new()),
254 prepared_staging: HashMap::new(),
255 endpoint_index: super::endpoint_index::EndpointIndex::new(),
256 bind_acks: Default::default(),
257 intercept: InterceptRules::default(),
258 frozen: false,
259 cohort: Arc::new(CohortActivationGate::new_closed()),
260 in_flight_total: Arc::new(AtomicU64::new(0)),
261 }
262 }
263
264 pub fn with_function_invoker(mut self, function_invoker: Arc<dyn FunctionInvoker>) -> Self {
265 self.function_invoker = Some(function_invoker);
266 self
267 }
268
269 pub(crate) fn set_idempotent_repositories(
270 &mut self,
271 repositories: crate::SharedIdempotentRegistry,
272 ) {
273 self.idempotent_repositories = repositories;
274 }
275
276 pub(crate) fn set_claim_check_repositories(
277 &mut self,
278 repositories: crate::SharedClaimCheckRegistry,
279 ) {
280 self.claim_check_repositories = repositories;
281 }
282
283 pub(crate) fn set_cache_repositories(&mut self, repositories: crate::SharedCacheRegistry) {
284 self.cache_repositories = repositories;
285 }
286
287 pub fn set_health_registry(&mut self, registry: Arc<HealthCheckRegistry>) {
288 self.health_registry = Some(registry);
289 }
290
291 pub fn set_function_invoker(&mut self, invoker: Arc<dyn FunctionInvoker>) {
292 self.function_invoker = Some(invoker);
293 }
294
295 /// Set runtime handle for ProducerContext creation.
296 pub fn set_runtime_handle(&mut self, runtime: Arc<dyn RuntimeHandle>) {
297 self.runtime = Some(Arc::downgrade(&runtime));
298 }
299
300 /// Set the crash notifier for supervision.
301 ///
302 /// When set, the controller will send a `CrashNotification` whenever
303 /// a consumer crashes.
304 pub fn set_crash_notifier(&mut self, tx: mpsc::Sender<CrashNotification>) {
305 self.crash_notifier = Some(tx);
306 }
307
308 /// Set a global error handler applied to all routes without a per-route handler.
309 /// Install operator acknowledgements for per-bind public exposure
310 /// (ADR-0061). Built by the CLI from `CamelConfig.binds`.
311 pub fn set_bind_exposure_acks(&mut self, acks: BindExposureAcks) {
312 self.bind_acks = acks;
313 }
314
315 /// Install route send-point interception rules (pre-first-use only).
316 ///
317 /// Fails with `CamelError::Config` once the freeze has tripped: compiled
318 /// pipelines capture the rules at compile time, so the rule set must not
319 /// change after a route is added or the context is started.
320 pub fn set_intercept_rules(&mut self, rules: InterceptRules) -> Result<(), CamelError> {
321 if self.frozen {
322 return Err(CamelError::Config(
323 "intercept rules are frozen: a route was added or the context was started; \
324 rules cannot be changed after first use"
325 .into(),
326 ));
327 }
328 self.intercept = rules;
329 Ok(())
330 }
331
332 /// Builder-style build-time configuration on a fresh controller (which
333 /// is never frozen); mirrors `with_function_invoker`.
334 pub fn with_intercept_rules(mut self, rules: InterceptRules) -> Self {
335 self.intercept = rules;
336 self
337 }
338
339 /// Trip the intercept-rules freeze. Dispatched by the `MarkStarted`
340 /// actor command so the freeze applies even with zero routes. Never
341 /// unset.
342 pub fn mark_started(&mut self) {
343 self.frozen = true;
344 }
345
346 /// All compiled security plans whose routes bind the same listener
347 /// address (bind key), for the per-bind exposure gate. Routes still
348 /// staging (no plan yet) are skipped — classification failures already
349 /// aborted their own staging (Task 1.8).
350 pub(super) fn plans_for_bind(
351 &self,
352 bind_key: &str,
353 ) -> Vec<(String, camel_api::security_policy::RouteSecurityPlan)> {
354 self.routes
355 .iter()
356 .filter_map(|(route_id, managed)| {
357 let plan = managed.compiled.security_plan.as_ref()?;
358 let bind = bind_key_from_uri(&managed.from_uri)?;
359 if bind.key == bind_key {
360 Some((route_id.clone(), plan.clone()))
361 } else {
362 None
363 }
364 })
365 .collect()
366 }
367
368 /// Return whether a listener for `bind_key` is already serving a route.
369 ///
370 /// This is deliberately based on the consumer handle, rather than on the
371 /// presence of a compiled route: the exposure gate applies only to late
372 /// registration against an already-running bind (Task 2.2). Startup and
373 /// resume perform their own gate checks in the lifecycle implementation.
374 fn bind_is_running(&self, bind_key: &str) -> bool {
375 self.routes.iter().any(|(_, managed)| {
376 bind_key_from_uri(&managed.from_uri).is_some_and(|bind| {
377 bind.key == bind_key && handle_is_running(&managed.consumer_handle)
378 })
379 })
380 }
381
382 /// Gate a route before it is inserted into a listener that is already
383 /// running. The candidate is included with all sibling plans so the gate
384 /// has the same aggregation semantics as the start path.
385 fn enforce_late_registration_gate(
386 &self,
387 from_uri: &str,
388 route_id: &str,
389 plan: Option<&camel_api::security_policy::RouteSecurityPlan>,
390 ) -> Result<(), CamelError> {
391 let Some(bind) = bind_key_from_uri(from_uri) else {
392 return Ok(());
393 };
394 if !self.bind_is_running(&bind.key) {
395 return Ok(());
396 }
397
398 let mut owned = self.plans_for_bind(&bind.key);
399 if let Some(plan) = plan {
400 owned.push((route_id.to_string(), plan.clone()));
401 }
402 let plans: Vec<(&str, &camel_api::security_policy::RouteSecurityPlan)> =
403 owned.iter().map(|(id, plan)| (id.as_str(), plan)).collect();
404 enforce_bind_exposure_gate(
405 &bind.key,
406 bind.loopback,
407 &plans,
408 self.bind_acks.acknowledged(&bind.key),
409 )
410 .map_err(|err| {
411 CamelError::RouteError(format!(
412 "late registration of route '{route_id}' rejected for bind '{}': {err}",
413 bind.key
414 ))
415 })
416 }
417
418 pub fn set_error_handler(&mut self, config: ErrorHandlerConfig) {
419 self.global_error_handler = Some(config);
420 }
421
422 /// Configure tracing for this route controller.
423 pub fn set_tracer_config(&mut self, config: &TracerConfig) {
424 // Pipeline wrapping follows tracing unless the effective assembly
425 // raised it (exporter active); spans follow `enabled` alone.
426 self.tracer_gating = TracerPipelineGating {
427 pipeline_enabled: config.enabled || config.pipeline_enabled,
428 spans_enabled: config.enabled,
429 levers: config.metrics_levers.clone(),
430 };
431 self.tracer_detail_level = config.detail_level.clone();
432 }
433
434 /// Seed the tracer metrics collector — the shared late-bound
435 /// `MetricsHandle` built once by `CamelContextBuilder::build()`.
436 ///
437 /// Replaces the deleted `TracerConfig.metrics_collector` snapshot
438 /// injection: the collector is wired here, at construction, and late
439 /// registrations flow through the handle without re-snapshotting.
440 pub fn set_tracer_metrics(&mut self, metrics: Arc<dyn MetricsCollector>) {
441 self.tracer_metrics = Some(metrics);
442 }
443
444 /// Install the context-global accepted-not-completed counter
445 /// (drainclaim) — the SAME `Arc<AtomicU64>` the owning
446 /// `CamelContext` reads through `total_in_flight()`. Called once by
447 /// `CamelContextBuilder::build()`; contexts built without a builder
448 /// (or standalone controllers) keep the isolated zero counter from
449 /// construction.
450 pub fn set_in_flight_total(&mut self, counter: Arc<AtomicU64>) {
451 self.in_flight_total = counter;
452 }
453
454 fn build_producer_context(&self, route_id: &str) -> Result<ProducerContext, CamelError> {
455 let mut producer_ctx = ProducerContext::new().with_route_id(route_id);
456 if let Some(runtime) = self.runtime.as_ref().and_then(Weak::upgrade) {
457 producer_ctx = producer_ctx.with_runtime(runtime);
458 }
459 Ok(producer_ctx)
460 }
461
462 /// Create a transient [`RouteCompilerExt`] from this controller's fields.
463 fn route_compiler_ext(&self) -> RouteCompilerExt<'_> {
464 RouteCompilerExt {
465 registry: &self.registry,
466 languages: &self.languages,
467 beans: &self.beans,
468 function_invoker: &self.function_invoker,
469 tracer_gating: self.tracer_gating.clone(),
470 tracer_detail_level: &self.tracer_detail_level,
471 tracer_metrics: &self.tracer_metrics,
472 platform_service: &self.platform_service,
473 runtime: &self.runtime,
474 global_error_handler: &self.global_error_handler,
475 health_registry: &self.health_registry,
476 route_registry: &self.routes,
477 idempotent_repositories: Arc::clone(&self.idempotent_repositories),
478 claim_check_repositories: Arc::clone(&self.claim_check_repositories),
479 cache_repositories: Arc::clone(&self.cache_repositories),
480 intercept: &self.intercept,
481 in_flight_total: Arc::clone(&self.in_flight_total),
482 }
483 }
484
485 /// Resolve BuilderSteps into BoxProcessors.
486 #[allow(dead_code)] // used by tests and may be needed for future split paths
487 pub(crate) fn resolve_steps(
488 &self,
489 steps: Vec<BuilderStep>,
490 producer_ctx: &ProducerContext,
491 registry: &Arc<std::sync::Mutex<Registry>>,
492 route_id: Option<&str>,
493 staging_mode: &super::step_resolution::FunctionStagingMode,
494 ) -> Result<Vec<CompiledStep>, CamelError> {
495 let component_ctx = Arc::new(ControllerComponentContext::new(
496 Arc::clone(registry),
497 Arc::clone(&self.languages),
498 self.tracer_metrics
499 .clone()
500 .unwrap_or_else(|| Arc::new(NoOpMetrics)),
501 Arc::clone(&self.platform_service),
502 self.health_registry(),
503 route_id.map(|s| s.to_string()),
504 self.tracer_gating.levers.components_enabled(),
505 ));
506 let rt: Arc<dyn camel_component_api::RuntimeObservability> =
507 Arc::clone(&component_ctx) as Arc<_>;
508
509 super::step_resolution::resolve_steps(
510 steps,
511 producer_ctx,
512 rt,
513 registry,
514 &self.languages,
515 &self.beans,
516 self.function_invoker.clone(),
517 component_ctx,
518 route_id,
519 staging_mode,
520 &self.idempotent_repositories,
521 &self.claim_check_repositories,
522 &self.cache_repositories,
523 self.intercept.clone(),
524 )
525 }
526
527 /// Add a route definition to the controller.
528 ///
529 /// Steps are resolved immediately using the registry.
530 ///
531 /// # Errors
532 ///
533 /// Returns an error if:
534 /// - A route with the same ID already exists
535 /// - Step resolution fails
536 pub async fn add_route(&mut self, definition: RouteDefinition) -> Result<(), CamelError> {
537 let route_id = definition.route_id().to_string();
538 let from_uri = definition.from_uri().to_string();
539
540 if self.routes.contains_key(&route_id) {
541 return Err(CamelError::RouteError(format!(
542 "duplicate route ID '{route_id}'"
543 )));
544 }
545
546 debug!(route_id = %route_id, "Adding route to controller");
547
548 let managed = match self.build_managed_route(
549 definition,
550 &super::step_resolution::FunctionStagingMode::DirectAdd,
551 ) {
552 Ok(managed) => managed,
553 Err(err) => {
554 self.discard_function_staging();
555 return Err(err);
556 }
557 };
558
559 // A running listener can observe a newly inserted route immediately.
560 // Gate the complete sibling set before committing function staging or
561 // inserting the route, so a rejected late registration is unreachable.
562 if let Err(err) = self.enforce_late_registration_gate(
563 &from_uri,
564 &route_id,
565 managed.compiled.security_plan.as_ref(),
566 ) {
567 self.discard_function_staging();
568 return Err(err);
569 }
570
571 if let Some(invoker) = &self.function_invoker
572 && let Err(err) = invoker.commit_staged().await
573 {
574 invoker.discard_staging(0);
575 return Err(CamelError::Config(err.to_string()));
576 }
577
578 self.routes
579 .insert(managed.definition.route_id().to_string(), managed);
580
581 self.endpoint_index.insert(&from_uri, &route_id);
582 // First successful route registration freezes the intercept rules:
583 // this route's pipeline compiled against the current rule set.
584 self.frozen = true;
585 Ok(())
586 }
587
588 pub(super) fn build_managed_route(
589 &self,
590 definition: RouteDefinition,
591 staging_mode: &super::step_resolution::FunctionStagingMode,
592 ) -> Result<ManagedRoute, CamelError> {
593 let route_id = definition.route_id().to_string();
594
595 let definition_info = definition.to_info();
596
597 // Security plan compilation (Task 1.8): consumer-backed routes get a
598 // plan BEFORE any consumer starts; a declared route that fails
599 // classification aborts staging (never a Public downgrade). Routes
600 // without a provider registry compile against an empty view.
601 let empty_providers;
602 let providers = match &definition.provider_registry {
603 Some(registry) => registry.as_ref(),
604 None => {
605 empty_providers = camel_auth::ProviderRegistry::new();
606 &empty_providers
607 }
608 };
609 let security_plan =
610 super::route_compiler_ext::compile_route_security_plan(&definition, providers)?;
611
612 let RouteDefinition {
613 from_uri,
614 steps,
615 error_handler,
616 circuit_breaker,
617 circuit_breaker_fallback,
618 security_policy,
619 security_authenticator,
620 provider_registry,
621 unit_of_work,
622 concurrency,
623 ..
624 } = definition;
625
626 let producer_ctx = self.build_producer_context(&route_id)?;
627
628 // N2: reject mixed Aggregate + Resequence top-level splits
629 assert_no_mixed_top_level_splits(&steps)?;
630
631 let (aggregate_split, processors_with_contracts) = self
632 .route_compiler_ext()
633 .detect_and_validate_route_split(steps, &producer_ctx, &route_id, staging_mode)?;
634 let mut lifecycle = super::route_helpers::collect_lifecycle(&processors_with_contracts);
635
636 // CB fallback (mirrors on_miss lifecycle packing): attach via the
637 // shared helper, then merge the fallback lifecycle handles into the
638 // route vec.
639 let (circuit_breaker, fallback_lifecycle) = self.route_compiler_ext().attach_cb_fallback(
640 circuit_breaker,
641 circuit_breaker_fallback,
642 &producer_ctx,
643 &route_id,
644 staging_mode,
645 )?;
646 lifecycle.extend(fallback_lifecycle);
647 let route_id_for_tracing = route_id.clone();
648 let eh_config = error_handler.or_else(|| self.global_error_handler.clone());
649 let transport = transport_from_uri(&from_uri);
650
651 let mut pipeline = build_eh_config_pipeline(
652 eh_config.as_ref(),
653 Arc::clone(&self.registry),
654 Arc::clone(&self.languages),
655 self.tracer_metrics.clone(),
656 Arc::clone(&self.platform_service),
657 self.health_registry(),
658 &route_id_for_tracing,
659 &producer_ctx,
660 processors_with_contracts,
661 self.tracer_gating.clone(),
662 self.tracer_detail_level.clone(),
663 security_policy.clone(),
664 transport,
665 circuit_breaker,
666 Arc::clone(&self.in_flight_total),
667 )?;
668
669 let uow_counter = if let Some(uow_config) = &unit_of_work {
670 let component_ctx = Arc::new(
671 ControllerComponentContext::new(
672 Arc::clone(&self.registry),
673 Arc::clone(&self.languages),
674 self.tracer_metrics
675 .clone()
676 .unwrap_or_else(|| Arc::new(NoOpMetrics)),
677 Arc::clone(&self.platform_service),
678 self.health_registry(),
679 Some(route_id.clone()),
680 self.tracer_gating.levers.components_enabled(),
681 )
682 .with_in_flight(Arc::clone(&self.in_flight_total)),
683 );
684 let rt: Arc<dyn camel_component_api::RuntimeObservability> =
685 Arc::clone(&component_ctx) as Arc<_>;
686 let (uow_layer, counter) = super::route_compiler_ext::resolve_uow_layer(
687 uow_config,
688 &producer_ctx,
689 rt,
690 component_ctx.as_ref(),
691 None,
692 )?;
693 pipeline = BoxProcessor::new(uow_layer.layer(pipeline));
694 Some(counter)
695 } else {
696 None
697 };
698
699 Ok(ManagedRoute {
700 definition: definition_info,
701 from_uri,
702 pipeline: super::pipeline_runtime::new_shared_pipeline_with_lifecycle(
703 pipeline, lifecycle,
704 ),
705 concurrency,
706 consumer_handle: None,
707 pipeline_handle: None,
708 consumer_cancel_token: CancellationToken::new(),
709 pipeline_cancel_token: CancellationToken::new(),
710 channel_sender: None,
711 in_flight: uow_counter,
712 drain_in_flight: Arc::new(std::sync::atomic::AtomicU64::new(0)),
713 aggregate_split,
714 agg_service: None,
715 compiled: route_runtime_state::CompiledRoute {
716 security_policy,
717 security_authenticator,
718 provider_registry,
719 security_plan,
720 },
721 })
722 }
723
724 pub async fn add_route_with_generation(
725 &mut self,
726 definition: RouteDefinition,
727 generation: u64,
728 ) -> Result<(), CamelError> {
729 let route_id = definition.route_id().to_string();
730 let from_uri = definition.from_uri().to_string();
731
732 if self.routes.contains_key(&route_id) {
733 return Err(CamelError::RouteError(format!(
734 "duplicate route ID '{route_id}'"
735 )));
736 }
737
738 debug!(route_id = %route_id, generation, "Adding route to controller with generation");
739
740 let managed = self.build_managed_route(
741 definition,
742 &super::step_resolution::FunctionStagingMode::HotReload { generation },
743 )?;
744
745 // Symmetric with `add_route`: an already-running bind must never
746 // observe an ungated insertion through the hot-reload path.
747 if let Err(err) = self.enforce_late_registration_gate(
748 &from_uri,
749 &route_id,
750 managed.compiled.security_plan.as_ref(),
751 ) {
752 self.discard_function_staging();
753 return Err(err);
754 }
755
756 self.routes.insert(route_id.clone(), managed);
757
758 self.endpoint_index.insert(&from_uri, &route_id);
759 Ok(())
760 }
761
762 pub async fn remove_route_preserving_functions(
763 &mut self,
764 route_id: &str,
765 ) -> Result<(), CamelError> {
766 let managed = self.routes.get(route_id).ok_or_else(|| {
767 CamelError::RouteError(format!("Route '{}' not found for removal", route_id))
768 })?;
769 if handle_is_running(&managed.consumer_handle)
770 || handle_is_running(&managed.pipeline_handle)
771 {
772 return Err(CamelError::RouteError(format!(
773 "Route '{}' must be stopped before removal (current execution lifecycle: {})",
774 route_id,
775 inferred_lifecycle_label(managed)
776 )));
777 }
778 self.routes.remove(route_id);
779 if let Some(reg) = &self.health_registry {
780 reg.unregister_for_route(route_id);
781 }
782 self.endpoint_index.remove(route_id);
783 debug!(route_id = %route_id, "Route removed from controller (functions preserved for reload finalize)");
784 Ok(())
785 }
786
787 /// Compile a route definition into a processor pipeline, without adding it
788 /// to the controller. Used for validation and testing.
789 pub fn compile_route_definition(
790 &self,
791 def: RouteDefinition,
792 ) -> Result<BoxProcessor, CamelError> {
793 self.route_compiler_ext().compile_route_definition(def)
794 }
795
796 /// Compile a route definition with a specific generation (for hot-reload).
797 pub fn compile_route_definition_with_generation(
798 &self,
799 def: RouteDefinition,
800 generation: u64,
801 ) -> Result<BoxProcessor, CamelError> {
802 self.route_compiler_ext()
803 .compile_route_definition_with_generation(def, generation)
804 }
805
806 /// Compile a route definition into a [`CompiledPipeline`] (processor +
807 /// lifecycle handles). Used by the hot-reload Restart path so that
808 /// lifecycle handles are threaded through
809 /// [`swap_pipeline_raw`](Self::swap_pipeline_raw).
810 pub(crate) fn compile_route_definition_pipeline(
811 &self,
812 def: RouteDefinition,
813 generation: u64,
814 ) -> Result<CompiledPipeline, CamelError> {
815 self.route_compiler_ext()
816 .compile_route_definition_pipeline(def, generation)
817 }
818
819 /// Compile without function generation, returning full [`CompiledPipeline`].
820 ///
821 /// Oracle Fix 1: used by the stateless hot-reload path so that
822 /// lifecycle-bearing routes have their handles preserved.
823 pub(crate) fn compile_route_definition_dry_pipeline(
824 &self,
825 def: RouteDefinition,
826 ) -> Result<CompiledPipeline, CamelError> {
827 self.route_compiler_ext()
828 .compile_route_definition_dry_pipeline(def)
829 }
830
831 /// Remove a route from the controller map.
832 ///
833 /// The route **must** be stopped before removal (status `Stopped` or `Failed`).
834 /// Returns an error if the route is still running or does not exist.
835 /// Does not cancel any running tasks — call `stop_route` first.
836 pub async fn remove_route(&mut self, route_id: &str) -> Result<(), CamelError> {
837 let managed = self.routes.get(route_id).ok_or_else(|| {
838 CamelError::RouteError(format!("Route '{}' not found for removal", route_id))
839 })?;
840 if handle_is_running(&managed.consumer_handle)
841 || handle_is_running(&managed.pipeline_handle)
842 {
843 return Err(CamelError::RouteError(format!(
844 "Route '{}' must be stopped before removal (current execution lifecycle: {})",
845 route_id,
846 inferred_lifecycle_label(managed)
847 )));
848 }
849 if let Some(invoker) = &self.function_invoker {
850 for (id, rid) in self.collect_function_refs(route_id) {
851 if let Err(e) = invoker.unregister(&id, rid.as_deref()).await {
852 warn!(route_id = %route_id, error = %e, "Failed to unregister function during route removal");
853 }
854 }
855 }
856 self.routes.remove(route_id);
857 if let Some(reg) = &self.health_registry {
858 reg.unregister_for_route(route_id);
859 }
860 self.endpoint_index.remove(route_id);
861 info!(route_id = %route_id, "Route removed from controller");
862 Ok(())
863 }
864
865 fn collect_function_refs(
866 &self,
867 route_id: &str,
868 ) -> Vec<(camel_api::FunctionId, Option<String>)> {
869 self.function_invoker
870 .as_ref()
871 .map(|invoker| invoker.function_refs_for_route(route_id))
872 .unwrap_or_default()
873 }
874
875 fn discard_function_staging(&self) {
876 if let Some(invoker) = &self.function_invoker {
877 invoker.discard_staging(0);
878 }
879 }
880
881 /// Returns the number of routes in the controller.
882 pub fn route_count(&self) -> usize {
883 self.routes.route_count()
884 }
885
886 pub fn in_flight_count(&self, route_id: &str) -> Option<u64> {
887 self.routes.in_flight_count(route_id)
888 }
889
890 /// Returns `true` if a route with the given ID exists.
891 pub fn route_exists(&self, route_id: &str) -> bool {
892 self.routes.route_exists(route_id)
893 }
894
895 /// Returns all route IDs.
896 pub fn route_ids(&self) -> Vec<String> {
897 self.routes.route_ids()
898 }
899
900 pub fn route_source_hash(&self, route_id: &str) -> Option<u64> {
901 self.routes.route_source_hash(route_id)
902 }
903
904 /// Returns route IDs that should auto-start, sorted by startup order (ascending).
905 pub fn auto_startup_route_ids(&self) -> Vec<String> {
906 self.routes.auto_startup_route_ids()
907 }
908
909 /// Returns route IDs sorted by shutdown order (startup order descending).
910 pub fn shutdown_route_ids(&self) -> Vec<String> {
911 self.routes.shutdown_route_ids()
912 }
913
914 /// Atomically swap the pipeline of a route (zero-downtime).
915 ///
916 /// In-flight requests finish with the old pipeline (kept alive by Arc).
917 /// New requests immediately use the new pipeline.
918 ///
919 /// ## Rejection policy
920 ///
921 /// Returns an error if the route has lifecycle-bearing steps or an active
922 /// aggregate — these require the **Restart path** (stop → swap → start).
923 ///
924 /// The caller (e.g. `reload_actions::apply_swap`) MUST catch this rejection
925 /// and fall back to:
926 /// 1. `stop_route_reload` — drain lifecycle, stop consumer
927 /// 2. `swap_pipeline_raw` — bypass the lifecycle check (route is stopped)
928 /// 3. `start_route_reload` — re-create consumer with the new pipeline
929 ///
930 /// This is the "reject, don't defer" policy (oracle Fix 3): the swap is
931 /// refused upfront rather than silently deferring or partially swapping.
932 pub fn swap_pipeline(
933 &self,
934 route_id: &str,
935 new_pipeline: BoxProcessor,
936 ) -> Result<(), CamelError> {
937 let managed = self
938 .routes
939 .get(route_id)
940 .ok_or_else(|| CamelError::RouteError(format!("Route '{}' not found", route_id)))?;
941
942 let assembly = managed.pipeline.load();
943 let has_lifecycle = !assembly.lifecycle.is_empty();
944
945 if has_lifecycle || managed.agg_service.is_some() {
946 warn!(
947 route_id = %route_id,
948 "Hot-swap rejected — route has lifecycle/agg steps; use Restart path"
949 );
950 return Err(CamelError::RouteError(format!(
951 "Route '{}' contains stateful steps (lifecycle-bearing). Hot-swap not supported — use restart.",
952 route_id
953 )));
954 }
955
956 drop(assembly);
957
958 if managed.aggregate_split.is_some() {
959 warn!(
960 route_id = %route_id,
961 "swap_pipeline: aggregate routes with timeout do not support hot-reload of pre/post segments"
962 );
963 }
964
965 super::pipeline_runtime::swap_pipeline_raw(&managed.pipeline, new_pipeline, vec![]);
966 debug!(route_id = %route_id, "Pipeline swapped atomically");
967 Ok(())
968 }
969
970 /// Non-checking raw pipeline swap — bypasses lifecycle/aggregate rejection.
971 ///
972 /// Only for use after the route has been stopped (Restart path).
973 /// Does NOT check for lifecycle handles or aggregate service — the caller
974 /// is responsible for ensuring the route is safe to swap.
975 ///
976 /// Accepts `lifecycle` so that the new pipeline assembly records the
977 /// lifecycle handles from the compiled steps. When the route is
978 /// subsequently stopped, these handles are drained.
979 pub(crate) fn swap_pipeline_raw(
980 &self,
981 route_id: &str,
982 new_pipeline: BoxProcessor,
983 lifecycle: Vec<Arc<dyn StepLifecycle>>,
984 ) -> Result<(), CamelError> {
985 let managed = self
986 .routes
987 .get(route_id)
988 .ok_or_else(|| CamelError::RouteError(format!("Route '{}' not found", route_id)))?;
989 super::pipeline_runtime::swap_pipeline_raw(&managed.pipeline, new_pipeline, lifecycle);
990 debug!(route_id = %route_id, "Pipeline swapped (raw — lifecycle bypass)");
991 Ok(())
992 }
993
994 /// Returns the from_uri of a route, if it exists.
995 pub fn route_from_uri(&self, route_id: &str) -> Option<String> {
996 self.routes.route_from_uri(route_id)
997 }
998
999 /// Return all route_ids that consume from the given source endpoint URI.
1000 pub fn routes_for_endpoint(&self, uri: &str) -> Vec<String> {
1001 self.endpoint_index.routes_for(uri)
1002 }
1003
1004 /// Return all registered source endpoint URIs.
1005 pub fn list_endpoint_uris(&self) -> Vec<String> {
1006 self.endpoint_index.list_uris()
1007 }
1008
1009 /// Get a clone of the current pipeline for a route.
1010 ///
1011 /// This is useful for testing and introspection.
1012 /// Returns `None` if the route doesn't exist.
1013 pub fn get_pipeline(&self, route_id: &str) -> Option<BoxProcessor> {
1014 self.routes.get_pipeline(route_id)
1015 }
1016
1017 /// Check whether the running route has lifecycle-bearing steps.
1018 ///
1019 /// Returns `false` when the route is missing.
1020 pub(crate) fn route_has_lifecycle(&self, route_id: &str) -> bool {
1021 self.routes
1022 .get(route_id)
1023 .map(|managed| !managed.pipeline.load().lifecycle.is_empty())
1024 .unwrap_or(false)
1025 }
1026
1027 /// Internal stop implementation that can set custom status.
1028 pub(super) async fn stop_route_internal(&mut self, route_id: &str) -> Result<(), CamelError> {
1029 self.routes.stop_route(route_id).await
1030 }
1031
1032 pub async fn start_route_reload(&mut self, route_id: &str) -> Result<(), CamelError> {
1033 self.start_route(route_id).await
1034 }
1035
1036 pub async fn stop_route_reload(&mut self, route_id: &str) -> Result<(), CamelError> {
1037 self.stop_route(route_id).await
1038 }
1039}
1040
1041// ── Aggregator route helpers ──
1042
1043impl DefaultRouteController {
1044 /// Start a route with an aggregate split (pre-pipeline → aggregator → post-pipeline).
1045 ///
1046 /// Spawns a biased-select forward loop that routes exchanges through the
1047 /// pre-pipeline, aggregator, and post-pipeline in sequence, with late-exchange
1048 /// handling and force-completion on stop.
1049 ///
1050 /// drainclaim claim propagation: the forward loop submits each
1051 /// envelope's [`InFlightClaim`] into the aggregator WITH its exchange
1052 /// ([`AggregatorService::submit_with_claim`]) — a pending stash parks
1053 /// the claim inside the bucket so the stashed exchange stays counted,
1054 /// and every emission path (sync completion, timeout `late_tx` fire,
1055 /// `force_complete_all` at stop) hands the bucket's claims back to
1056 /// this loop, which holds them across the post-pipeline
1057 /// continuation. Paths that drop a bucket without emitting (TTL
1058 /// eviction, unarmed-bucket release, discard-on-timeout, saturated
1059 /// late channel) release by dropping. Exactly one release per claim
1060 /// on every path. claimfamily (rc-e1a4f): every oneshot dispatch
1061 /// through the pre/post pipelines splits a sibling claim onto the
1062 /// exchange (in-band results take it back, stash emissions escape
1063 /// with theirs), so stash sites embedded in those pipelines stay
1064 /// counted.
1065 #[allow(clippy::too_many_arguments)]
1066 pub(super) async fn start_aggregate_route(
1067 &mut self,
1068 route_id: &str,
1069 split: AggregateSplitInfo,
1070 consumer: Box<dyn Consumer>,
1071 consumer_ctx: ConsumerContext,
1072 mut rx: mpsc::Receiver<ExchangeEnvelope>,
1073 crash_notifier: Option<mpsc::Sender<CrashNotification>>,
1074 runtime_for_consumer: Option<Weak<dyn RuntimeHandle>>,
1075 tx_for_storage: mpsc::Sender<ExchangeEnvelope>,
1076 // Pipeline cancellation — a child of the managed route's pipeline_cancel_token.
1077 pipeline_cancel: CancellationToken,
1078 drain_in_flight: Arc<std::sync::atomic::AtomicU64>,
1079 ) -> Result<(), CamelError> {
1080 // drainclaim claim propagation: the late channel carries each
1081 // emission's stashed claims alongside the aggregated exchange —
1082 // the loop holds them across the post-pipeline continuation.
1083 let (late_tx, late_rx) =
1084 mpsc::channel::<camel_processor::aggregator::AggregateEmission>(256);
1085
1086 let route_cancel_clone = pipeline_cancel.clone();
1087 let mut svc = AggregatorService::new(
1088 split.agg_config.clone(),
1089 late_tx,
1090 Arc::clone(&self.languages),
1091 route_cancel_clone,
1092 );
1093 // Queue-depth visibility: the TTL-sweep pass reports the buffered
1094 // group count as camel_queue_depth{queue="aggregator:<route>"}.
1095 if let Some(metrics) = self.tracer_metrics.clone() {
1096 svc = svc.with_queue_metrics(metrics, format!("aggregator:{route_id}"));
1097 }
1098 let agg = Arc::new(svc);
1099
1100 let pipeline_cancel_for_monitor = pipeline_cancel.clone();
1101 // rc-e2r9: capture route_id and metrics for b′ emission at reply-drop
1102 // sites inside the spawned task.
1103 let route_id_for_metrics = route_id.to_string();
1104 let metrics_for_reply_drop = self.tracer_metrics.clone();
1105 // rc-jxkj cohort gate: owned by the forward loop — the envelope arm
1106 // parks dispatch until the startup cohort opens the gate.
1107 let mut cohort_rx = self.cohort.subscribe();
1108 let agg_for_monitor = Arc::clone(&agg);
1109
1110 {
1111 let managed = self
1112 .routes
1113 .get_mut(route_id)
1114 .expect("invariant: route must exist"); // allow-unwrap
1115 managed.agg_service = Some(Arc::clone(&agg));
1116 }
1117
1118 let late_rx = Arc::new(tokio::sync::Mutex::new(late_rx));
1119 let pre_pipeline = split.pre_pipeline;
1120 let post_pipeline = split.post_pipeline;
1121
1122 // Spawn biased select forward loop
1123 let pipeline_handle = tokio::spawn(async move {
1124 loop {
1125 tokio::select! {
1126 biased;
1127
1128 // Ungated by design (D3, rc-jxkj): late exchanges exist
1129 // only after a dispatched envelope traversed the
1130 // aggregator — transitively post-activation. Gating here
1131 // would be dead code and a self-deadlock risk.
1132 late_ex = async {
1133 let mut rx = late_rx.lock().await;
1134 rx.recv().await
1135 } => {
1136 match late_ex {
1137 Some(emission) => {
1138 // drainclaim: hold the emission's stashed
1139 // claims across the post-pipeline — drop
1140 // at iteration end is the release.
1141 let AggregateEmission {
1142 mut exchange,
1143 claims: _in_flight_claims,
1144 } = emission;
1145 // claimfamily (rc-e1a4f): split a sibling onto the emission exchange so
1146 // residency inside stash sites embedded in the post-pipeline stays
1147 // counted after this iteration. An in-band result drops at iteration end
1148 // (its sibling releases with it); a stash emission escapes with its
1149 // sibling; an Err drops it with the exchange.
1150 exchange.in_flight_claim = _in_flight_claims
1151 .iter()
1152 .flatten()
1153 .next()
1154 .map(InFlightClaim::split);
1155 let pipe = post_pipeline.load();
1156 if let Err(e) =
1157 pipe.processor.clone_inner().oneshot(exchange).await
1158 {
1159 tracing::warn!(error = %e, "late exchange post-pipeline failed");
1160 }
1161 }
1162 None => return,
1163 }
1164 }
1165
1166 envelope_opt = rx.recv() => {
1167 match envelope_opt {
1168 Some(envelope) => {
1169 // rc-jxkj cohort gate: park dispatch until
1170 // the startup cohort completes (same guard
1171 // as the non-aggregate drain loops).
1172 // rc-z5qz: `biased` with the gate polled
1173 // FIRST — with the cohort open AND the
1174 // pipeline token cancelled, both branches
1175 // are ready and an unbiased select picks
1176 // randomly, letting the cancel arm drop a
1177 // deliverable envelope (force_complete_all
1178 // then sees no buckets). Gate-open must win
1179 // deterministically; a genuinely closed
1180 // gate still drops on cancel (rc-jxkj
1181 // semantics preserved).
1182 tokio::select! {
1183 biased;
1184 _ = cohort_rx.wait_for(|open| *open) => {}
1185 _ = pipeline_cancel.cancelled() => {
1186 // Drop the envelope; reply_tx (if
1187 // any) resolves to ChannelClosed for
1188 // the send_and_wait waiter.
1189 // `continue`, not `return`: the token
1190 // is already cancelled, so the next
1191 // loop iteration lands in the biased
1192 // outer select's cancel arm below,
1193 // which runs the force_complete_all +
1194 // late_rx cleanup. A `return` would
1195 // skip that cleanup and silently
1196 // drop a pending bucket across a
1197 // stop→restart gate re-arm.
1198 continue;
1199 }
1200 }
1201 let ExchangeEnvelope {
1202 exchange,
1203 reply_tx,
1204 in_flight_claim,
1205 } = envelope;
1206 let _drain_guard = super::route_helpers::DrainGuard::new(Arc::clone(&drain_in_flight));
1207 // drainclaim claim PROPAGATION: the
1208 // envelope's claim travels WITH the
1209 // exchange into the aggregator — a
1210 // pending stash parks it in the bucket
1211 // (counted until the bucket completes),
1212 // and a sync completion returns it (with
1213 // the rest of the bucket's claims) to be
1214 // held across the post-pipeline below.
1215 // Rejection paths drop it inside the
1216 // service — rejected = released.
1217 let pre_pipe = pre_pipeline.load();
1218 // claimfamily (rc-e1a4f): split a sibling claim onto the exchange so
1219 // residency inside stash sites embedded in the pre-pipeline stays
1220 // counted after this oneshot resolves. Taken back from an in-band Ok
1221 // result below — the envelope's own claim still travels with the
1222 // exchange into the aggregator; a stash emission escapes with its
1223 // sibling; an Err drops it with the exchange.
1224 let mut exchange = exchange;
1225 exchange.in_flight_claim = in_flight_claim.as_ref().map(InFlightClaim::split);
1226 let ex = match pre_pipe.processor.clone_inner().oneshot(exchange).await {
1227 // claimfamily: in-band completion — reclaim the sibling so release
1228 // stays at this loop iteration.
1229 Ok(mut ex) => {
1230 ex.in_flight_claim = None;
1231 ex
1232 }
1233 Err(e) => {
1234 // rc-e2r9: the real error rides with the
1235 // result so a dropped receiver still gets
1236 // it (ConsumerStopping suppressed).
1237 send_reply_or_b_prime(
1238 reply_tx,
1239 Err(e),
1240 &metrics_for_reply_drop,
1241 &route_id_for_metrics,
1242 "aggregate:pre-pipeline",
1243 );
1244 continue;
1245 }
1246 };
1247
1248 let AggregationReceipt { reply, claims } =
1249 agg.submit_with_claim(ex, in_flight_claim).await;
1250 // drainclaim: hold the completed bucket's
1251 // claims across the post-pipeline
1252 // continuation; dropped at iteration end or
1253 // any `continue`/`return` (scope exit).
1254 let _completed_bucket_claims = claims;
1255
1256 match reply {
1257 Ok(ex) => {
1258 if !is_pending(&ex) {
1259 let post_pipe = post_pipeline.load();
1260 // claimfamily (rc-e1a4f): split a sibling from the completed
1261 // bucket's claims onto the aggregated output so residency inside
1262 // stash sites embedded in the post-pipeline stays counted after
1263 // this iteration. Taken back from an in-band result before the
1264 // reply — release stays at iteration end; a stash emission
1265 // escapes with its sibling; an Err drops it with the exchange.
1266 let mut ex = ex;
1267 ex.in_flight_claim = _completed_bucket_claims
1268 .iter()
1269 .flatten()
1270 .next()
1271 .map(InFlightClaim::split);
1272 let mut out = post_pipe.processor.clone_inner().oneshot(ex).await;
1273 if let Ok(ref mut out_ex) = out {
1274 out_ex.in_flight_claim = None;
1275 }
1276 // rc-e2r9 review, Important 1: this site
1277 // previously `let _ = send`ed the
1278 // post-pipeline result — an Err with an
1279 // abandoned receiver vanished silently.
1280 // The helper closes that gap.
1281 send_reply_or_b_prime(
1282 reply_tx,
1283 out,
1284 &metrics_for_reply_drop,
1285 &route_id_for_metrics,
1286 "aggregate:post-pipeline",
1287 );
1288 } else {
1289 // Pending Ok: a dropped reply is silent —
1290 // b′ is an ERROR signal.
1291 send_reply_or_b_prime(
1292 reply_tx,
1293 Ok(ex),
1294 &metrics_for_reply_drop,
1295 &route_id_for_metrics,
1296 "aggregate:pending",
1297 );
1298 }
1299 }
1300 Err(e) => {
1301 send_reply_or_b_prime(
1302 reply_tx,
1303 Err(e),
1304 &metrics_for_reply_drop,
1305 &route_id_for_metrics,
1306 "aggregate:pipeline",
1307 );
1308 }
1309 }
1310 }
1311 None => return,
1312 }
1313 }
1314
1315 _ = pipeline_cancel.cancelled() => {
1316 agg.force_complete_all();
1317 let mut rx_guard = late_rx.lock().await;
1318 while let Ok(late_ex) = rx_guard.try_recv() {
1319 // drainclaim: each forced emission's claims are
1320 // held across its post-pipeline continuation —
1321 // dropped after the oneshot completes (or
1322 // immediately, which is also a release).
1323 let AggregateEmission {
1324 mut exchange,
1325 claims: _in_flight_claims,
1326 } = late_ex;
1327 // claimfamily (rc-e1a4f): split a sibling onto the forced emission so
1328 // residency inside stash sites embedded in the post-pipeline stays
1329 // counted. The discarded result releases an in-band sibling with its
1330 // drop; a stash emission escapes with its sibling.
1331 exchange.in_flight_claim = _in_flight_claims
1332 .iter()
1333 .flatten()
1334 .next()
1335 .map(InFlightClaim::split);
1336 let pipe = post_pipeline.load();
1337 let _ = pipe.processor.clone_inner().oneshot(exchange).await;
1338 }
1339 break;
1340 }
1341 }
1342 }
1343 });
1344 #[cfg(test)]
1345 emit_start_route_event("pipeline_spawned", route_id);
1346
1347 // Start consumer after pipeline loop is spawned to avoid startup races
1348 // where consumers emit exchanges before the route pipeline begins polling.
1349 // rc-kh7c cleanup parity: capture the consumer's cancel token before
1350 // consumer_ctx moves into the task so the failure arm can stop child
1351 // tasks spawned by consumer.start().
1352 let consumer_cancel_for_cleanup = consumer_ctx.cancel_token();
1353 let (consumer_handle, startup_rx, watcher_inputs, outer_inputs) =
1354 super::consumer_management::spawn_consumer_task(
1355 route_id.to_string(),
1356 consumer,
1357 consumer_ctx,
1358 crash_notifier,
1359 runtime_for_consumer,
1360 false,
1361 );
1362
1363 // rc-w1u9: await consumer startup handshake for aggregate routes too
1364 // so bind failures surface as route-start errors.
1365 // For Immediate consumers the receiver is pre-resolved (rc-slvd).
1366 let startup_result =
1367 super::consumer_management::await_consumer_startup(startup_rx, "startup").await;
1368 // rc-kh7c: on failure, abort the orphaned consumer task and cancel the
1369 // pipeline so neither runs detached. The aggregate pipeline loop would
1370 // eventually self-clean via rx-drop + late_tx-drop, but cancelling
1371 // pipeline_cancel also triggers force_complete_all (aggregate cleanup).
1372 if let Err(e) = startup_result {
1373 consumer_handle.abort();
1374 pipeline_cancel_for_monitor.cancel();
1375 // Deliberate Explicit-failure cleanup parity with the trait start arm (rc-kh7c).
1376 consumer_cancel_for_cleanup.cancel();
1377 return Err(e);
1378 }
1379
1380 // Detached failure watcher for Immediate consumers (rc-slvd).
1381 if let Some(inputs) = watcher_inputs {
1382 super::consumer_management::spawn_failure_watcher(inputs);
1383 }
1384
1385 // Detached outer-task watcher for Explicit consumers (rc-a7rh):
1386 // spawned only after the handshake resolved Ok — rollback
1387 // terminations (abort-then-cancel above) happen before this point
1388 // and are never watched. Explicit consumers on aggregate routes
1389 // get identical coverage — no aggregate carve-out.
1390 if let Some(outer) = outer_inputs {
1391 super::consumer_management::spawn_outer_task_watcher(outer);
1392 }
1393
1394 // Extend the stored consumer handle through aggregate force-completion.
1395 // While this monitor drains pending buckets, handle_is_running still reports
1396 // the Route as running because forced exchanges may still be in post-pipeline.
1397 //
1398 // bd rc-iioeq: a natural consumer exit (e.g. timer repeatCount
1399 // exhausted) must NOT destroy buckets whose inactivity timeout is
1400 // armed. With `force_completion_on_stop=false` (the default),
1401 // `force_complete_all` CANCELS the armed timeout task and silently
1402 // discards the bucket, so the inactivity emission never happens.
1403 // The forward loop stays up (the stored channel sender keeps the
1404 // input channel open), so an armed bucket still emits downstream
1405 // when its timeout fires. Buckets with no armed timeout task —
1406 // size/predicate-only, or timeout-configured but over the
1407 // `max_timeout_tasks` cap — can never complete after the consumer
1408 // exits; `release_unarmed_buckets` discards them eagerly so they
1409 // are not orphaned (the bucket_ttl sweep only runs inside the
1410 // pipeline's next exchange, which never arrives).
1411 let force_on_stop = agg_for_monitor.config().force_completion_on_stop;
1412 let consumer_handle = tokio::spawn(async move {
1413 let _ = consumer_handle.await;
1414 if !pipeline_cancel_for_monitor.is_cancelled() {
1415 if force_on_stop {
1416 agg_for_monitor.force_complete_all();
1417 pipeline_cancel_for_monitor.cancel();
1418 } else {
1419 agg_for_monitor.release_unarmed_buckets();
1420 }
1421 }
1422 });
1423 #[cfg(test)]
1424 emit_start_route_event("consumer_spawned", route_id);
1425
1426 {
1427 let managed = self
1428 .routes
1429 .get_mut(route_id)
1430 .expect("invariant: route must exist"); // allow-unwrap
1431 managed.consumer_handle = Some(consumer_handle);
1432 managed.pipeline_handle = Some(pipeline_handle);
1433 managed.channel_sender = Some(tx_for_storage);
1434 }
1435
1436 info!(route_id = %route_id, "Route started (aggregate with timeout)");
1437 Ok(())
1438 }
1439
1440 /// Test-only: inject lifecycle handles into an existing route's pipeline
1441 /// assembly. This makes the route lifecycle-bearing so that swap_pipeline
1442 /// rejects it, forcing callers (like reload_actions::apply_swap) to take
1443 /// the Restart path instead.
1444 #[cfg(test)]
1445 pub(crate) fn set_route_lifecycle_for_test(
1446 &mut self,
1447 route_id: &str,
1448 lifecycle: Vec<Arc<dyn StepLifecycle>>,
1449 ) -> Result<(), CamelError> {
1450 use super::pipeline_runtime::PipelineAssembly;
1451 use camel_api::SyncBoxProcessor;
1452 use std::sync::Arc;
1453
1454 let managed = self
1455 .routes
1456 .get_mut(route_id)
1457 .ok_or_else(|| CamelError::RouteError(format!("Route '{}' not found", route_id)))?;
1458 let old_processor = managed.pipeline.load().processor.clone_inner();
1459 managed.pipeline.store(Arc::new(PipelineAssembly::new(
1460 SyncBoxProcessor::new(old_processor),
1461 lifecycle,
1462 )));
1463 Ok(())
1464 }
1465}
1466
1467// ── rc-e2r9: b′ signal emission on reply-drop ──
1468// The shared `send_reply_or_b_prime` helper lives in
1469// `super::route_helpers` — it is used by both this module (aggregate drain
1470// loop) and `route_controller_trait` (Concurrent/Sequential pipelines).
1471
1472#[cfg(test)]
1473impl crate::hot_reload::ports::ReloadIntrospectionPort for DefaultRouteController {
1474 fn route_ids(&self) -> Vec<String> {
1475 self.route_ids() // inherent pub fn
1476 }
1477 fn route_from_uri(&self, route_id: &str) -> Option<String> {
1478 self.route_from_uri(route_id) // inherent pub fn
1479 }
1480 fn route_source_hash(&self, route_id: &str) -> Option<u64> {
1481 self.route_source_hash(route_id) // inherent pub fn
1482 }
1483}
1484
1485#[cfg(test)]
1486#[path = "route_controller_tests.rs"]
1487mod tests;
1488
1489#[cfg(test)]
1490#[path = "cohort_activation_regression.rs"]
1491mod cohort_activation_regression;