Skip to main content

camel_processor/
cache_eip.rs

1//! Cache EIP — outcome-aware Segment implementation.
2//!
3//! Implements the Caching pattern (lookup → on-miss sub-pipeline → write-back)
4//! at the `OutcomePipeline` layer (one layer above Tower), mirroring
5//! [`IdempotentConsumerSegment`]. On a cache HIT the body is reconstructed from
6//! the stored [`CacheEntry`] and the on-miss sub-pipeline is skipped entirely;
7//! on a MISS the sub-pipeline runs and its result body is written back into the
8//! repository (subject to `max_entry_bytes`).
9//!
10//! # Why Segment-mode (NOT Process-mode)
11//!
12//! Same rationale as the idempotent consumer: a Tower `Service<Exchange>` cannot
13//! propagate `PipelineOutcome::Stopped` distinctly from `Ok(ex)`. By implementing
14//! [`OutcomePipeline`] directly, a `Stopped` from the on-miss sub-pipeline flows
15//! out with the Exchange intact and NO write-back occurs (ADR-0024, ADR-0025).
16//!
17//! # Contract C1 (ADR-0023)
18//!
19//! [`CacheRepository::get`] / [`CacheRepository::set`] surface backend failures
20//! as `Err(CamelError)`. The segment propagates those as `PipelineOutcome::Failed`
21//! — it NEVER treats a failed read as a miss.
22
23use std::future::Future;
24use std::pin::Pin;
25use std::sync::Arc;
26use std::time::{Duration, SystemTime};
27
28use bytes::Bytes;
29
30use camel_api::body::Body;
31use camel_api::cache::{CacheEntry, CacheRepository, ContentType};
32use camel_api::{CamelError, Exchange, OutcomePipeline, OutcomeSegment, PipelineOutcome};
33use camel_component_api::RuntimeObservability;
34
35use crate::MessageIdExpression;
36
37// ── Singleflight miss coalescing (cache-admin task 2.4) ──
38
39/// Terminal state a coalescing leader publishes for its waiters.
40///
41/// Mirrors the leader's own `PipelineOutcome` in a clonable shape so
42/// every waiter of the wave receives it (waiters clone-read; nobody
43/// consumes the slot).
44#[derive(Clone)]
45pub(crate) enum CoalesceTerminal {
46    /// Leader completed: waiters adopt the leader's resulting body.
47    Completed(Body),
48    /// Leader failed: waiters fail with the same error (anti-burst).
49    Failed(CamelError),
50    /// Leader stopped: waiters stop their own exchanges (branch-filter).
51    Stopped,
52}
53
54/// One in-flight coalescing wave for a resolved cache key.
55///
56/// `terminal` is a write-once slot filled BEFORE `notify_waiters()` is
57/// called, so a woken waiter always re-reads a filled slot (no lost
58/// wakeup: `notify_waiters` alone wakes only currently-registered
59/// waiters).
60struct InFlight {
61    terminal: std::sync::Mutex<Option<CoalesceTerminal>>,
62    notify: tokio::sync::Notify,
63}
64
65impl Default for InFlight {
66    fn default() -> Self {
67        Self {
68            terminal: std::sync::Mutex::new(None),
69            notify: tokio::sync::Notify::new(),
70        }
71    }
72}
73
74impl InFlight {
75    /// Publish the terminal state. Write-once: an already-filled slot is
76    /// never cleared or overwritten (a late `LeaderGuard::drop` after a
77    /// normal completion is a no-op here).
78    fn publish(&self, terminal: CoalesceTerminal) {
79        if let Ok(mut slot) = self.terminal.lock()
80            && slot.is_none()
81        {
82            *slot = Some(terminal);
83        }
84    }
85
86    /// Clone-read the terminal state, if published.
87    fn terminal_snapshot(&self) -> Option<CoalesceTerminal> {
88        match self.terminal.lock() {
89            Ok(slot) => (*slot).clone(),
90            Err(_) => None,
91        }
92    }
93}
94
95/// In-flight coalescing waves, keyed by resolved cache key, scoped per
96/// compiled route-step instance (shared across `CacheService` clones).
97type InFlightMap = std::sync::Mutex<std::collections::HashMap<String, std::sync::Arc<InFlight>>>;
98
99/// Cancellation guard for the coalescing leader.
100///
101/// If the leader future is dropped before it publishes its terminal
102/// state (route shutdown, task abort), `Drop` publishes a cancellation
103/// terminal (`Failed`) into the write-once slot, wakes waiters, and
104/// removes the map entry — so waiters are never stranded. On normal
105/// completion the slot is already filled (publish is skipped) and the
106/// entry is already retired; both `Drop` actions become no-ops.
107struct LeaderGuard {
108    key: String,
109    map: Arc<InFlightMap>,
110    cell: Arc<InFlight>,
111}
112
113impl LeaderGuard {
114    /// Remove the map entry iff it still identifies `cell`.
115    ///
116    /// The `Arc::ptr_eq` identity check keeps a late guard (or a
117    /// completed leader) from evicting a NEWER wave's entry for the
118    /// same key.
119    fn retire(map: &Arc<InFlightMap>, key: &str, cell: &Arc<InFlight>) {
120        if let Ok(mut map) = map.lock()
121            && map
122                .get(key)
123                .is_some_and(|current| Arc::ptr_eq(current, cell))
124        {
125            map.remove(key);
126        }
127    }
128}
129
130impl Drop for LeaderGuard {
131    fn drop(&mut self) {
132        // Write-once: no-op when the leader already published a terminal.
133        self.cell
134            .publish(CoalesceTerminal::Failed(CamelError::Config(
135                "cache coalesce leader cancelled".into(),
136            )));
137        self.cell.notify.notify_waiters();
138        Self::retire(&self.map, &self.key, &self.cell);
139    }
140}
141
142/// Outcome-aware Cache segment (Caching EIP).
143///
144/// Wraps a named [`CacheRepository`] and an on-miss sub-pipeline
145/// ([`OutcomeSegment`]). On each exchange:
146///
147/// 1. Evaluate `key_expr`. `None` → not cacheable; forward directly to the
148///    on-miss sub-pipeline (no lookup, no write-back).
149/// 2. `repository.get(&key)`:
150///    - `Err(e)` → `Failed(e)` (contract C1).
151///    - `Ok(Some(entry))` → HIT: reconstruct `Body` from the entry, set it on
152///      the exchange, return `Completed` (skip on-miss).
153///    - `Ok(None)` → MISS: proceed to step 3.
154/// 3. Run the on-miss sub-pipeline.
155///    - `Stopped(ex)` / `Failed(e)` → propagate as-is (NO write-back).
156///    - `Completed(ex)` → proceed to write-back.
157/// 4. Write-back the resulting body (when it fits `max_entry_bytes`):
158///    - materialized variants (`Bytes`/`Text`/`Json`/`Xml`) → serialize, store.
159///    - `Stream` → materialize via [`Body::into_bytes`] (consumes the body,
160///      replaces it with `Body::Bytes`); `StreamLimitExceeded` propagates.
161///    - `Empty` / oversized body → pass through uncached, return `Completed`.
162///
163/// With `coalesce_misses` enabled ([`CacheService::with_coalesce`]),
164/// concurrent misses on the same resolved key are coalesced
165/// (singleflight): the first exchange (leader) runs `on_miss` and the
166/// single write-back `set`; concurrent exchanges (waiters) await the
167/// leader's terminal state instead of running `on_miss`. HIT, key-`None`,
168/// and `coalesce_misses == false` paths bypass the in-flight map
169/// entirely.
170pub struct CacheService {
171    repository: Arc<dyn CacheRepository>,
172    /// Cached `repository.name()` for OTel span tagging (Task 3.3).
173    repository_name: String,
174    key_expr: MessageIdExpression,
175    ttl: Option<Duration>,
176    max_entry_bytes: usize,
177    on_miss: OutcomeSegment,
178    rt: Arc<dyn RuntimeObservability>,
179    /// Singleflight miss coalescing toggle (default `false`).
180    coalesce_misses: bool,
181    /// In-flight coalescing waves. `Clone` clones the `Arc`, so every
182    /// service clone of one compiled route-step shares the same map.
183    inflight: Arc<InFlightMap>,
184}
185
186impl CacheService {
187    /// Build a new cache segment.
188    ///
189    /// `repository_name` is derived from `repository.name()` so OTel tags stay
190    /// in sync with the resolved backend.
191    pub fn new(
192        repository: Arc<dyn CacheRepository>,
193        key_expr: MessageIdExpression,
194        ttl: Option<Duration>,
195        max_entry_bytes: usize,
196        on_miss: OutcomeSegment,
197        rt: Arc<dyn RuntimeObservability>,
198    ) -> Self {
199        let repository_name = repository.name().to_string();
200        Self {
201            repository,
202            repository_name,
203            key_expr,
204            ttl,
205            max_entry_bytes,
206            on_miss,
207            rt,
208            coalesce_misses: false,
209            inflight: Arc::new(InFlightMap::default()),
210        }
211    }
212
213    /// Enable (or explicitly disable) singleflight miss coalescing.
214    ///
215    /// With coalescing on, concurrent misses on the same resolved key
216    /// run the `on_miss` sub-pipeline exactly once per wave (leader
217    /// runs + writes back; waiters receive the leader's terminal state).
218    pub fn with_coalesce(mut self, coalesce_misses: bool) -> Self {
219        self.coalesce_misses = coalesce_misses;
220        self
221    }
222
223    /// The configured repository name (for OTel tagging).
224    pub fn repository_name(&self) -> &str {
225        &self.repository_name
226    }
227}
228
229/// Shared write-back tail for materialized bodies.
230///
231/// Checks `max_entry_bytes`, builds a [`CacheEntry`], stores via
232/// the repository, and returns `Completed(exchange)`. The exchange body
233/// is not modified — it passes through as-is. On oversized body, logs a
234/// debug! skip message and returns `Completed(exchange)` without storing.
235/// On repository error, returns `Failed(e)`.
236#[allow(clippy::too_many_arguments)]
237async fn write_back(
238    repository: &Arc<dyn CacheRepository>,
239    repository_name: &str,
240    max_entry_bytes: usize,
241    ttl: Option<Duration>,
242    exchange: Exchange,
243    key: &str,
244    serialized: Vec<u8>,
245    content_type: ContentType,
246) -> PipelineOutcome {
247    if serialized.len() <= max_entry_bytes {
248        let entry = CacheEntry {
249            bytes: serialized,
250            payload_path: None,
251            content_type,
252            expires_at: None,
253        };
254        match repository.set(key, entry, ttl).await {
255            Ok(()) => {}
256            Err(e) => {
257                if matches!(&e, CamelError::Config(msg) if msg.starts_with("cache: max_entries")) {
258                    tracing::debug!(
259                        repository = %repository_name,
260                        key = %key,
261                        "cache at capacity, skipping write-back"
262                    ); // log-policy: g:cache:capacity-full-skip
263                } else {
264                    return PipelineOutcome::Failed(e);
265                }
266            }
267        }
268    } else {
269        // log-policy: g:cache:oversized-skip
270        tracing::debug!(
271            repository = %repository_name,
272            key = %key,
273            len = serialized.len(),
274            max = max_entry_bytes,
275            "cache write-back skipped: body exceeds max_entry_bytes"
276        );
277    }
278    PipelineOutcome::Completed(exchange)
279}
280
281impl Clone for CacheService {
282    fn clone(&self) -> Self {
283        Self {
284            repository: Arc::clone(&self.repository),
285            repository_name: self.repository_name.clone(),
286            key_expr: Arc::clone(&self.key_expr),
287            ttl: self.ttl,
288            max_entry_bytes: self.max_entry_bytes,
289            on_miss: self.on_miss.clone(),
290            rt: Arc::clone(&self.rt),
291            coalesce_misses: self.coalesce_misses,
292            inflight: Arc::clone(&self.inflight),
293        }
294    }
295}
296
297impl OutcomePipeline for CacheService {
298    fn clone_box(&self) -> Box<dyn OutcomePipeline> {
299        Box::new(self.clone())
300    }
301
302    fn run<'a>(
303        &'a mut self,
304        exchange: Exchange,
305    ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
306        Box::pin(async move {
307            // 1. Evaluate key. None → not cacheable, bypass straight to on_miss.
308            let key = match (self.key_expr)(&exchange) {
309                Some(k) => k,
310                None => return self.on_miss.run(exchange).await,
311            };
312
313            // 2. Lookup (contract C1: propagate Err, never treat as miss).
314            match self.repository.get(&key).await {
315                Err(e) => return PipelineOutcome::Failed(e),
316                Ok(Some(entry)) => {
317                    // HIT: record metric, reconstruct body, skip on-miss sub-pipeline.
318                    // allow-open-label rc-ycts (repository: user-declared cache repository name)
319                    self.rt.metrics().record_counter(
320                        "camel.cache.hits",
321                        1.0_f64,
322                        &[("repository", &self.repository_name)],
323                    );
324                    match reconstruct_body(&entry) {
325                        Ok(body) => {
326                            let mut exchange = exchange;
327                            exchange.input.body = body;
328                            return PipelineOutcome::Completed(exchange);
329                        }
330                        Err(e) => return PipelineOutcome::Failed(e),
331                    }
332                }
333                Ok(None) => {
334                    // MISS: record metric, fall through to on-miss sub-pipeline.
335                    // allow-open-label rc-ycts (repository: user-declared cache repository name)
336                    self.rt.metrics().record_counter(
337                        "camel.cache.misses",
338                        1.0_f64,
339                        &[("repository", &self.repository_name)],
340                    );
341                }
342            }
343
344            // 3./4. MISS flow, singleflight-coalesced when enabled.
345            if self.coalesce_misses {
346                return self.coalesced_miss(exchange, key).await;
347            }
348            self.run_miss(exchange, key).await
349        })
350    }
351}
352
353impl CacheService {
354    /// The un-coalesced MISS flow (spec steps 3-4): run the on-miss
355    /// sub-pipeline, then write the resulting body back (subject to
356    /// `max_entry_bytes` and the materialization policy). Also the
357    /// leader's flow under coalescing.
358    async fn run_miss(&mut self, exchange: Exchange, key: String) -> PipelineOutcome {
359        // 3. Run the on-miss sub-pipeline.
360        let mut exchange = match self.on_miss.run(exchange).await {
361            PipelineOutcome::Stopped(ex) => return PipelineOutcome::Stopped(ex),
362            PipelineOutcome::Failed(e) => return PipelineOutcome::Failed(e),
363            PipelineOutcome::Completed(ex) => ex,
364        };
365
366        // 4. Write-back. Take the body out so the Stream arm can consume it.
367        let body = std::mem::replace(&mut exchange.input.body, Body::Empty);
368        match body {
369            Body::Bytes(b) => {
370                let serialized = b.to_vec();
371                exchange.input.body = Body::Bytes(b);
372                write_back(
373                    &self.repository,
374                    &self.repository_name,
375                    self.max_entry_bytes,
376                    self.ttl,
377                    exchange,
378                    &key,
379                    serialized,
380                    ContentType::Bytes,
381                )
382                .await
383            }
384            Body::Text(s) => {
385                let serialized = s.as_bytes().to_vec();
386                exchange.input.body = Body::Text(s);
387                write_back(
388                    &self.repository,
389                    &self.repository_name,
390                    self.max_entry_bytes,
391                    self.ttl,
392                    exchange,
393                    &key,
394                    serialized,
395                    ContentType::Text,
396                )
397                .await
398            }
399            Body::Json(v) => {
400                let serialized = match serde_json::to_vec(&v) {
401                    Ok(b) => b,
402                    Err(e) => {
403                        exchange.input.body = Body::Json(v);
404                        return PipelineOutcome::Failed(CamelError::TypeConversionFailed(
405                            e.to_string(),
406                        ));
407                    }
408                };
409                exchange.input.body = Body::Json(v);
410                write_back(
411                    &self.repository,
412                    &self.repository_name,
413                    self.max_entry_bytes,
414                    self.ttl,
415                    exchange,
416                    &key,
417                    serialized,
418                    ContentType::Json,
419                )
420                .await
421            }
422            Body::Xml(s) => {
423                let serialized = s.as_bytes().to_vec();
424                exchange.input.body = Body::Xml(s);
425                write_back(
426                    &self.repository,
427                    &self.repository_name,
428                    self.max_entry_bytes,
429                    self.ttl,
430                    exchange,
431                    &key,
432                    serialized,
433                    ContentType::Xml,
434                )
435                .await
436            }
437            Body::Stream(stream_body) => {
438                // Materialize (consumes the stream). StreamLimitExceeded propagates.
439                let materialized = match Body::Stream(stream_body)
440                    .into_bytes(self.max_entry_bytes)
441                    .await
442                {
443                    Ok(b) => b,
444                    Err(e) => return PipelineOutcome::Failed(e),
445                };
446                // into_bytes already enforced max_entry_bytes, so it fits by construction.
447                let entry = CacheEntry {
448                    bytes: materialized.to_vec(),
449                    payload_path: None,
450                    content_type: ContentType::Bytes,
451                    expires_at: None,
452                };
453                if let Err(e) = self.repository.set(&key, entry, self.ttl).await {
454                    // Degrade capacity-exceeded to uncached — same policy as write_back.
455                    if matches!(&e, CamelError::Config(msg) if msg.starts_with("cache: max_entries"))
456                    {
457                        tracing::debug!(
458                            repository = %self.repository_name,
459                            key = %key,
460                            "cache at capacity, skipping write-back for stream"
461                        ); // log-policy: g:cache:capacity-full-skip
462                        exchange.input.body = Body::Bytes(materialized);
463                        return PipelineOutcome::Completed(exchange);
464                    }
465                    exchange.input.body = Body::Bytes(materialized);
466                    return PipelineOutcome::Failed(e);
467                }
468                exchange.input.body = Body::Bytes(materialized);
469                PipelineOutcome::Completed(exchange)
470            }
471            _ => {
472                // Empty (or any future variant): pass through uncached.
473                exchange.input.body = body;
474                PipelineOutcome::Completed(exchange)
475            }
476        }
477    }
478
479    /// The coalesced MISS flow (singleflight, cache-admin task 2.4).
480    ///
481    /// The first exchange on a key (leader) inserts the in-flight cell
482    /// and runs [`CacheService::run_miss`] (on_miss + the single
483    /// write-back `set`) under a [`LeaderGuard`]; concurrent misses on
484    /// the same key (waiters) do NOT run on_miss — they await the
485    /// leader's terminal state and clone-read it.
486    ///
487    /// Protocol (cancellation-safe, race-free):
488    /// - Waiter registration is atomic with the map lookup: the
489    ///   pinned-`Notified` `enable()` happens while STILL holding the
490    ///   map lock, so the leader's publish+notify cannot slip between
491    ///   the lookup and the registration.
492    /// - The terminal slot is filled BEFORE `notify_waiters()`, and a
493    ///   woken waiter re-reads the slot (no lost wakeup —
494    ///   `notify_waiters` alone wakes only currently-registered
495    ///   waiters).
496    /// - Map removal happens only on `Arc::ptr_eq` identity with this
497    ///   leader's cell (a late guard cannot evict a newer wave's entry).
498    /// - The slot is write-once: once filled it is never cleared or
499    ///   overwritten.
500    async fn coalesced_miss(&mut self, exchange: Exchange, key: String) -> PipelineOutcome {
501        let inflight = Arc::clone(&self.inflight);
502
503        // Resolve the role under ONE short lock scope; the guard (and
504        // the lock Result) never crosses an await. A waiter registers
505        // its Notified (Box::pin + enable) while STILL holding the map
506        // lock — registration atomic with the lookup. The registration
507        // borrows the outer `wave` binding (which outlives the guard),
508        // so it can be awaited after the guard is gone.
509        let mut wave: Option<Arc<InFlight>> = None;
510        let mut registered = None;
511        match inflight.lock() {
512            Ok(mut map) => {
513                match map.get(&key).cloned() {
514                    Some(existing) => {
515                        // WAITER: claim the existing wave, then register
516                        // (pin + enable) while STILL holding the lock.
517                        wave = Some(existing);
518                        if let Some(cell) = wave.as_ref() {
519                            let mut notified = Box::pin(cell.notify.notified());
520                            notified.as_mut().enable();
521                            registered = Some(notified);
522                        }
523                    }
524                    None => {
525                        // LEADER: claim the key.
526                        let cell = Arc::new(InFlight::default());
527                        map.insert(key.clone(), Arc::clone(&cell));
528                        wave = Some(cell);
529                    }
530                }
531            }
532            Err(poisoned) => drop(poisoned),
533        }
534        let Some(cell_ref) = wave.as_ref() else {
535            // Unreachable on the Ok path (both arms set `wave`); a
536            // poisoned in-flight map degrades to un-coalesced
537            // execution rather than stranding exchanges.
538            return self.run_miss(exchange, key).await;
539        };
540
541        if let Some(notified) = registered {
542            // WAITER. The slot may already be filled (leader finished
543            // between registration and this read): clone-read without
544            // parking. Otherwise await the leader's notify and re-read.
545            let terminal = match cell_ref.terminal_snapshot() {
546                Some(t) => t,
547                None => {
548                    notified.await;
549                    cell_ref.terminal_snapshot().unwrap_or_else(|| {
550                        CoalesceTerminal::Failed(CamelError::Config(
551                            "cache coalesce waiter woke without a terminal state".into(),
552                        ))
553                    })
554                }
555            };
556            match terminal {
557                CoalesceTerminal::Completed(body) => {
558                    let mut exchange = exchange;
559                    exchange.input.body = body;
560                    PipelineOutcome::Completed(exchange)
561                }
562                CoalesceTerminal::Failed(e) => PipelineOutcome::Failed(e),
563                CoalesceTerminal::Stopped => PipelineOutcome::Stopped(exchange),
564            }
565        } else {
566            // LEADER. Run the miss flow under a cancellation guard,
567            // publish the terminal state, wake waiters, retire the
568            // map entry.
569            let cell = Arc::clone(cell_ref);
570            let _guard = LeaderGuard {
571                key: key.clone(),
572                map: Arc::clone(&inflight),
573                cell: Arc::clone(&cell),
574            };
575            let outcome = self.run_miss(exchange, key.clone()).await;
576            let terminal = match &outcome {
577                PipelineOutcome::Completed(ex) => {
578                    CoalesceTerminal::Completed(ex.input.body.clone())
579                }
580                PipelineOutcome::Failed(e) => CoalesceTerminal::Failed(e.clone()),
581                PipelineOutcome::Stopped(_) => CoalesceTerminal::Stopped,
582            };
583            // Slot BEFORE notify: woken waiters re-read a filled slot.
584            cell.publish(terminal);
585            cell.notify.notify_waiters();
586            LeaderGuard::retire(&inflight, &key, &cell);
587            // Guard drop is a no-op now: the write-once slot is
588            // filled and the entry is retired.
589            outcome
590        }
591    }
592}
593
594/// Reconstruct a [`Body`] from a stored [`CacheEntry`].
595///
596/// Maps each [`ContentType`] back to the matching `Body` variant, decoding
597/// UTF-8 / JSON failures into `CamelError::TypeConversionFailed`.
598fn reconstruct_body(entry: &CacheEntry) -> Result<Body, CamelError> {
599    match entry.content_type {
600        ContentType::Bytes => Ok(Body::Bytes(Bytes::from(entry.bytes.clone()))),
601        ContentType::Text => {
602            let s = String::from_utf8(entry.bytes.clone()).map_err(|e| {
603                CamelError::TypeConversionFailed(format!("cached text is not valid UTF-8: {e}"))
604            })?;
605            Ok(Body::Text(s))
606        }
607        ContentType::Json => {
608            let v = serde_json::from_slice(&entry.bytes).map_err(|e| {
609                CamelError::TypeConversionFailed(format!("cached bytes are not valid JSON: {e}"))
610            })?;
611            Ok(Body::Json(v))
612        }
613        ContentType::Xml => {
614            let s = String::from_utf8(entry.bytes.clone()).map_err(|e| {
615                CamelError::TypeConversionFailed(format!("cached xml is not valid UTF-8: {e}"))
616            })?;
617            Ok(Body::Xml(s))
618        }
619    }
620}
621
622// ===========================================================================
623// CacheInvalidateService — invalidate a single cache entry or a namespace
624// ===========================================================================
625
626/// Exchange property set to the number of entries removed by a successful
627/// `cache_invalidate` step — always `1` for exact-key (removal is not
628/// observable by the backend), the returned count for a namespace purge. Not
629/// set when the key/prefix expression resolves to `None` or the backend
630/// reports an error.
631pub const CAMEL_CACHE_INVALIDATED_COUNT: &str = "CamelCacheInvalidatedCount";
632
633/// The invalidation target of a [`CacheInvalidateService`]: an exact key or a
634/// namespace prefix.
635#[derive(Clone)]
636pub enum CacheInvalidateTarget {
637    /// Invalidate the single entry under the resolved key.
638    Key(MessageIdExpression),
639    /// Invalidate every entry whose key starts with the resolved prefix.
640    Prefix(MessageIdExpression),
641}
642
643/// Outcome-aware segment that invalidates a single cache entry or a namespace.
644///
645/// Evaluates the configured [`CacheInvalidateTarget`]:
646/// - [`Key`](CacheInvalidateTarget::Key): expression `None` → `Completed`
647///   (nothing to invalidate); `Some(key)` → `repository.invalidate(&key).await`.
648///   - `Err(e)` → `Failed(e)`.
649///   - `Ok(())` → sets `CAMEL_CACHE_INVALIDATED_COUNT = 1`, emits
650///     `camel.cache.invalidations` +1, `Completed(exchange)`.
651/// - [`Prefix`](CacheInvalidateTarget::Prefix): expression `None` → `Completed`;
652///   `Some(prefix)` → `repository.invalidate_prefix(&prefix).await`.
653///   - `Err(e)` → `Failed(e)` (an unsupported backend surfaces as failure —
654///     fail-closed).
655///   - `Ok(count)` → sets `CAMEL_CACHE_INVALIDATED_COUNT = count`, emits
656///     `camel.cache.invalidations` +1, `Completed(exchange)`.
657pub struct CacheInvalidateService {
658    repository: Arc<dyn CacheRepository>,
659    target: CacheInvalidateTarget,
660    rt: Arc<dyn RuntimeObservability>,
661    repository_name: String,
662}
663
664impl CacheInvalidateService {
665    pub fn new(
666        repository: Arc<dyn CacheRepository>,
667        target: CacheInvalidateTarget,
668        rt: Arc<dyn RuntimeObservability>,
669    ) -> Self {
670        let repository_name = repository.name().to_string();
671        Self {
672            repository,
673            target,
674            rt,
675            repository_name,
676        }
677    }
678}
679
680impl Clone for CacheInvalidateService {
681    fn clone(&self) -> Self {
682        Self {
683            repository: Arc::clone(&self.repository),
684            target: self.target.clone(),
685            rt: Arc::clone(&self.rt),
686            repository_name: self.repository_name.clone(),
687        }
688    }
689}
690
691impl OutcomePipeline for CacheInvalidateService {
692    fn clone_box(&self) -> Box<dyn OutcomePipeline> {
693        Box::new(self.clone())
694    }
695
696    fn run<'a>(
697        &'a mut self,
698        exchange: Exchange,
699    ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
700        Box::pin(async move {
701            match self.target.clone() {
702                CacheInvalidateTarget::Key(key_expr) => {
703                    let key = match key_expr(&exchange) {
704                        Some(k) => k,
705                        None => return PipelineOutcome::Completed(exchange),
706                    };
707                    match self.repository.invalidate(&key).await {
708                        Err(e) => PipelineOutcome::Failed(e),
709                        Ok(()) => {
710                            // allow-open-label rc-ycts (repository: user-declared cache repository name)
711                            self.rt.metrics().record_counter(
712                                "camel.cache.invalidations",
713                                1.0_f64,
714                                &[("repository", &self.repository_name)],
715                            );
716                            let mut exchange = exchange;
717                            exchange.set_property(
718                                CAMEL_CACHE_INVALIDATED_COUNT,
719                                serde_json::Value::from(1u64),
720                            );
721                            PipelineOutcome::Completed(exchange)
722                        }
723                    }
724                }
725                CacheInvalidateTarget::Prefix(prefix_expr) => {
726                    let prefix = match prefix_expr(&exchange) {
727                        Some(p) => p,
728                        None => return PipelineOutcome::Completed(exchange),
729                    };
730                    match self.repository.invalidate_prefix(&prefix).await {
731                        Err(e) => PipelineOutcome::Failed(e),
732                        Ok(count) => {
733                            // allow-open-label rc-ycts (repository: user-declared cache repository name)
734                            self.rt.metrics().record_counter(
735                                "camel.cache.invalidations",
736                                1.0_f64,
737                                &[("repository", &self.repository_name)],
738                            );
739                            let mut exchange = exchange;
740                            exchange.set_property(
741                                CAMEL_CACHE_INVALIDATED_COUNT,
742                                serde_json::Value::from(count),
743                            );
744                            PipelineOutcome::Completed(exchange)
745                        }
746                    }
747                }
748            }
749        })
750    }
751}
752
753// ===========================================================================
754// CachePeekStaleService — serve a stale entry after expiry
755// ===========================================================================
756
757/// Exchange property set to `true` when a `cache_peek_stale` HIT occurred.
758pub const CAMEL_CACHE_PEEK_HIT: &str = "CamelCachePeekHit";
759/// Exchange property set to `true` when the served entry was stale (post-expiry).
760pub const CAMEL_CACHE_PEEK_STALE: &str = "CamelCachePeekStale";
761
762/// On-miss policy for [`CachePeekStaleService`].
763///
764/// - [`Stop`](PeekStaleMissPolicy::Stop) (default) preserves the
765///   `CircuitBreaker.fallback` absence-Stops contract.
766/// - [`Continue`](PeekStaleMissPolicy::Continue) leaves the body untouched on
767///   MISS so `choice` can branch on [`CAMEL_CACHE_PEEK_HIT`].
768#[derive(Debug, Clone, Copy, PartialEq, Eq)]
769pub enum PeekStaleMissPolicy {
770    /// MISS Stops the branch (no stale available — `CircuitBreaker.fallback`).
771    Stop,
772    /// MISS continues with the body unchanged.
773    Continue,
774}
775
776impl PeekStaleMissPolicy {
777    /// Parses the canonical/DSL `cache_peek_stale.on_miss` knob:
778    /// absent or `"stop"` → [`Stop`](Self::Stop), `"continue"` →
779    /// [`Continue`](Self::Continue). Any other value fails closed naming
780    /// the step.
781    pub fn parse_on_miss(raw: Option<&str>) -> Result<Self, CamelError> {
782        match raw {
783            None | Some("stop") => Ok(Self::Stop),
784            Some("continue") => Ok(Self::Continue),
785            Some(other) => Err(CamelError::Config(format!(
786                "cache_peek_stale: invalid on_miss '{other}'; must be \"stop\" or \"continue\""
787            ))),
788        }
789    }
790}
791
792/// Outcome-aware segment that serves a stale (post-expiry) cache entry.
793///
794/// Evaluates `key_expr`:
795/// - `None` → `Stopped(exchange)` with a `debug` log (an anomalous key
796///   resolution is fail-closed, not a miss).
797/// - `Some(key)` → `repository.peek_stale(&key).await`.
798///   - `Err(e)` → `Failed(e)`.
799///   - `Ok(Some(entry))` → reconstruct body from entry, set
800///     `CamelCachePeekHit=true` and `CamelCachePeekStale` (true when the
801///     entry's `expires_at` has elapsed at evaluation time; false when absent
802///     or not elapsed), return `Completed(exchange)`.
803///   - `Ok(None)` → MISS (absence), governed by [`PeekStaleMissPolicy`]:
804///     - `Stop` (default): set `CamelCachePeekHit=false` and
805///       `CamelCachePeekStale=false`, log at `debug`, return `Stopped(exchange)`
806///       (absence in `CircuitBreaker.fallback` means "no stale available").
807///     - `Continue`: set `CamelCachePeekHit=false` and
808///       `CamelCachePeekStale=false`, leave the body unchanged, return
809///       `Completed(exchange)` so `choice` can branch on `CamelCachePeekHit`.
810pub struct CachePeekStaleService {
811    repository: Arc<dyn CacheRepository>,
812    key_expr: MessageIdExpression,
813    miss_policy: PeekStaleMissPolicy,
814    rt: Arc<dyn RuntimeObservability>,
815    repository_name: String,
816}
817
818impl CachePeekStaleService {
819    pub fn new(
820        repository: Arc<dyn CacheRepository>,
821        key_expr: MessageIdExpression,
822        miss_policy: PeekStaleMissPolicy,
823        rt: Arc<dyn RuntimeObservability>,
824    ) -> Self {
825        let repository_name = repository.name().to_string();
826        Self {
827            repository,
828            key_expr,
829            miss_policy,
830            rt,
831            repository_name,
832        }
833    }
834}
835
836impl Clone for CachePeekStaleService {
837    fn clone(&self) -> Self {
838        Self {
839            repository: Arc::clone(&self.repository),
840            key_expr: Arc::clone(&self.key_expr),
841            miss_policy: self.miss_policy,
842            rt: Arc::clone(&self.rt),
843            repository_name: self.repository_name.clone(),
844        }
845    }
846}
847
848/// Write the peek-result exchange properties under [`CAMEL_CACHE_PEEK_HIT`] and
849/// [`CAMEL_CACHE_PEEK_STALE`] as `serde_json::Value::Bool` values.
850fn set_peek_properties(exchange: &mut Exchange, hit: bool, stale: bool) {
851    exchange.set_property(CAMEL_CACHE_PEEK_HIT, serde_json::Value::Bool(hit));
852    exchange.set_property(CAMEL_CACHE_PEEK_STALE, serde_json::Value::Bool(stale));
853}
854
855impl OutcomePipeline for CachePeekStaleService {
856    fn clone_box(&self) -> Box<dyn OutcomePipeline> {
857        Box::new(self.clone())
858    }
859
860    fn run<'a>(
861        &'a mut self,
862        exchange: Exchange,
863    ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
864        Box::pin(async move {
865            let key = match (self.key_expr)(&exchange) {
866                Some(k) => k,
867                None => {
868                    tracing::debug!(
869                        step = "cache_peek_stale",
870                        repository = %self.repository.name(),
871                        "key expression resolved to None; stopping branch"
872                    );
873                    return PipelineOutcome::Stopped(exchange);
874                }
875            };
876            match self.repository.peek_stale(&key).await {
877                Err(e) => PipelineOutcome::Failed(e),
878                Ok(Some(entry)) => {
879                    // Emit peek_stale_served (fresh or stale — both are serves).
880                    // allow-open-label rc-ycts (repository: user-declared cache repository name)
881                    self.rt.metrics().record_counter(
882                        "camel.cache.peek_stale_served",
883                        1.0_f64,
884                        &[("repository", &self.repository_name)],
885                    );
886                    // Staleness read before body reconstruction; reconstruct_body borrows the entry, so ordering is stylistic.
887                    let stale = entry
888                        .expires_at
889                        .map(|t| t <= SystemTime::now())
890                        .unwrap_or(false);
891                    match reconstruct_body(&entry) {
892                        Ok(body) => {
893                            let mut exchange = exchange;
894                            exchange.input.body = body;
895                            set_peek_properties(&mut exchange, true, stale);
896                            PipelineOutcome::Completed(exchange)
897                        }
898                        Err(e) => PipelineOutcome::Failed(e),
899                    }
900                }
901                Ok(None) => match self.miss_policy {
902                    PeekStaleMissPolicy::Stop => {
903                        let mut exchange = exchange;
904                        set_peek_properties(&mut exchange, false, false);
905                        tracing::debug!(
906                            step = "cache_peek_stale",
907                            repository = %self.repository.name(),
908                            "peek miss; stopping branch per on_miss=stop"
909                        );
910                        PipelineOutcome::Stopped(exchange)
911                    }
912                    PeekStaleMissPolicy::Continue => {
913                        let mut exchange = exchange;
914                        set_peek_properties(&mut exchange, false, false);
915                        PipelineOutcome::Completed(exchange)
916                    }
917                },
918            }
919        })
920    }
921}
922
923// ===========================================================================
924// CacheClearService — remove all entries from the repository
925// ===========================================================================
926
927/// Outcome-aware segment that clears the entire cache repository.
928///
929/// - `Err(e)` → `Failed(e)`.
930/// - `Ok(())` → `Completed(exchange)` with the body unchanged.
931pub struct CacheClearService {
932    repository: Arc<dyn CacheRepository>,
933}
934
935impl CacheClearService {
936    pub fn new(repository: Arc<dyn CacheRepository>) -> Self {
937        Self { repository }
938    }
939}
940
941impl Clone for CacheClearService {
942    fn clone(&self) -> Self {
943        Self {
944            repository: Arc::clone(&self.repository),
945        }
946    }
947}
948
949impl OutcomePipeline for CacheClearService {
950    fn clone_box(&self) -> Box<dyn OutcomePipeline> {
951        Box::new(self.clone())
952    }
953
954    fn run<'a>(
955        &'a mut self,
956        exchange: Exchange,
957    ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
958        Box::pin(async move {
959            match self.repository.clear().await {
960                Err(e) => PipelineOutcome::Failed(e),
961                Ok(()) => PipelineOutcome::Completed(exchange),
962            }
963        })
964    }
965}
966
967// ===========================================================================
968// CacheStatsService — emit the repository stats as a JSON body
969// ===========================================================================
970
971/// Outcome-aware segment that replaces the exchange body with a JSON snapshot
972/// of the repository's [`CacheStats`].
973pub struct CacheStatsService {
974    repository: Arc<dyn CacheRepository>,
975    repository_name: String,
976}
977
978impl CacheStatsService {
979    pub fn new(repository: Arc<dyn CacheRepository>) -> Self {
980        let repository_name = repository.name().to_string();
981        Self {
982            repository,
983            repository_name,
984        }
985    }
986}
987
988impl Clone for CacheStatsService {
989    fn clone(&self) -> Self {
990        Self {
991            repository: Arc::clone(&self.repository),
992            repository_name: self.repository_name.clone(),
993        }
994    }
995}
996
997impl OutcomePipeline for CacheStatsService {
998    fn clone_box(&self) -> Box<dyn OutcomePipeline> {
999        Box::new(self.clone())
1000    }
1001
1002    fn run<'a>(
1003        &'a mut self,
1004        mut exchange: Exchange,
1005    ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
1006        Box::pin(async move {
1007            let s = self.repository.stats().await;
1008            exchange.input.body = Body::Json(serde_json::json!({
1009                "repository": self.repository_name,
1010                "hits": s.hits,
1011                "misses": s.misses,
1012                "evictions": s.evictions,
1013                "entries": s.entries,
1014                "peek_stale_served": s.peek_stale_served,
1015                "invalidations": s.invalidations,
1016                "bytes": s.bytes,
1017            }));
1018            PipelineOutcome::Completed(exchange)
1019        })
1020    }
1021}
1022
1023// ===========================================================================
1024// Test utilities
1025// ===========================================================================
1026
1027#[cfg(test)]
1028mod test_utils {
1029    use super::*;
1030    use async_trait::async_trait;
1031    use camel_api::cache::CacheStats;
1032    use std::collections::HashMap;
1033    use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
1034    use tokio::sync::Mutex;
1035
1036    /// In-memory mock [`CacheRepository`] for cache segment tests. Allows tests
1037    /// to pre-seed entries, force `get`/`set` failures, and inspect the last
1038    /// `set` call (entry + TTL).
1039    #[derive(Debug, Default)]
1040    pub struct MockCacheRepository {
1041        name: String,
1042        entries: Arc<Mutex<HashMap<String, CacheEntry>>>,
1043        get_should_fail: Arc<AtomicBool>,
1044        set_should_fail: Arc<AtomicBool>,
1045        set_call_count: Arc<AtomicU32>,
1046        last_set_ttl: Arc<Mutex<Option<Duration>>>,
1047        invalidate_call_count: Arc<AtomicU32>,
1048        last_invalidate_key: Arc<Mutex<Option<String>>>,
1049        clear_call_count: Arc<AtomicU64>,
1050        clear_should_fail: Arc<AtomicBool>,
1051        invalidate_should_fail: Arc<AtomicBool>,
1052        prefix_unsupported: Arc<AtomicBool>,
1053        stats_override: std::sync::Mutex<CacheStats>,
1054    }
1055
1056    impl MockCacheRepository {
1057        pub fn new(name: &str) -> Self {
1058            Self {
1059                name: name.to_string(),
1060                ..Default::default()
1061            }
1062        }
1063
1064        pub fn invalidate_call_count(&self) -> u32 {
1065            self.invalidate_call_count.load(Ordering::SeqCst)
1066        }
1067
1068        pub async fn last_invalidate_key(&self) -> Option<String> {
1069            self.last_invalidate_key.lock().await.clone()
1070        }
1071
1072        pub fn clear_call_count(&self) -> u64 {
1073            self.clear_call_count.load(Ordering::SeqCst)
1074        }
1075
1076        pub fn set_should_fail_clear(&self, v: bool) {
1077            self.clear_should_fail.store(v, Ordering::SeqCst);
1078        }
1079
1080        pub fn set_should_fail_invalidate(&self, v: bool) {
1081            self.invalidate_should_fail.store(v, Ordering::SeqCst);
1082        }
1083
1084        /// Force `invalidate_prefix` to report the backend-naming unsupported
1085        /// error (mirrors the default `CacheRepository::invalidate_prefix`).
1086        pub fn set_prefix_unsupported(&self, v: bool) {
1087            self.prefix_unsupported.store(v, Ordering::SeqCst);
1088        }
1089
1090        pub fn set_stats(&self, stats: CacheStats) {
1091            *self.stats_override.lock().unwrap() = stats; // allow-unwrap: test-only
1092        }
1093
1094        /// Pre-seed a key so `get` returns a HIT.
1095        pub async fn seed(&self, key: &str, entry: CacheEntry) {
1096            self.entries.lock().await.insert(key.to_string(), entry);
1097        }
1098
1099        pub fn set_get_should_fail(&self, v: bool) {
1100            self.get_should_fail.store(v, Ordering::SeqCst);
1101        }
1102
1103        pub fn set_set_should_fail(&self, v: bool) {
1104            self.set_should_fail.store(v, Ordering::SeqCst);
1105        }
1106
1107        pub fn set_call_count(&self) -> u32 {
1108            self.set_call_count.load(Ordering::SeqCst)
1109        }
1110
1111        /// The TTL passed to the most recent `set` call.
1112        pub async fn last_set_ttl(&self) -> Option<Duration> {
1113            *self.last_set_ttl.lock().await
1114        }
1115
1116        /// Inspect the entry currently stored for `key` (if any).
1117        pub async fn stored_entry(&self, key: &str) -> Option<CacheEntry> {
1118            self.entries.lock().await.get(key).cloned()
1119        }
1120    }
1121
1122    #[async_trait]
1123    impl CacheRepository for MockCacheRepository {
1124        fn name(&self) -> &str {
1125            &self.name
1126        }
1127
1128        async fn get(&self, key: &str) -> Result<Option<CacheEntry>, CamelError> {
1129            if self.get_should_fail.load(Ordering::SeqCst) {
1130                return Err(CamelError::ProcessorError("synthetic get failure".into()));
1131            }
1132            // Read first, then yield: concurrent same-key lookups that
1133            // reach `get` before a write-back all observe the miss
1134            // (without the yield the uncontended tokio Mutex fast path
1135            // completes without rescheduling, serializing "concurrent"
1136            // exchanges and hiding the per-exchange behavior under test).
1137            let found = self.entries.lock().await.get(key).cloned();
1138            tokio::task::yield_now().await;
1139            Ok(found)
1140        }
1141
1142        async fn set(
1143            &self,
1144            key: &str,
1145            value: CacheEntry,
1146            ttl: Option<Duration>,
1147        ) -> Result<(), CamelError> {
1148            self.set_call_count.fetch_add(1, Ordering::SeqCst);
1149            *self.last_set_ttl.lock().await = ttl;
1150            if self.set_should_fail.load(Ordering::SeqCst) {
1151                return Err(CamelError::ProcessorError("synthetic set failure".into()));
1152            }
1153            self.entries.lock().await.insert(key.to_string(), value);
1154            Ok(())
1155        }
1156
1157        async fn peek_stale(&self, key: &str) -> Result<Option<CacheEntry>, CamelError> {
1158            self.get(key).await
1159        }
1160
1161        async fn invalidate(&self, key: &str) -> Result<(), CamelError> {
1162            self.invalidate_call_count.fetch_add(1, Ordering::SeqCst);
1163            *self.last_invalidate_key.lock().await = Some(key.to_string());
1164            if self.invalidate_should_fail.load(Ordering::SeqCst) {
1165                return Err(CamelError::ProcessorError(
1166                    "synthetic invalidate failure".into(),
1167                ));
1168            }
1169            self.entries.lock().await.remove(key);
1170            Ok(())
1171        }
1172
1173        async fn invalidate_prefix(&self, prefix: &str) -> Result<u64, CamelError> {
1174            if self.prefix_unsupported.load(Ordering::SeqCst) {
1175                return Err(CamelError::Config(format!(
1176                    "cache backend '{}' does not support invalidate_prefix (no key iteration)",
1177                    self.name()
1178                )));
1179            }
1180            let mut entries = self.entries.lock().await;
1181            let keys: Vec<String> = entries
1182                .keys()
1183                .filter(|k| k.starts_with(prefix))
1184                .cloned()
1185                .collect();
1186            let count = keys.len() as u64;
1187            for k in keys {
1188                entries.remove(&k);
1189            }
1190            Ok(count)
1191        }
1192
1193        async fn clear(&self) -> Result<(), CamelError> {
1194            self.clear_call_count.fetch_add(1, Ordering::SeqCst);
1195            if self.clear_should_fail.load(Ordering::SeqCst) {
1196                return Err(CamelError::ProcessorError("synthetic clear failure".into()));
1197            }
1198            self.entries.lock().await.clear();
1199            Ok(())
1200        }
1201
1202        async fn stats(&self) -> CacheStats {
1203            self.stats_override.lock().unwrap().clone() // allow-unwrap: test-only
1204        }
1205    }
1206}
1207
1208// ===========================================================================
1209// Tests
1210// ===========================================================================
1211
1212#[cfg(test)]
1213mod tests {
1214    use super::test_utils::MockCacheRepository;
1215    use super::*;
1216    use camel_api::body::{StreamBody, StreamMetadata};
1217    use camel_api::cache::CacheStats;
1218    use camel_api::metrics::NoOpMetrics;
1219    use camel_api::{Message, Value};
1220    use camel_component_api::health_registry::NoOpHealthCheckRegistry;
1221    use futures::stream;
1222    use std::sync::Mutex;
1223    use std::sync::atomic::{AtomicBool, Ordering};
1224    use std::time::SystemTime;
1225
1226    #[test]
1227    fn parse_on_miss_maps_absent_stop_and_continue() {
1228        assert_eq!(
1229            PeekStaleMissPolicy::parse_on_miss(None).unwrap(),
1230            PeekStaleMissPolicy::Stop
1231        );
1232        assert_eq!(
1233            PeekStaleMissPolicy::parse_on_miss(Some("stop")).unwrap(),
1234            PeekStaleMissPolicy::Stop
1235        );
1236        assert_eq!(
1237            PeekStaleMissPolicy::parse_on_miss(Some("continue")).unwrap(),
1238            PeekStaleMissPolicy::Continue
1239        );
1240    }
1241
1242    #[test]
1243    fn parse_on_miss_rejects_unknown_value_naming_the_step() {
1244        let err = PeekStaleMissPolicy::parse_on_miss(Some("explode")).unwrap_err();
1245        let msg = format!("{err}");
1246        assert!(msg.contains("cache_peek_stale"), "got: {msg}");
1247        assert!(msg.contains("explode"), "got: {msg}");
1248    }
1249
1250    /// Minimal no-op RuntimeObservability for tests that don't need OTel.
1251    #[derive(Clone)]
1252    struct NoopRt;
1253
1254    impl camel_component_api::HealthCheckRegistry for NoopRt {
1255        fn force_unhealthy_for_route(&self, _: &str, _: &str, _: &str) {}
1256    }
1257
1258    impl RuntimeObservability for NoopRt {
1259        fn metrics(&self) -> Arc<dyn camel_api::metrics::MetricsCollector> {
1260            Arc::new(NoOpMetrics)
1261        }
1262        fn health(&self) -> Arc<dyn camel_component_api::health_registry::HealthCheckRegistry> {
1263            Arc::new(NoOpHealthCheckRegistry)
1264        }
1265    }
1266
1267    fn noop_rt() -> Arc<dyn RuntimeObservability> {
1268        Arc::new(NoopRt)
1269    }
1270
1271    // ── Scripted on-miss sub-pipeline ──
1272
1273    #[derive(Clone)]
1274    enum ScriptedOutcome {
1275        Complete,
1276        Stop,
1277        Fail(CamelError),
1278    }
1279
1280    /// Test sub-pipeline: optionally replaces the body, then returns a
1281    /// scripted outcome. Records whether it was invoked.
1282    struct ScriptedOnMiss {
1283        body: Option<Body>,
1284        outcome: ScriptedOutcome,
1285        invoked: Arc<AtomicBool>,
1286    }
1287
1288    impl OutcomePipeline for ScriptedOnMiss {
1289        fn clone_box(&self) -> Box<dyn OutcomePipeline> {
1290            // clone_box is required by the trait but unused by these tests.
1291            unreachable!("clone_box not used in cache_eip tests")
1292        }
1293
1294        fn run<'a>(
1295            &'a mut self,
1296            mut exchange: Exchange,
1297        ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
1298            self.invoked.store(true, Ordering::SeqCst);
1299            let body = self.body.take();
1300            let outcome = self.outcome.clone();
1301            Box::pin(async move {
1302                if let Some(b) = body {
1303                    exchange.input.body = b;
1304                }
1305                match outcome {
1306                    ScriptedOutcome::Complete => PipelineOutcome::Completed(exchange),
1307                    ScriptedOutcome::Stop => PipelineOutcome::Stopped(exchange),
1308                    ScriptedOutcome::Fail(e) => PipelineOutcome::Failed(e),
1309                }
1310            })
1311        }
1312    }
1313
1314    // ── Builders ──
1315
1316    fn fixed_key() -> MessageIdExpression {
1317        Arc::new(|_| Some("cache-key".to_string()))
1318    }
1319
1320    fn none_key() -> MessageIdExpression {
1321        Arc::new(|_| None)
1322    }
1323
1324    fn prefix_key() -> MessageIdExpression {
1325        Arc::new(|_| Some("ns:".to_string()))
1326    }
1327
1328    /// Build a CacheService whose on-miss sets `body` and returns `outcome`.
1329    fn build_service(
1330        repo: Arc<MockCacheRepository>,
1331        key_expr: MessageIdExpression,
1332        max_entry_bytes: usize,
1333        body: Option<Body>,
1334        outcome: ScriptedOutcome,
1335        ttl: Option<Duration>,
1336        rt: Arc<dyn RuntimeObservability>,
1337    ) -> (CacheService, Arc<AtomicBool>) {
1338        let invoked = Arc::new(AtomicBool::new(false));
1339        let on_miss = OutcomeSegment::new(Box::new(ScriptedOnMiss {
1340            body,
1341            outcome,
1342            invoked: invoked.clone(),
1343        }));
1344        let svc = CacheService::new(repo, key_expr, ttl, max_entry_bytes, on_miss, rt);
1345        (svc, invoked)
1346    }
1347
1348    fn exchange() -> Exchange {
1349        let mut ex = Exchange::new(Message::new(""));
1350        ex.input.set_header("ignored", Value::String("v".into()));
1351        ex
1352    }
1353
1354    fn stream_body(data: &'static [u8]) -> Body {
1355        let chunks: Vec<Result<Bytes, CamelError>> = vec![Ok(Bytes::from_static(data))];
1356        let s = stream::iter(chunks);
1357        Body::Stream(StreamBody {
1358            stream: Arc::new(tokio::sync::Mutex::new(Some(Box::pin(s)))),
1359            metadata: StreamMetadata::default(),
1360        })
1361    }
1362
1363    fn stub_error(msg: &str) -> CamelError {
1364        CamelError::ProcessorError(msg.into())
1365    }
1366
1367    // ── Test 1: cache HIT short-circuits, on_miss NOT executed ──
1368
1369    #[tokio::test]
1370    async fn cache_hit_short_circuits_on_miss() {
1371        let repo = Arc::new(MockCacheRepository::new("mock"));
1372        repo.seed(
1373            "cache-key",
1374            CacheEntry {
1375                bytes: b"cached-payload".to_vec(),
1376                payload_path: None,
1377                content_type: ContentType::Bytes,
1378                expires_at: None,
1379            },
1380        )
1381        .await;
1382        let (mut svc, on_miss_invoked) = build_service(
1383            repo,
1384            fixed_key(),
1385            1024,
1386            Some(Body::Bytes(Bytes::from_static(b"unreached"))),
1387            ScriptedOutcome::Complete,
1388            None,
1389            noop_rt(),
1390        );
1391
1392        let outcome = svc.run(exchange()).await;
1393
1394        let ex = match outcome {
1395            PipelineOutcome::Completed(ex) => ex,
1396            other => panic!("expected Completed, got {other:?}"),
1397        };
1398        assert_eq!(
1399            ex.input.body,
1400            Body::Bytes(Bytes::from_static(b"cached-payload"))
1401        );
1402        assert!(
1403            !on_miss_invoked.load(Ordering::SeqCst),
1404            "on_miss must NOT run on a cache HIT"
1405        );
1406    }
1407
1408    // ── Test 2: cache MISS runs on_miss, writes back, continues ──
1409
1410    #[tokio::test]
1411    async fn cache_miss_runs_on_miss_sets_continues() {
1412        let ttl = Duration::from_secs(30);
1413        let repo = Arc::new(MockCacheRepository::new("mock"));
1414        let (mut svc, on_miss_invoked) = build_service(
1415            repo.clone(),
1416            fixed_key(),
1417            1024,
1418            Some(Body::Bytes(Bytes::from_static(b"x"))),
1419            ScriptedOutcome::Complete,
1420            Some(ttl),
1421            noop_rt(),
1422        );
1423
1424        let outcome = svc.run(exchange()).await;
1425
1426        let ex = match outcome {
1427            PipelineOutcome::Completed(ex) => ex,
1428            other => panic!("expected Completed, got {other:?}"),
1429        };
1430        assert!(on_miss_invoked.load(Ordering::SeqCst));
1431        assert_eq!(ex.input.body, Body::Bytes(Bytes::from_static(b"x")));
1432        assert_eq!(repo.set_call_count(), 1, "set must be called once on miss");
1433        let stored = repo
1434            .stored_entry("cache-key")
1435            .await
1436            .expect("entry must be stored");
1437        assert_eq!(stored.bytes, b"x");
1438        assert_eq!(stored.content_type, ContentType::Bytes);
1439        assert_eq!(repo.last_set_ttl().await, Some(ttl));
1440    }
1441
1442    // ── Test 3: oversized materialized body skips write-back ──
1443
1444    #[tokio::test]
1445    async fn cache_miss_oversized_materialized_body_skips_writeback() {
1446        let repo = Arc::new(MockCacheRepository::new("mock"));
1447        // max_entry_bytes = 4; on_miss produces 9 bytes.
1448        let (mut svc, _invoked) = build_service(
1449            repo.clone(),
1450            fixed_key(),
1451            4,
1452            Some(Body::Bytes(Bytes::from_static(b"oversized"))),
1453            ScriptedOutcome::Complete,
1454            None,
1455            noop_rt(),
1456        );
1457
1458        let outcome = svc.run(exchange()).await;
1459
1460        let ex = match outcome {
1461            PipelineOutcome::Completed(ex) => ex,
1462            other => panic!("expected Completed, got {other:?}"),
1463        };
1464        // Body passes through unchanged.
1465        assert_eq!(ex.input.body, Body::Bytes(Bytes::from_static(b"oversized")));
1466        assert_eq!(
1467            repo.set_call_count(),
1468            0,
1469            "set must NOT be called for oversized body"
1470        );
1471        assert!(repo.stored_entry("cache-key").await.is_none());
1472    }
1473
1474    // ── Test 4: oversized Stream propagates StreamLimitExceeded ──
1475
1476    #[tokio::test]
1477    async fn cache_miss_oversized_stream_propagates_err() {
1478        let repo = Arc::new(MockCacheRepository::new("mock"));
1479        let (mut svc, _invoked) = build_service(
1480            repo.clone(),
1481            fixed_key(),
1482            4,
1483            Some(stream_body(b"way-too-big-stream")),
1484            ScriptedOutcome::Complete,
1485            None,
1486            noop_rt(),
1487        );
1488
1489        let outcome = svc.run(exchange()).await;
1490
1491        match outcome {
1492            PipelineOutcome::Failed(CamelError::StreamLimitExceeded(n)) => {
1493                assert_eq!(n, 4);
1494            }
1495            other => panic!("expected Failed(StreamLimitExceeded(4)), got {other:?}"),
1496        }
1497        assert_eq!(
1498            repo.set_call_count(),
1499            0,
1500            "set must NOT be called when stream exceeds limit"
1501        );
1502    }
1503
1504    // ── Test 5: on_miss Stopped propagates without write-back ──
1505
1506    #[tokio::test]
1507    async fn cache_on_miss_stopped_propagates_without_writeback() {
1508        let repo = Arc::new(MockCacheRepository::new("mock"));
1509        let (mut svc, _invoked) = build_service(
1510            repo.clone(),
1511            fixed_key(),
1512            1024,
1513            None,
1514            ScriptedOutcome::Stop,
1515            None,
1516            noop_rt(),
1517        );
1518
1519        let outcome = svc.run(exchange()).await;
1520
1521        assert!(
1522            matches!(outcome, PipelineOutcome::Stopped(_)),
1523            "Stopped from on_miss MUST propagate as Stopped"
1524        );
1525        assert_eq!(
1526            repo.set_call_count(),
1527            0,
1528            "set must NOT be called when on_miss Stops"
1529        );
1530    }
1531
1532    // ── Test 6: on_miss Err propagates without write-back ──
1533
1534    #[tokio::test]
1535    async fn cache_on_miss_err_propagates_without_writeback() {
1536        let repo = Arc::new(MockCacheRepository::new("mock"));
1537        let (mut svc, _invoked) = build_service(
1538            repo.clone(),
1539            fixed_key(),
1540            1024,
1541            None,
1542            ScriptedOutcome::Fail(stub_error("on-miss blew up")),
1543            None,
1544            noop_rt(),
1545        );
1546
1547        let outcome = svc.run(exchange()).await;
1548
1549        match outcome {
1550            PipelineOutcome::Failed(e) => {
1551                assert!(e.to_string().contains("on-miss blew up"), "got: {e}");
1552            }
1553            other => panic!("expected Failed, got {other:?}"),
1554        }
1555        assert_eq!(
1556            repo.set_call_count(),
1557            0,
1558            "set must NOT be called when on_miss fails"
1559        );
1560    }
1561
1562    // ── Test 7: repository get Err propagates ──
1563
1564    #[tokio::test]
1565    async fn cache_repository_get_err_propagates() {
1566        let repo = Arc::new(MockCacheRepository::new("mock"));
1567        repo.set_get_should_fail(true);
1568        let (mut svc, on_miss_invoked) = build_service(
1569            repo,
1570            fixed_key(),
1571            1024,
1572            Some(Body::Bytes(Bytes::from_static(b"x"))),
1573            ScriptedOutcome::Complete,
1574            None,
1575            noop_rt(),
1576        );
1577
1578        let outcome = svc.run(exchange()).await;
1579
1580        match outcome {
1581            PipelineOutcome::Failed(e) => {
1582                assert!(e.to_string().contains("synthetic get failure"), "got: {e}");
1583            }
1584            other => panic!("expected Failed, got {other:?}"),
1585        }
1586        assert!(
1587            !on_miss_invoked.load(Ordering::SeqCst),
1588            "on_miss must NOT run when get fails"
1589        );
1590    }
1591
1592    // ── Test 8: repository set Err propagates ──
1593
1594    #[tokio::test]
1595    async fn cache_repository_set_err_propagates() {
1596        let repo = Arc::new(MockCacheRepository::new("mock"));
1597        repo.set_set_should_fail(true);
1598        let (mut svc, _invoked) = build_service(
1599            repo.clone(),
1600            fixed_key(),
1601            1024,
1602            Some(Body::Bytes(Bytes::from_static(b"x"))),
1603            ScriptedOutcome::Complete,
1604            None,
1605            noop_rt(),
1606        );
1607
1608        let outcome = svc.run(exchange()).await;
1609
1610        match outcome {
1611            PipelineOutcome::Failed(e) => {
1612                assert!(e.to_string().contains("synthetic set failure"), "got: {e}");
1613            }
1614            other => panic!("expected Failed, got {other:?}"),
1615        }
1616        assert_eq!(repo.set_call_count(), 1, "set was attempted (and failed)");
1617    }
1618
1619    // ── Test 9: None key bypasses to on_miss, no set ──
1620
1621    #[tokio::test]
1622    async fn cache_none_key_bypasses_to_on_miss() {
1623        let repo = Arc::new(MockCacheRepository::new("mock"));
1624        let (mut svc, on_miss_invoked) = build_service(
1625            repo.clone(),
1626            none_key(),
1627            1024,
1628            Some(Body::Bytes(Bytes::from_static(b"x"))),
1629            ScriptedOutcome::Complete,
1630            None,
1631            noop_rt(),
1632        );
1633
1634        let outcome = svc.run(exchange()).await;
1635
1636        let ex = match outcome {
1637            PipelineOutcome::Completed(ex) => ex,
1638            other => panic!("expected Completed, got {other:?}"),
1639        };
1640        assert!(
1641            on_miss_invoked.load(Ordering::SeqCst),
1642            "on_miss MUST run when key is None"
1643        );
1644        assert_eq!(ex.input.body, Body::Bytes(Bytes::from_static(b"x")));
1645        assert_eq!(
1646            repo.set_call_count(),
1647            0,
1648            "set must NOT be called when key_expr returns None"
1649        );
1650    }
1651
1652    // ── Extra: HIT reconstruction for each ContentType ──
1653
1654    #[tokio::test]
1655    async fn cache_content_type_reconstruction() {
1656        async fn run_case(entry: CacheEntry, expected: Body) {
1657            let repo = Arc::new(MockCacheRepository::new("mock"));
1658            repo.seed("cache-key", entry).await;
1659            let (mut svc, on_miss_invoked) = build_service(
1660                repo,
1661                fixed_key(),
1662                1024,
1663                Some(Body::Bytes(Bytes::from_static(b"unreached"))),
1664                ScriptedOutcome::Complete,
1665                None,
1666                noop_rt(),
1667            );
1668            let outcome = svc.run(exchange()).await;
1669            let ex = match outcome {
1670                PipelineOutcome::Completed(ex) => ex,
1671                other => panic!("expected Completed, got {other:?}"),
1672            };
1673            assert_eq!(ex.input.body, expected);
1674            assert!(!on_miss_invoked.load(Ordering::SeqCst));
1675        }
1676
1677        run_case(
1678            CacheEntry {
1679                bytes: b"raw".to_vec(),
1680                payload_path: None,
1681                content_type: ContentType::Bytes,
1682                expires_at: None,
1683            },
1684            Body::Bytes(Bytes::from_static(b"raw")),
1685        )
1686        .await;
1687        run_case(
1688            CacheEntry {
1689                bytes: b"hi".to_vec(),
1690                payload_path: None,
1691                content_type: ContentType::Text,
1692                expires_at: None,
1693            },
1694            Body::Text("hi".into()),
1695        )
1696        .await;
1697        run_case(
1698            CacheEntry {
1699                bytes: br#"{"k":1}"#.to_vec(),
1700                payload_path: None,
1701                content_type: ContentType::Json,
1702                expires_at: None,
1703            },
1704            Body::Json(serde_json::json!({"k": 1})),
1705        )
1706        .await;
1707        run_case(
1708            CacheEntry {
1709                bytes: b"<a/>".to_vec(),
1710                payload_path: None,
1711                content_type: ContentType::Xml,
1712                expires_at: None,
1713            },
1714            Body::Xml("<a/>".into()),
1715        )
1716        .await;
1717    }
1718
1719    // ── Extra: Stream body write-back materializes into Body::Bytes ──
1720
1721    #[tokio::test]
1722    async fn cache_miss_stream_body_is_materialized_and_cached() {
1723        let repo = Arc::new(MockCacheRepository::new("mock"));
1724        let (mut svc, _invoked) = build_service(
1725            repo.clone(),
1726            fixed_key(),
1727            1024,
1728            Some(stream_body(b"chunky")),
1729            ScriptedOutcome::Complete,
1730            None,
1731            noop_rt(),
1732        );
1733
1734        let outcome = svc.run(exchange()).await;
1735
1736        let ex = match outcome {
1737            PipelineOutcome::Completed(ex) => ex,
1738            other => panic!("expected Completed, got {other:?}"),
1739        };
1740        // Stream is replaced by materialized Bytes.
1741        assert_eq!(ex.input.body, Body::Bytes(Bytes::from_static(b"chunky")));
1742        assert_eq!(repo.set_call_count(), 1);
1743        let stored = repo.stored_entry("cache-key").await.expect("stored");
1744        assert_eq!(stored.bytes, b"chunky");
1745        assert_eq!(stored.content_type, ContentType::Bytes);
1746    }
1747
1748    // ── CachePeekStaleService tests ──
1749
1750    #[tokio::test]
1751    async fn cache_peek_stale_serves_post_expiry_entry() {
1752        let repo = Arc::new(MockCacheRepository::new("mock"));
1753        repo.seed(
1754            "cache-key",
1755            CacheEntry {
1756                bytes: b"stale-payload".to_vec(),
1757                payload_path: None,
1758                content_type: ContentType::Text,
1759                expires_at: None,
1760            },
1761        )
1762        .await;
1763        let mut svc =
1764            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, noop_rt());
1765
1766        let outcome = svc.run(exchange()).await;
1767
1768        let ex = match outcome {
1769            PipelineOutcome::Completed(ex) => ex,
1770            other => panic!("expected Completed, got {other:?}"),
1771        };
1772        assert_eq!(ex.input.body, Body::Text("stale-payload".into()));
1773    }
1774
1775    #[tokio::test]
1776    #[allow(clippy::await_holding_lock)]
1777    async fn cache_peek_stale_on_absence_stops_branch() {
1778        let _lock = PEEK_STALE_LOG_LOCK
1779            .lock()
1780            .unwrap_or_else(|e| e.into_inner());
1781        let repo = Arc::new(MockCacheRepository::new("mock"));
1782        let mut svc =
1783            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, noop_rt());
1784
1785        let outcome = svc.run(exchange()).await;
1786
1787        assert!(
1788            matches!(outcome, PipelineOutcome::Stopped(_)),
1789            "expected Stopped when no stale entry, got {outcome:?}"
1790        );
1791    }
1792
1793    #[tokio::test]
1794    #[allow(clippy::await_holding_lock)]
1795    async fn cache_peek_stale_none_key_stops() {
1796        let _lock = PEEK_STALE_LOG_LOCK
1797            .lock()
1798            .unwrap_or_else(|e| e.into_inner());
1799        let repo = Arc::new(MockCacheRepository::new("mock"));
1800        let mut svc =
1801            CachePeekStaleService::new(repo, none_key(), PeekStaleMissPolicy::Stop, noop_rt());
1802
1803        let outcome = svc.run(exchange()).await;
1804
1805        assert!(
1806            matches!(outcome, PipelineOutcome::Stopped(_)),
1807            "expected Stopped when key_expr returns None, got {outcome:?}"
1808        );
1809    }
1810
1811    #[tokio::test]
1812    #[allow(clippy::await_holding_lock)]
1813    async fn peek_stale_miss_stop_sets_properties_and_stops() {
1814        let _lock = PEEK_STALE_LOG_LOCK
1815            .lock()
1816            .unwrap_or_else(|e| e.into_inner());
1817        let repo = Arc::new(MockCacheRepository::new("mock"));
1818        let mut svc =
1819            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, noop_rt());
1820
1821        let outcome = svc.run(exchange()).await;
1822
1823        let ex = match outcome {
1824            PipelineOutcome::Stopped(ex) => ex,
1825            other => panic!("expected Stopped, got {other:?}"),
1826        };
1827        assert_eq!(ex.property(CAMEL_CACHE_PEEK_HIT), Some(&Value::Bool(false)));
1828        assert_eq!(
1829            ex.property(CAMEL_CACHE_PEEK_STALE),
1830            Some(&Value::Bool(false))
1831        );
1832    }
1833
1834    #[tokio::test]
1835    async fn peek_stale_miss_continue_completes_with_body_untouched() {
1836        let repo = Arc::new(MockCacheRepository::new("mock"));
1837        let mut svc =
1838            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Continue, noop_rt());
1839
1840        let mut ex = exchange();
1841        ex.input.body = Body::Text("orig".into());
1842
1843        let outcome = svc.run(ex).await;
1844
1845        let ex = match outcome {
1846            PipelineOutcome::Completed(ex) => ex,
1847            other => panic!("expected Completed, got {other:?}"),
1848        };
1849        assert_eq!(ex.input.body, Body::Text("orig".into()));
1850        assert_eq!(ex.property(CAMEL_CACHE_PEEK_HIT), Some(&Value::Bool(false)));
1851        assert_eq!(
1852            ex.property(CAMEL_CACHE_PEEK_STALE),
1853            Some(&Value::Bool(false))
1854        );
1855    }
1856
1857    #[tokio::test]
1858    async fn peek_stale_hit_sets_hit_and_stale_properties() {
1859        let repo = Arc::new(MockCacheRepository::new("mock"));
1860        repo.seed(
1861            "cache-key",
1862            CacheEntry {
1863                bytes: b"stale-payload".to_vec(),
1864                payload_path: None,
1865                content_type: ContentType::Bytes,
1866                expires_at: Some(SystemTime::now() - Duration::from_millis(1)),
1867            },
1868        )
1869        .await;
1870        let mut svc =
1871            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, noop_rt());
1872
1873        let outcome = svc.run(exchange()).await;
1874
1875        let ex = match outcome {
1876            PipelineOutcome::Completed(ex) => ex,
1877            other => panic!("expected Completed, got {other:?}"),
1878        };
1879        assert_eq!(
1880            ex.input.body,
1881            Body::Bytes(Bytes::from_static(b"stale-payload"))
1882        );
1883        assert_eq!(ex.property(CAMEL_CACHE_PEEK_HIT), Some(&Value::Bool(true)));
1884        assert_eq!(
1885            ex.property(CAMEL_CACHE_PEEK_STALE),
1886            Some(&Value::Bool(true))
1887        );
1888    }
1889
1890    #[tokio::test]
1891    async fn peek_stale_hit_fresh_sets_stale_false() {
1892        let repo = Arc::new(MockCacheRepository::new("mock"));
1893        repo.seed(
1894            "cache-key",
1895            CacheEntry {
1896                bytes: b"fresh-payload".to_vec(),
1897                payload_path: None,
1898                content_type: ContentType::Bytes,
1899                expires_at: Some(SystemTime::now() + Duration::from_secs(3600)),
1900            },
1901        )
1902        .await;
1903        let mut svc =
1904            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, noop_rt());
1905
1906        let outcome = svc.run(exchange()).await;
1907
1908        let ex = match outcome {
1909            PipelineOutcome::Completed(ex) => ex,
1910            other => panic!("expected Completed, got {other:?}"),
1911        };
1912        assert_eq!(ex.property(CAMEL_CACHE_PEEK_HIT), Some(&Value::Bool(true)));
1913        assert_eq!(
1914            ex.property(CAMEL_CACHE_PEEK_STALE),
1915            Some(&Value::Bool(false))
1916        );
1917    }
1918
1919    // --- Tracing capture helper for debug-log assertions ---
1920
1921    // ── log-capture harness ──────────────────────────────────────────────────
1922
1923    /// Records dispatched events from this module as `LEVEL field=value…`
1924    /// lines. `on_event` fires only for events actually dispatched to this
1925    /// layer: unlike fmt-writer capture it has no registration-time side
1926    /// effects and does not depend on the process-wide callsite interest
1927    /// cache, which parallel tests rebuild concurrently (bd rc-pna5).
1928    #[derive(Default)]
1929    struct EventRecorder {
1930        records: Arc<Mutex<Vec<String>>>,
1931    }
1932
1933    /// Serializes tests that emit at the shared `cache_peek_stale` debug
1934    /// callsites (the miss-stop arm and the none-key arm).
1935    ///
1936    /// Why: tracing-core caches each callsite's interest process-wide. When a
1937    /// non-recorder thread first-registers one of these callsites while
1938    /// exactly one recorder is installed, `Rebuilder::JustOne` consults the
1939    /// *current thread's* default dispatcher (the no-op one) instead of the
1940    /// registered subscribers, caching `Interest::never()`. The recorder
1941    /// test's later emission at the same callsite is then silently filtered
1942    /// before it reaches the layer, yielding zero captured records. Holding
1943    /// this lock for the whole test body keeps recorder and non-recorder
1944    /// tests from overlapping on the same callsite, so the recorder test
1945    /// always first-registers the callsite with its own subscriber active.
1946    static PEEK_STALE_LOG_LOCK: Mutex<()> = Mutex::new(());
1947
1948    impl EventRecorder {
1949        /// Installs the recorder as this thread's default subscriber and
1950        /// returns the shared record list plus the dispatcher guard.
1951        fn install(self) -> (Arc<Mutex<Vec<String>>>, tracing::subscriber::DefaultGuard) {
1952            use tracing_subscriber::prelude::*;
1953            let records = Arc::clone(&self.records);
1954            let guard = tracing_subscriber::registry().with(self).set_default();
1955            (records, guard)
1956        }
1957    }
1958
1959    /// Formats each visited field as `name=debug-value`, preserving str
1960    /// quoting so `step="cache_peek_stale"`-style assertions keep working.
1961    struct FieldFmt<'a>(&'a mut String);
1962
1963    impl tracing::field::Visit for FieldFmt<'_> {
1964        fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
1965            use std::fmt::Write as _;
1966            let _ = write!(self.0, " {}={:?}", field.name(), value);
1967        }
1968        fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
1969            self.record_debug(field, &value);
1970        }
1971    }
1972
1973    impl<C> tracing_subscriber::Layer<C> for EventRecorder
1974    where
1975        C: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
1976    {
1977        fn on_event(
1978            &self,
1979            event: &tracing::Event<'_>,
1980            _ctx: tracing_subscriber::layer::Context<'_, C>,
1981        ) {
1982            let meta = event.metadata();
1983            if meta.target() != "camel_processor::cache_eip" {
1984                return;
1985            }
1986            let mut line = format!("{} ", meta.level());
1987            event.record(&mut FieldFmt(&mut line));
1988            self.records.lock().unwrap().push(line); // allow-unwrap: test-only
1989        }
1990    }
1991
1992    #[tokio::test]
1993    #[allow(clippy::await_holding_lock)]
1994    async fn peek_stale_miss_stop_emits_debug_log() {
1995        let _lock = PEEK_STALE_LOG_LOCK
1996            .lock()
1997            .unwrap_or_else(|e| e.into_inner());
1998        let repo = Arc::new(MockCacheRepository::new("mock"));
1999        let mut svc =
2000            CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, noop_rt());
2001
2002        let (records, _guard) = EventRecorder::default().install();
2003        let outcome = svc.run(exchange()).await;
2004        drop(_guard);
2005
2006        assert!(matches!(outcome, PipelineOutcome::Stopped(_)));
2007
2008        let captured = records.lock().unwrap().join("\n"); // allow-unwrap: test-only
2009        let miss_records: Vec<&str> = captured
2010            .lines()
2011            .filter(|l| l.contains("peek miss"))
2012            .collect();
2013        assert_eq!(
2014            miss_records.len(),
2015            1,
2016            "expected exactly one DEBUG record containing \"peek miss\"; got: {captured}"
2017        );
2018        assert!(
2019            miss_records[0].contains("DEBUG"),
2020            "expected DEBUG level record; got: {captured}"
2021        );
2022        assert!(
2023            miss_records[0].contains("repository=mock"),
2024            "expected repository field in record; got: {captured}"
2025        );
2026        assert!(
2027            miss_records[0].contains("step=\"cache_peek_stale\""),
2028            "expected step field in record; got: {captured}"
2029        );
2030    }
2031
2032    #[tokio::test]
2033    #[allow(clippy::await_holding_lock)]
2034    async fn peek_stale_key_none_stops_with_debug_log() {
2035        let _lock = PEEK_STALE_LOG_LOCK
2036            .lock()
2037            .unwrap_or_else(|e| e.into_inner());
2038        let repo = Arc::new(MockCacheRepository::new("mock"));
2039        let mut svc =
2040            CachePeekStaleService::new(repo, none_key(), PeekStaleMissPolicy::Stop, noop_rt());
2041
2042        let (records, _guard) = EventRecorder::default().install();
2043        let outcome = svc.run(exchange()).await;
2044        drop(_guard);
2045
2046        assert!(matches!(outcome, PipelineOutcome::Stopped(_)));
2047
2048        let captured = records.lock().unwrap().join("\n"); // allow-unwrap: test-only
2049        let none_records: Vec<&str> = captured
2050            .lines()
2051            .filter(|l| l.contains("resolved to None"))
2052            .collect();
2053        assert_eq!(
2054            none_records.len(),
2055            1,
2056            "expected exactly one DEBUG record containing \"resolved to None\"; got: {captured}"
2057        );
2058        assert!(
2059            none_records[0].contains("DEBUG"),
2060            "expected DEBUG level record; got: {captured}"
2061        );
2062        assert!(
2063            none_records[0].contains("repository=mock"),
2064            "expected repository field in record; got: {captured}"
2065        );
2066        assert!(
2067            none_records[0].contains("step=\"cache_peek_stale\""),
2068            "expected step field in record; got: {captured}"
2069        );
2070    }
2071
2072    // ── CacheInvalidateService tests ──
2073
2074    #[tokio::test]
2075    async fn cache_invalidate_calls_repository_invalidate() {
2076        let repo = Arc::new(MockCacheRepository::new("mock"));
2077        repo.seed(
2078            "cache-key",
2079            CacheEntry {
2080                bytes: b"to-go".to_vec(),
2081                payload_path: None,
2082                content_type: ContentType::Bytes,
2083                expires_at: None,
2084            },
2085        )
2086        .await;
2087        let mut svc = CacheInvalidateService::new(
2088            repo.clone(),
2089            CacheInvalidateTarget::Key(fixed_key()),
2090            noop_rt(),
2091        );
2092
2093        let outcome = svc.run(exchange()).await;
2094
2095        let ex = match outcome {
2096            PipelineOutcome::Completed(ex) => ex,
2097            other => panic!("expected Completed, got {other:?}"),
2098        };
2099        assert_eq!(
2100            ex.property(CAMEL_CACHE_INVALIDATED_COUNT),
2101            Some(&serde_json::Value::from(1u64)),
2102            "exact-key success must set CamelCacheInvalidatedCount = 1"
2103        );
2104        assert_eq!(
2105            repo.invalidate_call_count(),
2106            1,
2107            "invalidate must be called once"
2108        );
2109        assert_eq!(
2110            repo.last_invalidate_key().await,
2111            Some("cache-key".to_string()),
2112            "invalidate must be called with the correct key"
2113        );
2114        assert!(
2115            repo.stored_entry("cache-key").await.is_none(),
2116            "entry must be removed after invalidation"
2117        );
2118    }
2119
2120    #[tokio::test]
2121    async fn cache_invalidate_none_key_completes() {
2122        let repo = Arc::new(MockCacheRepository::new("mock"));
2123        let mut svc = CacheInvalidateService::new(
2124            repo.clone(),
2125            CacheInvalidateTarget::Key(none_key()),
2126            noop_rt(),
2127        );
2128
2129        let outcome = svc.run(exchange()).await;
2130
2131        let _ex = match outcome {
2132            PipelineOutcome::Completed(ex) => ex,
2133            other => panic!("expected Completed, got {other:?}"),
2134        };
2135        assert_eq!(
2136            repo.invalidate_call_count(),
2137            0,
2138            "invalidate must NOT be called when key_expr returns None"
2139        );
2140    }
2141
2142    // ── OTel metrics tests ──
2143
2144    /// Records every `record_counter` call for test assertions.
2145    type CounterRecording = Vec<(String, f64, Vec<(String, String)>)>;
2146
2147    #[derive(Clone)]
2148    struct RecordingMetricsCollector {
2149        counters: Arc<Mutex<CounterRecording>>,
2150    }
2151
2152    impl RecordingMetricsCollector {
2153        fn new() -> Self {
2154            Self {
2155                counters: Arc::new(Mutex::new(Vec::new())),
2156            }
2157        }
2158    }
2159
2160    impl camel_api::metrics::MetricsCollector for RecordingMetricsCollector {
2161        fn record_exchange_duration(&self, _route_id: &str, _duration: Duration) {}
2162        fn increment_errors(&self, _route_id: &str, _error_type: &str) {}
2163        fn increment_exchanges(&self, _route_id: &str) {}
2164        fn set_queue_depth(&self, _route_id: &str, _depth: usize) {}
2165        fn record_circuit_breaker_change(&self, _route_id: &str, _from: &str, _to: &str) {}
2166        fn record_counter(&self, name: &str, value: f64, labels: &[(&str, &str)]) {
2167            self.counters.lock().unwrap().push((
2168                name.to_string(),
2169                value,
2170                labels
2171                    .iter()
2172                    .map(|(k, v)| (k.to_string(), v.to_string()))
2173                    .collect(),
2174            ));
2175        }
2176    }
2177
2178    #[derive(Clone)]
2179    struct TestOtelmRt {
2180        collector: Arc<RecordingMetricsCollector>,
2181    }
2182
2183    impl camel_component_api::health_registry::HealthCheckRegistry for TestOtelmRt {
2184        fn force_unhealthy_for_route(&self, _: &str, _: &str, _: &str) {}
2185    }
2186
2187    impl RuntimeObservability for TestOtelmRt {
2188        fn metrics(&self) -> Arc<dyn camel_api::metrics::MetricsCollector> {
2189            self.collector.clone()
2190        }
2191        fn health(&self) -> Arc<dyn camel_component_api::health_registry::HealthCheckRegistry> {
2192            Arc::new(NoOpHealthCheckRegistry)
2193        }
2194    }
2195
2196    #[tokio::test]
2197    async fn cache_step_hit_increments_otel_counter() {
2198        let repo = Arc::new(MockCacheRepository::new("mock"));
2199        repo.seed(
2200            "cache-key",
2201            CacheEntry {
2202                bytes: b"cached".to_vec(),
2203                payload_path: None,
2204                content_type: ContentType::Bytes,
2205                expires_at: None,
2206            },
2207        )
2208        .await;
2209        let collector = RecordingMetricsCollector::new();
2210        let counters = collector.counters.clone();
2211        let rt = Arc::new(TestOtelmRt {
2212            collector: Arc::new(collector),
2213        });
2214        let (mut svc, _invoked) = build_service(
2215            repo,
2216            fixed_key(),
2217            1024,
2218            None,
2219            ScriptedOutcome::Complete,
2220            None,
2221            rt,
2222        );
2223
2224        let outcome = svc.run(exchange()).await;
2225        assert!(matches!(outcome, PipelineOutcome::Completed(_)));
2226
2227        let recorded = counters.lock().unwrap().clone();
2228        assert!(
2229            recorded.contains(&(
2230                "camel.cache.hits".to_string(),
2231                1.0,
2232                vec![("repository".to_string(), "mock".to_string())]
2233            )),
2234            "expected camel.cache.hits counter, got: {recorded:?}"
2235        );
2236    }
2237
2238    #[tokio::test]
2239    async fn cache_step_miss_increments_otel_counter() {
2240        let repo = Arc::new(MockCacheRepository::new("mock"));
2241        let collector = RecordingMetricsCollector::new();
2242        let counters = collector.counters.clone();
2243        let rt = Arc::new(TestOtelmRt {
2244            collector: Arc::new(collector),
2245        });
2246        let (mut svc, _invoked) = build_service(
2247            repo.clone(),
2248            fixed_key(),
2249            1024,
2250            Some(Body::Bytes(Bytes::from_static(b"x"))),
2251            ScriptedOutcome::Complete,
2252            None,
2253            rt,
2254        );
2255
2256        let outcome = svc.run(exchange()).await;
2257        assert!(matches!(outcome, PipelineOutcome::Completed(_)));
2258
2259        let recorded = counters.lock().unwrap().clone();
2260        assert!(
2261            recorded.contains(&(
2262                "camel.cache.misses".to_string(),
2263                1.0,
2264                vec![("repository".to_string(), "mock".to_string())]
2265            )),
2266            "expected camel.cache.misses counter, got: {recorded:?}"
2267        );
2268    }
2269
2270    // ── CacheClearService tests ──
2271
2272    #[tokio::test]
2273    async fn cache_clear_calls_repository_clear() {
2274        let repo = Arc::new(MockCacheRepository::new("mock"));
2275        repo.seed(
2276            "k",
2277            CacheEntry {
2278                bytes: b"v".to_vec(),
2279                payload_path: None,
2280                content_type: ContentType::Bytes,
2281                expires_at: None,
2282            },
2283        )
2284        .await;
2285        let mut svc = CacheClearService::new(repo.clone());
2286
2287        let outcome = svc.run(exchange()).await;
2288
2289        match outcome {
2290            PipelineOutcome::Completed(_) => {}
2291            other => panic!("expected Completed, got {other:?}"),
2292        }
2293        assert_eq!(repo.clear_call_count(), 1, "clear must be called once");
2294        assert!(
2295            repo.stored_entry("k").await.is_none(),
2296            "entry must be removed after clear"
2297        );
2298    }
2299
2300    #[tokio::test]
2301    async fn cache_clear_err_propagates_failed() {
2302        let repo = Arc::new(MockCacheRepository::new("mock"));
2303        repo.set_should_fail_clear(true);
2304        let mut svc = CacheClearService::new(repo);
2305
2306        let outcome = svc.run(exchange()).await;
2307
2308        match outcome {
2309            PipelineOutcome::Failed(e) => {
2310                assert!(
2311                    e.to_string().contains("synthetic clear failure"),
2312                    "got: {e}"
2313                );
2314            }
2315            other => panic!("expected Failed, got {other:?}"),
2316        }
2317    }
2318
2319    // ── CacheStatsService tests ──
2320
2321    #[tokio::test]
2322    async fn cache_stats_sets_json_body() {
2323        let repo = Arc::new(MockCacheRepository::new("mock"));
2324        repo.set_stats(CacheStats {
2325            hits: 2,
2326            misses: 1,
2327            evictions: 0,
2328            entries: 3,
2329            peek_stale_served: 4,
2330            invalidations: 1,
2331            bytes: None,
2332        });
2333        let mut svc = CacheStatsService::new(repo);
2334
2335        let outcome = svc.run(exchange()).await;
2336
2337        let ex = match outcome {
2338            PipelineOutcome::Completed(ex) => ex,
2339            other => panic!("expected Completed, got {other:?}"),
2340        };
2341        let expected = serde_json::json!({
2342            "repository": "mock",
2343            "hits": 2,
2344            "misses": 1,
2345            "evictions": 0,
2346            "entries": 3,
2347            "peek_stale_served": 4,
2348            "invalidations": 1,
2349            "bytes": null
2350        });
2351        assert_eq!(ex.input.body, Body::Json(expected));
2352
2353        // Exact key-set assertion: the stats JSON snapshot contract is frozen
2354        // to the eight canonical keys (bd rc-22wj) — no extras, none missing.
2355        let Body::Json(v) = &ex.input.body else {
2356            panic!("expected Json body");
2357        };
2358        let keys: std::collections::BTreeSet<&str> = v
2359            .as_object()
2360            .expect("stats body must be a JSON object")
2361            .keys()
2362            .map(String::as_str)
2363            .collect();
2364        let expected_keys: std::collections::BTreeSet<&str> = [
2365            "repository",
2366            "hits",
2367            "misses",
2368            "evictions",
2369            "entries",
2370            "peek_stale_served",
2371            "invalidations",
2372            "bytes",
2373        ]
2374        .into_iter()
2375        .collect();
2376        assert_eq!(
2377            keys, expected_keys,
2378            "stats body must have exactly the eight canonical keys"
2379        );
2380    }
2381
2382    // ── Peek/invalidate OTel counter tests ──
2383
2384    #[tokio::test]
2385    async fn peek_stale_hit_emits_peek_served_counter() {
2386        let repo = Arc::new(MockCacheRepository::new("mock"));
2387        repo.seed(
2388            "cache-key",
2389            CacheEntry {
2390                bytes: b"cached".to_vec(),
2391                payload_path: None,
2392                content_type: ContentType::Bytes,
2393                expires_at: None,
2394            },
2395        )
2396        .await;
2397        let collector = RecordingMetricsCollector::new();
2398        let counters = collector.counters.clone();
2399        let rt = Arc::new(TestOtelmRt {
2400            collector: Arc::new(collector),
2401        });
2402        let mut svc = CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, rt);
2403
2404        let outcome = svc.run(exchange()).await;
2405        assert!(matches!(outcome, PipelineOutcome::Completed(_)));
2406
2407        let recorded = counters.lock().unwrap().clone();
2408        assert!(
2409            recorded.contains(&(
2410                "camel.cache.peek_stale_served".to_string(),
2411                1.0,
2412                vec![("repository".to_string(), "mock".to_string())]
2413            )),
2414            "expected camel.cache.peek_stale_served counter, got: {recorded:?}"
2415        );
2416    }
2417
2418    #[tokio::test]
2419    async fn invalidate_emits_invalidations_counter() {
2420        let repo = Arc::new(MockCacheRepository::new("mock"));
2421        repo.seed(
2422            "cache-key",
2423            CacheEntry {
2424                bytes: b"to-go".to_vec(),
2425                payload_path: None,
2426                content_type: ContentType::Bytes,
2427                expires_at: None,
2428            },
2429        )
2430        .await;
2431        let collector = RecordingMetricsCollector::new();
2432        let counters = collector.counters.clone();
2433        let rt = Arc::new(TestOtelmRt {
2434            collector: Arc::new(collector),
2435        });
2436        let mut svc =
2437            CacheInvalidateService::new(repo, CacheInvalidateTarget::Key(fixed_key()), rt);
2438
2439        let outcome = svc.run(exchange()).await;
2440        assert!(matches!(outcome, PipelineOutcome::Completed(_)));
2441
2442        let recorded = counters.lock().unwrap().clone();
2443        assert!(
2444            recorded.contains(&(
2445                "camel.cache.invalidations".to_string(),
2446                1.0,
2447                vec![("repository".to_string(), "mock".to_string())]
2448            )),
2449            "expected camel.cache.invalidations counter, got: {recorded:?}"
2450        );
2451    }
2452
2453    #[tokio::test]
2454    #[allow(clippy::await_holding_lock)]
2455    async fn peek_stale_miss_emits_no_peek_served_counter() {
2456        // Absent key -> MISS path must NOT emit camel.cache.peek_stale_served.
2457        let _lock = PEEK_STALE_LOG_LOCK
2458            .lock()
2459            .unwrap_or_else(|e| e.into_inner());
2460        let repo = Arc::new(MockCacheRepository::new("mock"));
2461        let collector = RecordingMetricsCollector::new();
2462        let counters = collector.counters.clone();
2463        let rt = Arc::new(TestOtelmRt {
2464            collector: Arc::new(collector),
2465        });
2466        let mut svc = CachePeekStaleService::new(repo, fixed_key(), PeekStaleMissPolicy::Stop, rt);
2467
2468        let outcome = svc.run(exchange()).await;
2469        assert!(matches!(outcome, PipelineOutcome::Stopped(_)));
2470
2471        let recorded = counters.lock().unwrap().clone();
2472        assert!(
2473            !recorded
2474                .iter()
2475                .any(|(name, _, _)| name == "camel.cache.peek_stale_served"),
2476            "expected zero camel.cache.peek_stale_served counters, got: {recorded:?}"
2477        );
2478    }
2479
2480    #[tokio::test]
2481    async fn invalidate_err_emits_no_invalidations_counter() {
2482        // Failing invalidate -> Failed outcome must NOT emit camel.cache.invalidations.
2483        let repo = Arc::new(MockCacheRepository::new("mock"));
2484        repo.seed(
2485            "cache-key",
2486            CacheEntry {
2487                bytes: b"to-go".to_vec(),
2488                payload_path: None,
2489                content_type: ContentType::Bytes,
2490                expires_at: None,
2491            },
2492        )
2493        .await;
2494        repo.set_should_fail_invalidate(true);
2495        let collector = RecordingMetricsCollector::new();
2496        let counters = collector.counters.clone();
2497        let rt = Arc::new(TestOtelmRt {
2498            collector: Arc::new(collector),
2499        });
2500        let mut svc =
2501            CacheInvalidateService::new(repo, CacheInvalidateTarget::Key(fixed_key()), rt);
2502
2503        let outcome = svc.run(exchange()).await;
2504        assert!(matches!(outcome, PipelineOutcome::Failed(_)));
2505
2506        let recorded = counters.lock().unwrap().clone();
2507        assert!(
2508            !recorded
2509                .iter()
2510                .any(|(name, _, _)| name == "camel.cache.invalidations"),
2511            "expected zero camel.cache.invalidations counters, got: {recorded:?}"
2512        );
2513    }
2514
2515    // ── CacheInvalidateService prefix tests ──
2516
2517    #[tokio::test]
2518    async fn cache_invalidate_prefix_removes_namespace_sets_count() {
2519        let repo = Arc::new(MockCacheRepository::new("mock"));
2520        for key in ["ns:one", "ns:two", "other:x"] {
2521            repo.seed(
2522                key,
2523                CacheEntry {
2524                    bytes: key.as_bytes().to_vec(),
2525                    payload_path: None,
2526                    content_type: ContentType::Bytes,
2527                    expires_at: None,
2528                },
2529            )
2530            .await;
2531        }
2532
2533        let collector = RecordingMetricsCollector::new();
2534        let counters = collector.counters.clone();
2535        let rt = Arc::new(TestOtelmRt {
2536            collector: Arc::new(collector),
2537        });
2538        let mut svc = CacheInvalidateService::new(
2539            repo.clone(),
2540            CacheInvalidateTarget::Prefix(prefix_key()),
2541            rt,
2542        );
2543
2544        let outcome = svc.run(exchange()).await;
2545
2546        let ex = match outcome {
2547            PipelineOutcome::Completed(ex) => ex,
2548            other => panic!("expected Completed, got {other:?}"),
2549        };
2550        assert_eq!(
2551            ex.property(CAMEL_CACHE_INVALIDATED_COUNT),
2552            Some(&serde_json::Value::from(2u64)),
2553            "prefix purge must report the removed count"
2554        );
2555        assert!(
2556            repo.stored_entry("ns:one").await.is_none(),
2557            "ns:one must be removed"
2558        );
2559        assert!(
2560            repo.stored_entry("ns:two").await.is_none(),
2561            "ns:two must be removed"
2562        );
2563        assert!(
2564            repo.stored_entry("other:x").await.is_some(),
2565            "other:x must be preserved"
2566        );
2567
2568        let recorded = counters.lock().unwrap().clone();
2569        assert!(
2570            recorded.contains(&(
2571                "camel.cache.invalidations".to_string(),
2572                1.0,
2573                vec![("repository".to_string(), "mock".to_string())]
2574            )),
2575            "expected one camel.cache.invalidations counter, got: {recorded:?}"
2576        );
2577    }
2578
2579    #[tokio::test]
2580    async fn cache_invalidate_prefix_none_expr_completes() {
2581        let repo = Arc::new(MockCacheRepository::new("mock"));
2582        let mut svc = CacheInvalidateService::new(
2583            repo.clone(),
2584            CacheInvalidateTarget::Prefix(none_key()),
2585            noop_rt(),
2586        );
2587
2588        let outcome = svc.run(exchange()).await;
2589
2590        let ex = match outcome {
2591            PipelineOutcome::Completed(ex) => ex,
2592            other => panic!("expected Completed, got {other:?}"),
2593        };
2594        assert_eq!(
2595            repo.invalidate_call_count(),
2596            0,
2597            "no invalidate calls when prefix expr resolves to None"
2598        );
2599        assert!(
2600            ex.property(CAMEL_CACHE_INVALIDATED_COUNT).is_none(),
2601            "no count property when prefix expr resolves to None"
2602        );
2603    }
2604
2605    #[tokio::test]
2606    async fn cache_invalidate_prefix_unsupported_fails_closed() {
2607        let repo = Arc::new(MockCacheRepository::new("mock"));
2608        repo.set_prefix_unsupported(true);
2609        let mut svc = CacheInvalidateService::new(
2610            repo,
2611            CacheInvalidateTarget::Prefix(prefix_key()),
2612            noop_rt(),
2613        );
2614
2615        let outcome = svc.run(exchange()).await;
2616
2617        match outcome {
2618            PipelineOutcome::Failed(e) => {
2619                let msg = format!("{e}");
2620                assert!(
2621                    msg.contains("mock"),
2622                    "error must name the backend, got: {msg}"
2623                );
2624            }
2625            other => panic!("expected Failed, got {other:?}"),
2626        }
2627    }
2628
2629    // ── Task 2.4: singleflight miss coalescing ──
2630
2631    use std::sync::atomic::AtomicUsize;
2632    use tokio::sync::Notify;
2633
2634    /// Terminal behavior of the gated on-miss sub-pipeline.
2635    #[derive(Clone)]
2636    enum GatedOutcome {
2637        Complete(Body),
2638        Fail(CamelError),
2639        Stop,
2640    }
2641
2642    /// Test on-miss sub-pipeline with two Notify gates: signals
2643    /// `leader_entered` as its FIRST action, parks on `release.notified()`,
2644    /// then bumps the invocation counter and returns the scripted outcome.
2645    /// Both gates are optional so the same struct serves ungated tests.
2646    #[derive(Clone)]
2647    struct GatedOnMiss {
2648        leader_entered: Option<Arc<Notify>>,
2649        release: Option<Arc<Notify>>,
2650        invocations: Arc<AtomicUsize>,
2651        outcome: GatedOutcome,
2652    }
2653
2654    impl OutcomePipeline for GatedOnMiss {
2655        fn clone_box(&self) -> Box<dyn OutcomePipeline> {
2656            Box::new(self.clone())
2657        }
2658
2659        fn run<'a>(
2660            &'a mut self,
2661            mut exchange: Exchange,
2662        ) -> Pin<Box<dyn Future<Output = PipelineOutcome> + Send + 'a>> {
2663            let leader_entered = self.leader_entered.clone();
2664            let release = self.release.clone();
2665            let invocations = Arc::clone(&self.invocations);
2666            let outcome = self.outcome.clone();
2667            Box::pin(async move {
2668                // FIRST action: the leader is inside on_miss.
2669                if let Some(entered) = leader_entered.as_ref() {
2670                    entered.notify_one();
2671                }
2672                // Park until the test releases the wave.
2673                if let Some(release) = release.as_ref() {
2674                    release.notified().await;
2675                }
2676                invocations.fetch_add(1, Ordering::SeqCst);
2677                match outcome {
2678                    GatedOutcome::Complete(body) => {
2679                        exchange.input.body = body;
2680                        PipelineOutcome::Completed(exchange)
2681                    }
2682                    GatedOutcome::Fail(e) => PipelineOutcome::Failed(e),
2683                    GatedOutcome::Stop => PipelineOutcome::Stopped(exchange),
2684                }
2685            })
2686        }
2687    }
2688
2689    /// Build a coalescing CacheService around a `GatedOnMiss`.
2690    fn build_gated_service(
2691        repo: Arc<MockCacheRepository>,
2692        outcome: GatedOutcome,
2693        leader_entered: Option<Arc<Notify>>,
2694        release: Option<Arc<Notify>>,
2695    ) -> (CacheService, Arc<AtomicUsize>) {
2696        let invocations = Arc::new(AtomicUsize::new(0));
2697        let on_miss = OutcomeSegment::new(Box::new(GatedOnMiss {
2698            leader_entered,
2699            release,
2700            invocations: Arc::clone(&invocations),
2701            outcome,
2702        }));
2703        let svc = CacheService::new(repo, fixed_key(), None, 1024, on_miss, noop_rt())
2704            .with_coalesce(true);
2705        (svc, invocations)
2706    }
2707
2708    fn exchange_with_body(text: &str) -> Exchange {
2709        Exchange::new(Message::new(text))
2710    }
2711
2712    #[tokio::test]
2713    async fn coalesce_three_concurrent_misses_fetch_once() {
2714        let repo = Arc::new(MockCacheRepository::new("mock"));
2715        let leader_entered = Arc::new(Notify::new());
2716        let release = Arc::new(Notify::new());
2717        let (svc, invocations) = build_gated_service(
2718            repo.clone(),
2719            GatedOutcome::Complete(Body::Text("fetched".into())),
2720            Some(Arc::clone(&leader_entered)),
2721            Some(Arc::clone(&release)),
2722        );
2723
2724        // Register on leader_entered BEFORE spawning the leader so its
2725        // notify_one cannot be missed (enable-before-check).
2726        let entered = leader_entered.notified();
2727        tokio::pin!(entered);
2728        entered.as_mut().enable();
2729
2730        let mut leader_svc = svc.clone();
2731        let leader = tokio::spawn(async move { leader_svc.run(exchange()).await });
2732        entered.await; // deterministic proof: the leader is inside on_miss.
2733
2734        let mut w1_svc = svc.clone();
2735        let mut w1 = tokio::spawn(async move { w1_svc.run(exchange()).await });
2736        let mut w2_svc = svc.clone();
2737        let mut w2 = tokio::spawn(async move { w2_svc.run(exchange()).await });
2738
2739        // Both waiters park (registered on the wave, not running on_miss).
2740        for waiter in [&mut w1, &mut w2] {
2741            if let Ok(done) = tokio::time::timeout(Duration::from_millis(50), waiter).await {
2742                panic!("waiter resolved before release: {done:?}")
2743            }
2744        }
2745
2746        release.notify_waiters();
2747
2748        let leader_ex = match leader.await.expect("leader task join") {
2749            PipelineOutcome::Completed(ex) => ex,
2750            other => panic!("expected leader Completed, got {other:?}"),
2751        };
2752        let w1_ex = match w1.await.expect("waiter 1 task join") {
2753            PipelineOutcome::Completed(ex) => ex,
2754            other => panic!("expected waiter 1 Completed, got {other:?}"),
2755        };
2756        let w2_ex = match w2.await.expect("waiter 2 task join") {
2757            PipelineOutcome::Completed(ex) => ex,
2758            other => panic!("expected waiter 2 Completed, got {other:?}"),
2759        };
2760        assert_eq!(leader_ex.input.body, Body::Text("fetched".into()));
2761        assert_eq!(w1_ex.input.body, Body::Text("fetched".into()));
2762        assert_eq!(w2_ex.input.body, Body::Text("fetched".into()));
2763        assert_eq!(invocations.load(Ordering::SeqCst), 1, "on_miss ran once");
2764        assert_eq!(repo.set_call_count(), 1, "single write-back set");
2765    }
2766
2767    #[tokio::test]
2768    async fn coalesce_leader_failure_fails_waiters_once() {
2769        let repo = Arc::new(MockCacheRepository::new("mock"));
2770        let leader_entered = Arc::new(Notify::new());
2771        let release = Arc::new(Notify::new());
2772        let (svc, invocations) = build_gated_service(
2773            repo.clone(),
2774            GatedOutcome::Fail(stub_error("coalesce-boom")),
2775            Some(Arc::clone(&leader_entered)),
2776            Some(Arc::clone(&release)),
2777        );
2778
2779        let entered = leader_entered.notified();
2780        tokio::pin!(entered);
2781        entered.as_mut().enable();
2782
2783        let mut leader_svc = svc.clone();
2784        let leader = tokio::spawn(async move { leader_svc.run(exchange()).await });
2785        entered.await;
2786
2787        let mut w1_svc = svc.clone();
2788        let mut w1 = tokio::spawn(async move { w1_svc.run(exchange()).await });
2789        let mut w2_svc = svc.clone();
2790        let mut w2 = tokio::spawn(async move { w2_svc.run(exchange()).await });
2791
2792        for waiter in [&mut w1, &mut w2] {
2793            if let Ok(done) = tokio::time::timeout(Duration::from_millis(50), waiter).await {
2794                panic!("waiter resolved before release: {done:?}")
2795            }
2796        }
2797
2798        release.notify_waiters();
2799
2800        let leader_err = match leader.await.expect("leader task join") {
2801            PipelineOutcome::Failed(e) => e,
2802            other => panic!("expected leader Failed, got {other:?}"),
2803        };
2804        let w1_err = match w1.await.expect("waiter 1 task join") {
2805            PipelineOutcome::Failed(e) => e,
2806            other => panic!("expected waiter 1 Failed, got {other:?}"),
2807        };
2808        let w2_err = match w2.await.expect("waiter 2 task join") {
2809            PipelineOutcome::Failed(e) => e,
2810            other => panic!("expected waiter 2 Failed, got {other:?}"),
2811        };
2812        assert_eq!(format!("{leader_err}"), format!("{w1_err}"));
2813        assert_eq!(format!("{leader_err}"), format!("{w2_err}"));
2814        assert_eq!(invocations.load(Ordering::SeqCst), 1, "on_miss ran once");
2815        assert_eq!(repo.set_call_count(), 0, "no write-back on failure");
2816    }
2817
2818    #[tokio::test]
2819    async fn coalesce_leader_stopped_stops_waiters() {
2820        let repo = Arc::new(MockCacheRepository::new("mock"));
2821        let leader_entered = Arc::new(Notify::new());
2822        let release = Arc::new(Notify::new());
2823        let (svc, invocations) = build_gated_service(
2824            repo.clone(),
2825            GatedOutcome::Stop,
2826            Some(Arc::clone(&leader_entered)),
2827            Some(Arc::clone(&release)),
2828        );
2829
2830        let entered = leader_entered.notified();
2831        tokio::pin!(entered);
2832        entered.as_mut().enable();
2833
2834        let mut leader_svc = svc.clone();
2835        let leader =
2836            tokio::spawn(async move { leader_svc.run(exchange_with_body("leader-orig")).await });
2837        entered.await;
2838
2839        let mut w1_svc = svc.clone();
2840        let mut w1 =
2841            tokio::spawn(async move { w1_svc.run(exchange_with_body("waiter-orig")).await });
2842
2843        if let Ok(done) = tokio::time::timeout(Duration::from_millis(50), &mut w1).await {
2844            panic!("waiter resolved before release: {done:?}")
2845        }
2846
2847        release.notify_waiters();
2848
2849        let leader_ex = match leader.await.expect("leader task join") {
2850            PipelineOutcome::Stopped(ex) => ex,
2851            other => panic!("expected leader Stopped, got {other:?}"),
2852        };
2853        let waiter_ex = match w1.await.expect("waiter task join") {
2854            PipelineOutcome::Stopped(ex) => ex,
2855            other => panic!("expected waiter Stopped, got {other:?}"),
2856        };
2857        assert_eq!(
2858            leader_ex.input.body,
2859            Body::Text("leader-orig".into()),
2860            "leader stopped with its own exchange"
2861        );
2862        assert_eq!(
2863            waiter_ex.input.body,
2864            Body::Text("waiter-orig".into()),
2865            "waiter stopped with its own exchange, body untouched"
2866        );
2867        assert_eq!(invocations.load(Ordering::SeqCst), 1, "on_miss ran once");
2868        assert_eq!(repo.set_call_count(), 0, "no write-back on stop");
2869    }
2870
2871    #[tokio::test]
2872    async fn coalesce_leader_dropped_does_not_strand_waiters() {
2873        let repo = Arc::new(MockCacheRepository::new("mock"));
2874        let leader_entered = Arc::new(Notify::new());
2875        let release = Arc::new(Notify::new());
2876        let (svc, _invocations) = build_gated_service(
2877            repo,
2878            GatedOutcome::Complete(Body::Text("fetched".into())),
2879            Some(Arc::clone(&leader_entered)),
2880            Some(Arc::clone(&release)),
2881        );
2882
2883        let entered = leader_entered.notified();
2884        tokio::pin!(entered);
2885        entered.as_mut().enable();
2886
2887        let mut leader_svc = svc.clone();
2888        let leader = tokio::spawn(async move { leader_svc.run(exchange()).await });
2889        entered.await;
2890
2891        let mut w1_svc = svc.clone();
2892        let mut w1 = tokio::spawn(async move { w1_svc.run(exchange()).await });
2893
2894        if let Ok(done) = tokio::time::timeout(Duration::from_millis(50), &mut w1).await {
2895            panic!("waiter resolved before leader abort: {done:?}")
2896        }
2897
2898        // Drop the leader future mid-flight: the cancellation guard must
2899        // publish a Failed terminal and retire the wave's map entry.
2900        leader.abort();
2901
2902        let joined = tokio::time::timeout(Duration::from_secs(1), w1)
2903            .await
2904            .expect("waiter completes within 1s after leader drop")
2905            .expect("waiter task join");
2906        match joined {
2907            PipelineOutcome::Failed(e) => {
2908                let msg = format!("{e}");
2909                assert!(
2910                    msg.contains("cancelled"),
2911                    "expected cancellation terminal, got: {msg}"
2912                );
2913            }
2914            other => panic!("expected waiter Failed, got {other:?}"),
2915        }
2916        assert!(
2917            svc.inflight.lock().unwrap().is_empty(),
2918            "in-flight map must not retain the aborted wave's entry"
2919        );
2920    }
2921
2922    #[tokio::test]
2923    async fn no_coalesce_runs_per_exchange() {
2924        let repo = Arc::new(MockCacheRepository::new("mock"));
2925        let invocations = Arc::new(AtomicUsize::new(0));
2926        let on_miss = OutcomeSegment::new(Box::new(GatedOnMiss {
2927            leader_entered: None,
2928            release: None,
2929            invocations: Arc::clone(&invocations),
2930            outcome: GatedOutcome::Complete(Body::Text("per-exchange".into())),
2931        }));
2932        // Default construction (no with_coalesce): per-exchange execution.
2933        let svc = CacheService::new(repo.clone(), fixed_key(), None, 1024, on_miss, noop_rt());
2934
2935        let mut a = svc.clone();
2936        let mut b = svc.clone();
2937        let mut c = svc.clone();
2938        let (ra, rb, rc) = tokio::join!(a.run(exchange()), b.run(exchange()), c.run(exchange()));
2939
2940        for outcome in [ra, rb, rc] {
2941            match outcome {
2942                PipelineOutcome::Completed(ex) => {
2943                    assert_eq!(ex.input.body, Body::Text("per-exchange".into()));
2944                }
2945                other => panic!("expected Completed, got {other:?}"),
2946            }
2947        }
2948        assert_eq!(
2949            invocations.load(Ordering::SeqCst),
2950            3,
2951            "on_miss ran per exchange"
2952        );
2953        assert_eq!(repo.set_call_count(), 3, "set called per exchange");
2954    }
2955}