1use std::any::Any;
33use std::collections::HashMap;
34use std::fmt;
35use std::sync::{Arc, Mutex, RwLock};
36use std::time::Duration as StdDuration;
37
38use async_trait::async_trait;
39use bock_ai::{
40 AiError, AiProvider, Decision, DecisionType, ManifestScope, ManifestWriter, SelectContext,
41 SelectOption, SelectRequest,
42};
43use bock_types::Strictness;
44use chrono::Utc;
45use sha2::{Digest, Sha256};
46
47pub trait ErrorValue: Send + Sync + fmt::Debug {
67 fn type_name(&self) -> &str;
71
72 fn display(&self) -> String;
74
75 fn structural_props(&self) -> Vec<(&'static str, String)> {
84 Vec::new()
85 }
86
87 fn as_any(&self) -> Option<&dyn Any> {
91 None
92 }
93}
94
95#[derive(Debug, Clone)]
98pub struct SimpleError {
99 type_name: String,
100 message: String,
101 props: Vec<(&'static str, String)>,
102}
103
104impl SimpleError {
105 #[must_use]
108 pub fn new(
109 type_name: impl Into<String>,
110 message: impl Into<String>,
111 props: Vec<(&'static str, String)>,
112 ) -> Self {
113 Self {
114 type_name: type_name.into(),
115 message: message.into(),
116 props,
117 }
118 }
119}
120
121impl ErrorValue for SimpleError {
122 fn type_name(&self) -> &str {
123 &self.type_name
124 }
125
126 fn display(&self) -> String {
127 format!("{}: {}", self.type_name, self.message)
128 }
129
130 fn structural_props(&self) -> Vec<(&'static str, String)> {
131 self.props.clone()
132 }
133}
134
135#[derive(Debug, Clone, Default)]
142pub struct Annotations {
143 pub context: Vec<String>,
145 pub performance: Vec<String>,
147 pub domain: Vec<String>,
149 pub security: Vec<String>,
151}
152
153impl Annotations {
154 #[must_use]
157 pub fn to_strings(&self) -> Vec<String> {
158 let mut out = Vec::new();
159 for c in &self.context {
160 out.push(format!("@context({c})"));
161 }
162 for p in &self.performance {
163 out.push(format!("@performance({p})"));
164 }
165 for d in &self.domain {
166 out.push(format!("@domain({d})"));
167 }
168 for s in &self.security {
169 out.push(format!("@security({s})"));
170 }
171 out
172 }
173}
174
175#[derive(Debug, Clone)]
178pub struct ErrorOccurrence {
179 pub error: Arc<dyn ErrorValue>,
181 pub timestamp: chrono::DateTime<Utc>,
183 pub attempt: u32,
185}
186
187#[derive(Debug, Clone)]
196pub struct RecoveryContext {
197 pub error: Arc<dyn ErrorValue>,
199 pub operation: String,
201 pub annotations: Annotations,
203 pub elapsed: StdDuration,
205 pub attempt: u32,
207 pub history: Vec<ErrorOccurrence>,
209}
210
211impl RecoveryContext {
212 pub const HISTORY_CAP: usize = 10;
214
215 #[must_use]
217 pub fn first_attempt(
218 error: Arc<dyn ErrorValue>,
219 operation: impl Into<String>,
220 annotations: Annotations,
221 ) -> Self {
222 Self {
223 error,
224 operation: operation.into(),
225 annotations,
226 elapsed: StdDuration::ZERO,
227 attempt: 1,
228 history: Vec::new(),
229 }
230 }
231
232 pub fn push_history(&mut self, occurrence: ErrorOccurrence) {
234 if self.history.len() == Self::HISTORY_CAP {
235 self.history.remove(0);
236 }
237 self.history.push(occurrence);
238 }
239}
240
241#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
250pub struct Cancelled;
251
252#[derive(Debug)]
258pub enum StrategyOutcome<T, E> {
259 Ok(T),
261 Err(E),
263 Cancelled,
266}
267
268impl<T, E> StrategyOutcome<T, E> {
269 #[must_use]
271 pub fn is_cancelled(&self) -> bool {
272 matches!(self, Self::Cancelled)
273 }
274}
275
276#[derive(Debug, Default, Clone)]
285pub struct CancelCheckpoint {
286 flag: Arc<std::sync::atomic::AtomicBool>,
287}
288
289impl CancelCheckpoint {
290 #[must_use]
292 pub fn new() -> Self {
293 Self::default()
294 }
295
296 pub fn cancel(&self) {
299 self.flag.store(true, std::sync::atomic::Ordering::SeqCst);
300 }
301
302 #[must_use]
304 pub fn is_cancelled(&self) -> bool {
305 self.flag.load(std::sync::atomic::Ordering::SeqCst)
306 }
307}
308
309pub type RecoveryOperation<T, E> =
315 Arc<dyn Fn() -> futures::future::BoxFuture<'static, StrategyOutcome<T, E>> + Send + Sync>;
316
317pub type CacheLookup<T> = Arc<dyn Fn() -> Option<T> + Send + Sync>;
319
320#[async_trait::async_trait]
332pub trait RecoveryStrategy<T, E>: Send + Sync
333where
334 T: Send + 'static,
335 E: Send + 'static,
336{
337 fn name(&self) -> String;
339
340 fn description(&self) -> String;
342
343 async fn attempt(
347 &self,
348 error: &E,
349 context: &RecoveryContext,
350 op: RecoveryOperation<T, E>,
351 cancel: &CancelCheckpoint,
352 ) -> StrategyOutcome<T, E>;
353
354 async fn on_cancel(&self, _context: &RecoveryContext) {}
357}
358
359pub type BoxedStrategy<T, E> = Arc<dyn RecoveryStrategy<T, E>>;
362
363#[derive(Debug, Clone)]
367pub enum Backoff {
368 Fixed(StdDuration),
370 Linear(StdDuration),
372 Exponential(StdDuration),
374}
375
376impl Backoff {
377 #[must_use]
379 pub fn delay(&self, attempt: u32) -> StdDuration {
380 match self {
381 Self::Fixed(d) => *d,
382 Self::Linear(d) => d.saturating_mul(attempt),
383 Self::Exponential(d) => {
384 let shift = (attempt.saturating_sub(1)).min(32);
385 d.saturating_mul(1u32 << shift)
386 }
387 }
388 }
389}
390
391pub struct RetryStrategy {
395 max: u32,
396 backoff: Backoff,
397}
398
399#[must_use]
401pub fn retry(max: u32, backoff: Backoff) -> Arc<RetryStrategy> {
402 Arc::new(RetryStrategy { max, backoff })
403}
404
405#[async_trait]
406impl<T, E> RecoveryStrategy<T, E> for RetryStrategy
407where
408 T: Send + 'static,
409 E: Send + 'static,
410{
411 fn name(&self) -> String {
412 "retry".into()
413 }
414
415 fn description(&self) -> String {
416 format!(
417 "Retry the failed operation up to {} times with {:?} backoff",
418 self.max, self.backoff
419 )
420 }
421
422 async fn attempt(
423 &self,
424 _error: &E,
425 _context: &RecoveryContext,
426 op: RecoveryOperation<T, E>,
427 cancel: &CancelCheckpoint,
428 ) -> StrategyOutcome<T, E> {
429 let mut last_err: Option<E> = None;
430 for attempt in 1..=self.max {
431 if cancel.is_cancelled() {
432 return StrategyOutcome::Cancelled;
433 }
434 let delay = self.backoff.delay(attempt);
435 if !delay.is_zero() {
436 tokio::time::sleep(delay).await;
437 if cancel.is_cancelled() {
438 return StrategyOutcome::Cancelled;
439 }
440 }
441 match (op)().await {
442 StrategyOutcome::Ok(t) => return StrategyOutcome::Ok(t),
443 StrategyOutcome::Err(e) => last_err = Some(e),
444 StrategyOutcome::Cancelled => return StrategyOutcome::Cancelled,
445 }
446 }
447 match last_err {
448 Some(e) => StrategyOutcome::Err(e),
449 None => unreachable!("retry(max=0) configured; use escalate() for no-op recovery"),
459 }
460 }
461
462 async fn on_cancel(&self, _context: &RecoveryContext) {
463 }
466}
467
468pub struct UseCachedStrategy<T> {
475 ttl: StdDuration,
476 lookup: CacheLookup<T>,
477}
478
479#[must_use]
483pub fn use_cached<T>(ttl: StdDuration, lookup: CacheLookup<T>) -> Arc<UseCachedStrategy<T>>
484where
485 T: Send + Sync + 'static,
486{
487 Arc::new(UseCachedStrategy { ttl, lookup })
488}
489
490#[async_trait]
491impl<T, E> RecoveryStrategy<T, E> for UseCachedStrategy<T>
492where
493 T: Send + Sync + Clone + 'static,
494 E: Send + 'static,
495{
496 fn name(&self) -> String {
497 "use_cached".into()
498 }
499
500 fn description(&self) -> String {
501 format!("Return a cached result within {:?} TTL", self.ttl)
502 }
503
504 async fn attempt(
505 &self,
506 _error: &E,
507 _context: &RecoveryContext,
508 _op: RecoveryOperation<T, E>,
509 _cancel: &CancelCheckpoint,
510 ) -> StrategyOutcome<T, E> {
511 match (self.lookup)() {
512 Some(cached) => StrategyOutcome::Ok(cached),
513 None => {
514 (_op)().await
525 }
526 }
527 }
528}
529
530pub struct DegradeStrategy<T> {
533 fallback: T,
534 label: String,
535}
536
537#[must_use]
540pub fn degrade<T>(fallback: T) -> Arc<DegradeStrategy<T>>
541where
542 T: Clone + Send + Sync + 'static,
543{
544 Arc::new(DegradeStrategy {
545 fallback,
546 label: std::any::type_name::<T>().into(),
547 })
548}
549
550#[async_trait]
551impl<T, E> RecoveryStrategy<T, E> for DegradeStrategy<T>
552where
553 T: Clone + Send + Sync + 'static,
554 E: Send + 'static,
555{
556 fn name(&self) -> String {
557 "degrade".into()
558 }
559
560 fn description(&self) -> String {
561 format!("Return a fallback {} immediately", self.label)
562 }
563
564 async fn attempt(
565 &self,
566 _error: &E,
567 _context: &RecoveryContext,
568 _op: RecoveryOperation<T, E>,
569 _cancel: &CancelCheckpoint,
570 ) -> StrategyOutcome<T, E> {
571 StrategyOutcome::Ok(self.fallback.clone())
572 }
573}
574
575pub struct CircuitBreakerStrategy<T> {
581 threshold: u32,
582 reset_after: StdDuration,
583 open_fallback: Arc<dyn Fn() -> T + Send + Sync>,
584 state: Mutex<BreakerState>,
585}
586
587#[derive(Debug, Clone, Copy)]
588enum BreakerState {
589 Closed { consecutive_failures: u32 },
590 Open { opened_at: std::time::Instant },
591}
592
593#[must_use]
595pub fn circuit_break<T, F>(
596 threshold: u32,
597 reset_after: StdDuration,
598 open_fallback: F,
599) -> Arc<CircuitBreakerStrategy<T>>
600where
601 T: Send + Sync + 'static,
602 F: Fn() -> T + Send + Sync + 'static,
603{
604 Arc::new(CircuitBreakerStrategy {
605 threshold,
606 reset_after,
607 open_fallback: Arc::new(open_fallback),
608 state: Mutex::new(BreakerState::Closed {
609 consecutive_failures: 0,
610 }),
611 })
612}
613
614#[async_trait]
615impl<T, E> RecoveryStrategy<T, E> for CircuitBreakerStrategy<T>
616where
617 T: Send + Sync + 'static,
618 E: Send + 'static,
619{
620 fn name(&self) -> String {
621 "circuit_break".into()
622 }
623
624 fn description(&self) -> String {
625 format!(
626 "Trip after {} consecutive failures, reset after {:?}",
627 self.threshold, self.reset_after
628 )
629 }
630
631 async fn attempt(
632 &self,
633 _error: &E,
634 _context: &RecoveryContext,
635 op: RecoveryOperation<T, E>,
636 cancel: &CancelCheckpoint,
637 ) -> StrategyOutcome<T, E> {
638 if cancel.is_cancelled() {
640 return StrategyOutcome::Cancelled;
641 }
642 let now = std::time::Instant::now();
643 let is_open = {
644 let mut state = self.state.lock().expect("breaker state poisoned");
645 match *state {
646 BreakerState::Open { opened_at }
647 if now.duration_since(opened_at) < self.reset_after =>
648 {
649 true
650 }
651 BreakerState::Open { .. } => {
652 *state = BreakerState::Closed {
653 consecutive_failures: 0,
654 };
655 false
656 }
657 BreakerState::Closed { .. } => false,
658 }
659 };
660 if is_open {
661 return StrategyOutcome::Ok((self.open_fallback)());
662 }
663 if cancel.is_cancelled() {
664 return StrategyOutcome::Cancelled;
665 }
666 let outcome = (op)().await;
667 match outcome {
668 StrategyOutcome::Ok(t) => {
669 let mut state = self.state.lock().expect("breaker state poisoned");
670 *state = BreakerState::Closed {
671 consecutive_failures: 0,
672 };
673 StrategyOutcome::Ok(t)
674 }
675 StrategyOutcome::Err(e) => {
676 let mut state = self.state.lock().expect("breaker state poisoned");
677 let next = match *state {
678 BreakerState::Closed {
679 consecutive_failures,
680 } => consecutive_failures + 1,
681 BreakerState::Open { .. } => 1,
682 };
683 if next >= self.threshold {
684 *state = BreakerState::Open { opened_at: now };
685 } else {
686 *state = BreakerState::Closed {
687 consecutive_failures: next,
688 };
689 }
690 StrategyOutcome::Err(e)
691 }
692 StrategyOutcome::Cancelled => StrategyOutcome::Cancelled,
693 }
694 }
695
696 async fn on_cancel(&self, _context: &RecoveryContext) {
697 let mut state = self.state.lock().expect("breaker state poisoned");
700 if matches!(*state, BreakerState::Closed { .. }) {
701 *state = BreakerState::Closed {
702 consecutive_failures: 0,
703 };
704 }
705 }
706}
707
708pub struct EscalateStrategy;
710
711#[must_use]
713pub fn escalate() -> Arc<EscalateStrategy> {
714 Arc::new(EscalateStrategy)
715}
716
717#[async_trait]
718impl<T, E> RecoveryStrategy<T, E> for EscalateStrategy
719where
720 T: Send + 'static,
721 E: Send + 'static,
722{
723 fn name(&self) -> String {
724 "escalate".into()
725 }
726
727 fn description(&self) -> String {
728 "Propagate the error without recovery".into()
729 }
730
731 async fn attempt(
732 &self,
733 _error: &E,
734 _context: &RecoveryContext,
735 op: RecoveryOperation<T, E>,
736 _cancel: &CancelCheckpoint,
737 ) -> StrategyOutcome<T, E> {
738 (op)().await
742 }
743}
744
745#[derive(Debug, Clone, PartialEq, Eq, Hash)]
756pub struct AdaptivePinKey {
757 pub error_signature: String,
759 pub operation: String,
761}
762
763impl AdaptivePinKey {
764 #[must_use]
766 pub fn from_error_and_op(error: &dyn ErrorValue, operation: &str) -> Self {
767 let hash = sha256_short(&error.structural_props());
768 Self {
769 error_signature: format!("{}:{}", error.type_name(), hash),
770 operation: operation.to_string(),
771 }
772 }
773
774 #[must_use]
777 pub fn decision_id(&self) -> String {
778 let mut hasher = Sha256::new();
779 hasher.update(self.error_signature.as_bytes());
780 hasher.update(b"|");
781 hasher.update(self.operation.as_bytes());
782 let digest = hasher.finalize();
783 hex::encode_short(&digest[..8])
784 }
785}
786
787fn sha256_short(props: &[(&'static str, String)]) -> String {
788 let mut sorted = props.to_vec();
789 sorted.sort_by(|a, b| a.0.cmp(b.0));
790 let mut hasher = Sha256::new();
791 for (k, v) in sorted {
792 hasher.update(k.as_bytes());
793 hasher.update(b"=");
794 hasher.update(v.as_bytes());
795 hasher.update(b";");
796 }
797 hex::encode_short(&hasher.finalize()[..6])
798}
799
800mod hex {
801 pub(super) fn encode_short(bytes: &[u8]) -> String {
802 let mut s = String::with_capacity(bytes.len() * 2);
803 for b in bytes {
804 s.push(nibble(b >> 4));
805 s.push(nibble(b & 0x0f));
806 }
807 s
808 }
809
810 fn nibble(n: u8) -> char {
811 match n {
812 0..=9 => (b'0' + n) as char,
813 10..=15 => (b'a' + (n - 10)) as char,
814 _ => unreachable!(),
815 }
816 }
817}
818
819pub type PinTable = Arc<RwLock<HashMap<AdaptivePinKey, String>>>;
824
825pub struct AdaptiveHandler<T, E> {
831 strategies: Vec<BoxedStrategy<T, E>>,
832 provider: Option<Arc<dyn AiProvider>>,
833 context_aware: bool,
834 strictness: Strictness,
835 module_path: std::path::PathBuf,
836 pins: PinTable,
838 manifest: Option<Arc<Mutex<ManifestWriter>>>,
840}
841
842pub struct AdaptiveHandlerBuilder<T, E> {
845 strategies: Vec<BoxedStrategy<T, E>>,
846 provider: Option<Arc<dyn AiProvider>>,
847 context_aware: bool,
848 strictness: Strictness,
849 module_path: std::path::PathBuf,
850 pins: PinTable,
851 manifest: Option<Arc<Mutex<ManifestWriter>>>,
852}
853
854impl<T, E> AdaptiveHandlerBuilder<T, E>
855where
856 T: Send + Sync + 'static,
857 E: Send + 'static,
858{
859 #[must_use]
862 pub fn context_aware(mut self, enabled: bool) -> Self {
863 self.context_aware = enabled;
864 self
865 }
866
867 #[must_use]
869 pub fn with_provider(mut self, provider: Arc<dyn AiProvider>) -> Self {
870 self.provider = Some(provider);
871 self
872 }
873
874 #[must_use]
876 pub fn strictness(mut self, strictness: Strictness) -> Self {
877 self.strictness = strictness;
878 self
879 }
880
881 #[must_use]
883 pub fn module(mut self, module_path: impl Into<std::path::PathBuf>) -> Self {
884 self.module_path = module_path.into();
885 self
886 }
887
888 #[must_use]
891 pub fn with_pins(mut self, pins: PinTable) -> Self {
892 self.pins = pins;
893 self
894 }
895
896 #[must_use]
900 pub fn with_manifest(mut self, manifest: Arc<Mutex<ManifestWriter>>) -> Self {
901 self.manifest = Some(manifest);
902 self
903 }
904
905 #[must_use]
907 pub fn build(self) -> AdaptiveHandler<T, E> {
908 AdaptiveHandler {
909 strategies: self.strategies,
910 provider: self.provider,
911 context_aware: self.context_aware,
912 strictness: self.strictness,
913 module_path: self.module_path,
914 pins: self.pins,
915 manifest: self.manifest,
916 }
917 }
918}
919
920#[must_use]
924pub fn adaptive<T, E>(strategies: Vec<BoxedStrategy<T, E>>) -> AdaptiveHandlerBuilder<T, E> {
925 AdaptiveHandlerBuilder {
926 strategies,
927 provider: None,
928 context_aware: true,
929 strictness: Strictness::Development,
930 module_path: std::path::PathBuf::from("unknown.bock"),
931 pins: Arc::new(RwLock::new(HashMap::new())),
932 manifest: None,
933 }
934}
935
936#[derive(Debug)]
940pub struct RecoveryResult<T, E> {
941 pub outcome: StrategyOutcome<T, E>,
943 pub selection: SelectionRecord,
945}
946
947#[derive(Debug, Clone)]
950pub struct SelectionRecord {
951 pub selected: String,
953 pub source: SelectionSource,
955 pub confidence: f64,
958 pub reasoning: Option<String>,
960}
961
962#[derive(Debug, Clone, PartialEq, Eq)]
964pub enum SelectionSource {
965 Pinned,
967 Provider,
969 FirstStrategy,
971}
972
973#[derive(Debug, thiserror::Error)]
976pub enum AdaptiveError {
977 #[error(
981 "adaptive handler: unpinned pattern in production — \
982 error_signature={signature}, operation={operation}"
983 )]
984 UnpinnedInProduction {
985 signature: String,
987 operation: String,
989 },
990 #[error("adaptive handler: provider error: {0}")]
992 Provider(#[from] AiError),
993 #[error("adaptive handler: empty strategy list")]
995 EmptyStrategies,
996 #[error("adaptive handler: pinned strategy '{0}' not in configured set")]
999 UnknownPinnedStrategy(String),
1000}
1001
1002impl<T, E> AdaptiveHandler<T, E>
1003where
1004 T: Send + Sync + 'static,
1005 E: Send + 'static,
1006{
1007 pub async fn recover(
1016 &self,
1017 error: E,
1018 operation: &str,
1019 context: RecoveryContext,
1020 op: RecoveryOperation<T, E>,
1021 cancel: &CancelCheckpoint,
1022 ) -> Result<RecoveryResult<T, E>, AdaptiveError>
1023 where
1024 E: 'static,
1025 {
1026 if self.strategies.is_empty() {
1027 return Err(AdaptiveError::EmptyStrategies);
1028 }
1029
1030 let pin_key = AdaptivePinKey::from_error_and_op(&*context.error, operation);
1031
1032 if self.strictness == Strictness::Production {
1034 let pinned = {
1035 let pins = self.pins.read().expect("pin table poisoned");
1036 pins.get(&pin_key).cloned()
1037 };
1038 match pinned {
1039 Some(name) => {
1040 let strat = self.strategy_by_name(&name)?;
1041 let outcome = strat.attempt(&error, &context, op, cancel).await;
1042 let selection = SelectionRecord {
1043 selected: name.clone(),
1044 source: SelectionSource::Pinned,
1045 confidence: 1.0,
1046 reasoning: Some("replay of pinned selection".into()),
1047 };
1048 self.finish(&error, pin_key, outcome, selection, strat, &context)
1049 .await
1050 }
1051 None => Err(AdaptiveError::UnpinnedInProduction {
1052 signature: pin_key.error_signature,
1053 operation: pin_key.operation,
1054 }),
1055 }
1056 } else {
1057 let selection = match (self.provider.as_ref(), self.context_aware) {
1059 (Some(provider), true) => {
1060 let options = self
1061 .strategies
1062 .iter()
1063 .map(|s| SelectOption {
1064 id: s.name(),
1065 description: s.description(),
1066 })
1067 .collect::<Vec<_>>();
1068 let req = SelectRequest {
1069 options: options.clone(),
1070 context: select_context_from_recovery(&context, operation),
1071 rationale_prompt: "Select the recovery strategy best suited to this error \
1072 given the operation context and annotations. The closed \
1073 set of options is authoritative — choose exactly one."
1074 .into(),
1075 };
1076 match provider.select(&req).await {
1077 Ok(resp) => SelectionRecord {
1078 selected: resp.selected_id.clone(),
1079 source: SelectionSource::Provider,
1080 confidence: resp.confidence,
1081 reasoning: resp.reasoning,
1082 },
1083 Err(_) => self.first_strategy_selection(),
1084 }
1085 }
1086 _ => self.first_strategy_selection(),
1087 };
1088
1089 let strat = self.strategy_by_name(&selection.selected)?;
1090 let outcome = strat.attempt(&error, &context, op, cancel).await;
1091 self.finish(&error, pin_key, outcome, selection, strat, &context)
1092 .await
1093 }
1094 }
1095
1096 fn first_strategy_selection(&self) -> SelectionRecord {
1097 let first = &self.strategies[0];
1098 SelectionRecord {
1099 selected: first.name(),
1100 source: SelectionSource::FirstStrategy,
1101 confidence: 1.0,
1102 reasoning: Some("fallback: first strategy (AI unavailable)".into()),
1103 }
1104 }
1105
1106 fn strategy_by_name(&self, name: &str) -> Result<BoxedStrategy<T, E>, AdaptiveError> {
1107 self.strategies
1108 .iter()
1109 .find(|s| s.name() == name)
1110 .cloned()
1111 .ok_or_else(|| AdaptiveError::UnknownPinnedStrategy(name.to_string()))
1112 }
1113
1114 async fn finish(
1115 &self,
1116 _error: &E,
1117 pin_key: AdaptivePinKey,
1118 outcome: StrategyOutcome<T, E>,
1119 selection: SelectionRecord,
1120 strat: BoxedStrategy<T, E>,
1121 context: &RecoveryContext,
1122 ) -> Result<RecoveryResult<T, E>, AdaptiveError> {
1123 if outcome.is_cancelled() {
1125 strat.on_cancel(context).await;
1126 }
1127
1128 if let Some(mgr) = &self.manifest {
1131 if selection.source != SelectionSource::Pinned {
1132 let alternatives: Vec<String> = self
1133 .strategies
1134 .iter()
1135 .map(|s| s.name())
1136 .filter(|n| n != &selection.selected)
1137 .collect();
1138 let decision = Decision {
1139 id: pin_key.decision_id(),
1140 module: self.module_path.clone(),
1141 target: None,
1142 decision_type: DecisionType::AdaptiveRecovery,
1143 choice: selection.selected.clone(),
1144 alternatives,
1145 reasoning: selection.reasoning.clone(),
1146 model_id: self
1147 .provider
1148 .as_ref()
1149 .map(|p| p.model_id())
1150 .unwrap_or_else(|| "none".into()),
1151 confidence: selection.confidence,
1152 pinned: false,
1153 pin_reason: None,
1154 pinned_at: None,
1155 pinned_by: None,
1156 superseded_by: None,
1157 timestamp: Utc::now(),
1158 };
1159 let mut writer = mgr.lock().expect("manifest writer poisoned");
1160 writer.record(decision);
1161 }
1162 }
1163
1164 Ok(RecoveryResult { outcome, selection })
1165 }
1166}
1167
1168fn select_context_from_recovery(ctx: &RecoveryContext, operation: &str) -> SelectContext {
1171 let mut metadata = HashMap::new();
1172 metadata.insert("operation".into(), operation.to_string());
1173 metadata.insert("elapsed_ms".into(), ctx.elapsed.as_millis().to_string());
1174 metadata.insert("attempt".into(), ctx.attempt.to_string());
1175 SelectContext {
1176 error: Some(ctx.error.display()),
1177 annotations: ctx.annotations.to_strings(),
1178 history: ctx
1179 .history
1180 .iter()
1181 .map(|e| format!("{} at attempt {}", e.error.type_name(), e.attempt))
1182 .collect(),
1183 metadata,
1184 }
1185}
1186
1187#[allow(dead_code)]
1191const _ADAPTIVE_RECOVERY_IS_RUNTIME: () = {
1192 };
1195
1196#[must_use]
1199pub fn adaptive_scope() -> ManifestScope {
1200 DecisionType::AdaptiveRecovery.scope()
1201}
1202
1203#[cfg(test)]
1204mod tests {
1205 use super::*;
1206 use bock_ai::{AiProvider, SelectResponse, StubProvider};
1207 use std::sync::atomic::{AtomicU32, Ordering};
1208
1209 fn err(kind: &str, msg: &str, props: Vec<(&'static str, String)>) -> SimpleError {
1210 SimpleError::new(kind, msg, props)
1211 }
1212
1213 fn op_always_fail<T: Clone + Send + 'static>(
1214 _fallback: T,
1215 ) -> RecoveryOperation<T, SimpleError> {
1216 Arc::new(move || {
1217 Box::pin(async move {
1218 StrategyOutcome::<T, SimpleError>::Err(err("Boom", "always fails", Vec::new()))
1219 })
1220 })
1221 }
1222
1223 fn op_fail_then_ok<T>(n_fails: Arc<AtomicU32>, ok: T) -> RecoveryOperation<T, SimpleError>
1224 where
1225 T: Clone + Send + Sync + 'static,
1226 {
1227 Arc::new(move || {
1228 let ok = ok.clone();
1229 let n = n_fails.clone();
1230 Box::pin(async move {
1231 let left = n.fetch_sub(1, Ordering::SeqCst);
1232 if left > 0 {
1233 StrategyOutcome::<T, SimpleError>::Err(err("Transient", "retrying", Vec::new()))
1234 } else {
1235 StrategyOutcome::Ok(ok)
1236 }
1237 })
1238 })
1239 }
1240
1241 #[test]
1242 fn annotations_to_strings_tags_each_category() {
1243 let a = Annotations {
1244 context: vec!["PCI-DSS".into()],
1245 performance: vec!["latency: 200ms".into()],
1246 domain: vec!["payments".into()],
1247 security: vec!["tokenized".into()],
1248 };
1249 let s = a.to_strings();
1250 assert!(s.iter().any(|x| x == "@context(PCI-DSS)"));
1251 assert!(s.iter().any(|x| x == "@performance(latency: 200ms)"));
1252 assert!(s.iter().any(|x| x == "@domain(payments)"));
1253 assert!(s.iter().any(|x| x == "@security(tokenized)"));
1254 }
1255
1256 #[test]
1257 fn history_cap_bounds_to_ten() {
1258 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1259 let mut ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1260 for i in 0..20 {
1261 ctx.push_history(ErrorOccurrence {
1262 error: e.clone(),
1263 timestamp: Utc::now(),
1264 attempt: i + 1,
1265 });
1266 }
1267 assert_eq!(ctx.history.len(), RecoveryContext::HISTORY_CAP);
1268 assert_eq!(ctx.history.last().unwrap().attempt, 20);
1270 assert_eq!(ctx.history.first().unwrap().attempt, 11);
1271 }
1272
1273 #[test]
1274 fn pin_key_same_signature_for_same_structure() {
1275 let a = err(
1276 "ConnectionTimeout",
1277 "after 30s",
1278 vec![("kind", "timeout".into())],
1279 );
1280 let b = err(
1281 "ConnectionTimeout",
1282 "after 45s",
1283 vec![("kind", "timeout".into())],
1284 );
1285 let ka = AdaptivePinKey::from_error_and_op(&a, "Net.fetch");
1286 let kb = AdaptivePinKey::from_error_and_op(&b, "Net.fetch");
1287 assert_eq!(ka, kb);
1288 }
1289
1290 #[test]
1291 fn pin_key_differs_by_type_name() {
1292 let a = err("ConnectionTimeout", "x", Vec::new());
1293 let b = err("ConnectionRefused", "x", Vec::new());
1294 let ka = AdaptivePinKey::from_error_and_op(&a, "Net.fetch");
1295 let kb = AdaptivePinKey::from_error_and_op(&b, "Net.fetch");
1296 assert_ne!(ka, kb);
1297 }
1298
1299 #[test]
1300 fn pin_key_differs_by_operation() {
1301 let e = err("Timeout", "x", Vec::new());
1302 let k1 = AdaptivePinKey::from_error_and_op(&e, "Net.fetch");
1303 let k2 = AdaptivePinKey::from_error_and_op(&e, "Net.post");
1304 assert_ne!(k1, k2);
1305 }
1306
1307 #[test]
1308 fn pin_key_decision_id_is_deterministic() {
1309 let e = err("Timeout", "x", Vec::new());
1310 let k = AdaptivePinKey::from_error_and_op(&e, "Net.fetch");
1311 assert_eq!(k.decision_id(), k.decision_id());
1312 }
1313
1314 #[test]
1315 fn adaptive_scope_is_runtime() {
1316 assert_eq!(adaptive_scope(), ManifestScope::Runtime);
1317 }
1318
1319 #[test]
1320 fn backoff_exponential_doubles() {
1321 let b = Backoff::Exponential(StdDuration::from_millis(100));
1322 assert_eq!(b.delay(1), StdDuration::from_millis(100));
1323 assert_eq!(b.delay(2), StdDuration::from_millis(200));
1324 assert_eq!(b.delay(3), StdDuration::from_millis(400));
1325 }
1326
1327 #[tokio::test]
1328 async fn adaptive_fallback_to_first_strategy_when_no_provider() {
1329 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1330 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1331 let handler = adaptive::<i32, SimpleError>(vec![degrade(42), escalate()])
1332 .context_aware(false)
1333 .build();
1334 let op = op_always_fail::<i32>(0);
1335 let cancel = CancelCheckpoint::new();
1336 let res = handler
1337 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1338 .await
1339 .expect("ok");
1340 assert_eq!(res.selection.selected, "degrade");
1341 assert_eq!(res.selection.source, SelectionSource::FirstStrategy);
1342 assert!(matches!(res.outcome, StrategyOutcome::Ok(42)));
1343 }
1344
1345 #[tokio::test]
1346 async fn adaptive_uses_provider_select_in_development() {
1347 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1348 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1349 let provider: Arc<dyn AiProvider> = Arc::new(StubProvider::default());
1350 let handler = adaptive::<i32, SimpleError>(vec![escalate(), degrade(7)])
1351 .with_provider(provider)
1352 .build();
1353 let op = op_always_fail::<i32>(0);
1355 let cancel = CancelCheckpoint::new();
1356 let res = handler
1357 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1358 .await
1359 .expect("ok");
1360 assert_eq!(res.selection.selected, "escalate");
1361 assert_eq!(res.selection.source, SelectionSource::Provider);
1362 }
1363
1364 #[tokio::test]
1365 async fn adaptive_production_unpinned_errors() {
1366 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1367 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1368 let handler = adaptive::<i32, SimpleError>(vec![degrade(1)])
1369 .strictness(Strictness::Production)
1370 .build();
1371 let op = op_always_fail::<i32>(0);
1372 let cancel = CancelCheckpoint::new();
1373 let err = handler
1374 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1375 .await
1376 .expect_err("should require pin");
1377 assert!(matches!(err, AdaptiveError::UnpinnedInProduction { .. }));
1378 }
1379
1380 #[tokio::test]
1381 async fn adaptive_production_pinned_replays_strategy() {
1382 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1383 let key = AdaptivePinKey::from_error_and_op(&*e, "op");
1384 let pins = Arc::new(RwLock::new(HashMap::from([(key, "degrade".to_string())])));
1385 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1386
1387 let handler = adaptive::<i32, SimpleError>(vec![escalate(), degrade(99)])
1388 .strictness(Strictness::Production)
1389 .with_pins(pins)
1390 .build();
1391 let op = op_always_fail::<i32>(0);
1392 let cancel = CancelCheckpoint::new();
1393 let res = handler
1394 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1395 .await
1396 .expect("ok");
1397 assert_eq!(res.selection.selected, "degrade");
1398 assert_eq!(res.selection.source, SelectionSource::Pinned);
1399 assert!(matches!(res.outcome, StrategyOutcome::Ok(99)));
1400 }
1401
1402 #[tokio::test]
1403 async fn adaptive_cancellation_propagates_and_fires_on_cancel() {
1404 struct CancelStrat {
1405 on_cancel_fired: Arc<AtomicU32>,
1406 }
1407 #[async_trait]
1408 impl RecoveryStrategy<i32, SimpleError> for CancelStrat {
1409 fn name(&self) -> String {
1410 "cancel_strat".into()
1411 }
1412 fn description(&self) -> String {
1413 "always returns Cancelled".into()
1414 }
1415 async fn attempt(
1416 &self,
1417 _e: &SimpleError,
1418 _c: &RecoveryContext,
1419 _op: RecoveryOperation<i32, SimpleError>,
1420 _cancel: &CancelCheckpoint,
1421 ) -> StrategyOutcome<i32, SimpleError> {
1422 StrategyOutcome::Cancelled
1423 }
1424 async fn on_cancel(&self, _c: &RecoveryContext) {
1425 self.on_cancel_fired.fetch_add(1, Ordering::SeqCst);
1426 }
1427 }
1428 let fired = Arc::new(AtomicU32::new(0));
1429 let strat: BoxedStrategy<i32, SimpleError> = Arc::new(CancelStrat {
1430 on_cancel_fired: fired.clone(),
1431 });
1432 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1433 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1434 let handler = adaptive::<i32, SimpleError>(vec![strat])
1435 .context_aware(false)
1436 .build();
1437 let op = op_always_fail::<i32>(0);
1438 let cancel = CancelCheckpoint::new();
1439 let res = handler
1440 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1441 .await
1442 .expect("ok");
1443 assert!(res.outcome.is_cancelled());
1444 assert_eq!(fired.load(Ordering::SeqCst), 1);
1445 }
1446
1447 #[tokio::test]
1448 async fn adaptive_records_to_manifest_when_configured() {
1449 use tempfile::tempdir;
1450 let tmp = tempdir().unwrap();
1451 let manifest = Arc::new(Mutex::new(ManifestWriter::new(tmp.path())));
1452 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1453 let ctx = RecoveryContext::first_attempt(e.clone(), "Net.fetch", Annotations::default());
1454 let handler = adaptive::<i32, SimpleError>(vec![degrade(42)])
1455 .context_aware(false)
1456 .module("src/main.bock")
1457 .with_manifest(manifest.clone())
1458 .build();
1459 let op = op_always_fail::<i32>(0);
1460 let cancel = CancelCheckpoint::new();
1461 handler
1462 .recover(err("X", "x", Vec::new()), "Net.fetch", ctx, op, &cancel)
1463 .await
1464 .expect("ok");
1465 let entries = manifest.lock().unwrap().read_runtime().unwrap();
1466 assert_eq!(entries.len(), 1);
1467 assert_eq!(entries[0].choice, "degrade");
1468 assert_eq!(entries[0].decision_type, DecisionType::AdaptiveRecovery);
1469 }
1470
1471 #[tokio::test]
1472 async fn retry_eventually_succeeds() {
1473 let left = Arc::new(AtomicU32::new(2));
1474 let op = op_fail_then_ok(left.clone(), 100);
1475 let strat: BoxedStrategy<i32, SimpleError> = retry(3, Backoff::Fixed(StdDuration::ZERO));
1476 let e = Arc::new(err("T", "t", Vec::new())) as Arc<dyn ErrorValue>;
1477 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1478 let cancel = CancelCheckpoint::new();
1479 let out = strat
1480 .attempt(&err("T", "t", Vec::new()), &ctx, op, &cancel)
1481 .await;
1482 match out {
1483 StrategyOutcome::Ok(v) => assert_eq!(v, 100),
1484 other => panic!("expected Ok, got {other:?}"),
1485 }
1486 }
1487
1488 #[tokio::test]
1489 async fn retry_observes_cancel() {
1490 let strat: BoxedStrategy<i32, SimpleError> = retry(5, Backoff::Fixed(StdDuration::ZERO));
1491 let op = op_always_fail::<i32>(0);
1492 let e = Arc::new(err("T", "t", Vec::new())) as Arc<dyn ErrorValue>;
1493 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1494 let cancel = CancelCheckpoint::new();
1495 cancel.cancel();
1496 let out = strat
1497 .attempt(&err("T", "t", Vec::new()), &ctx, op, &cancel)
1498 .await;
1499 assert!(matches!(out, StrategyOutcome::Cancelled));
1500 }
1501
1502 #[tokio::test]
1503 async fn use_cached_returns_cached_value() {
1504 let lookup: CacheLookup<i32> = Arc::new(|| Some(777));
1505 let strat: BoxedStrategy<i32, SimpleError> = use_cached(StdDuration::from_secs(60), lookup);
1506 let op = op_always_fail::<i32>(0);
1507 let e = Arc::new(err("T", "t", Vec::new())) as Arc<dyn ErrorValue>;
1508 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1509 let cancel = CancelCheckpoint::new();
1510 let out = strat
1511 .attempt(&err("T", "t", Vec::new()), &ctx, op, &cancel)
1512 .await;
1513 match out {
1514 StrategyOutcome::Ok(v) => assert_eq!(v, 777),
1515 other => panic!("expected Ok, got {other:?}"),
1516 }
1517 }
1518
1519 #[tokio::test]
1520 async fn degrade_returns_fallback() {
1521 let strat: BoxedStrategy<i32, SimpleError> = degrade(55);
1522 let op = op_always_fail::<i32>(0);
1523 let e = Arc::new(err("T", "t", Vec::new())) as Arc<dyn ErrorValue>;
1524 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1525 let cancel = CancelCheckpoint::new();
1526 let out = strat
1527 .attempt(&err("T", "t", Vec::new()), &ctx, op, &cancel)
1528 .await;
1529 match out {
1530 StrategyOutcome::Ok(v) => assert_eq!(v, 55),
1531 other => panic!("expected Ok, got {other:?}"),
1532 }
1533 }
1534
1535 #[tokio::test]
1536 async fn circuit_break_opens_after_threshold() {
1537 let strat: BoxedStrategy<i32, SimpleError> =
1538 circuit_break(2, StdDuration::from_secs(60), || 0);
1539 let op = op_always_fail::<i32>(0);
1540 let e = Arc::new(err("T", "t", Vec::new())) as Arc<dyn ErrorValue>;
1541 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1542 let cancel = CancelCheckpoint::new();
1543 let o1 = strat
1545 .attempt(&err("T", "t", Vec::new()), &ctx, op.clone(), &cancel)
1546 .await;
1547 assert!(matches!(o1, StrategyOutcome::Err(_)));
1548 let o2 = strat
1549 .attempt(&err("T", "t", Vec::new()), &ctx, op.clone(), &cancel)
1550 .await;
1551 assert!(matches!(o2, StrategyOutcome::Err(_)));
1552 let o3 = strat
1554 .attempt(&err("T", "t", Vec::new()), &ctx, op, &cancel)
1555 .await;
1556 match o3 {
1557 StrategyOutcome::Ok(v) => assert_eq!(v, 0),
1558 other => panic!("expected Ok fallback, got {other:?}"),
1559 }
1560 }
1561
1562 #[tokio::test]
1563 async fn escalate_forwards_error() {
1564 let strat: BoxedStrategy<i32, SimpleError> = escalate();
1565 let op = op_always_fail::<i32>(0);
1566 let e = Arc::new(err("T", "t", Vec::new())) as Arc<dyn ErrorValue>;
1567 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1568 let cancel = CancelCheckpoint::new();
1569 let out = strat
1570 .attempt(&err("T", "t", Vec::new()), &ctx, op, &cancel)
1571 .await;
1572 assert!(matches!(out, StrategyOutcome::Err(_)));
1573 }
1574
1575 struct FixedChoiceProvider {
1578 choice: String,
1579 }
1580
1581 #[async_trait]
1582 impl AiProvider for FixedChoiceProvider {
1583 async fn generate(
1584 &self,
1585 _r: &bock_ai::GenerateRequest,
1586 ) -> Result<bock_ai::GenerateResponse, AiError> {
1587 unreachable!()
1588 }
1589 async fn repair(
1590 &self,
1591 _r: &bock_ai::RepairRequest,
1592 ) -> Result<bock_ai::RepairResponse, AiError> {
1593 unreachable!()
1594 }
1595 async fn optimize(
1596 &self,
1597 _r: &bock_ai::OptimizeRequest,
1598 ) -> Result<bock_ai::OptimizeResponse, AiError> {
1599 unreachable!()
1600 }
1601 async fn select(&self, _request: &SelectRequest) -> Result<SelectResponse, AiError> {
1602 Ok(SelectResponse {
1603 selected_id: self.choice.clone(),
1604 confidence: 0.9,
1605 reasoning: Some("fixed choice for test".into()),
1606 })
1607 }
1608 fn model_id(&self) -> String {
1609 "test:fixed".into()
1610 }
1611 }
1612
1613 #[tokio::test]
1614 async fn provider_driven_selection_uses_non_first_option() {
1615 let provider: Arc<dyn AiProvider> = Arc::new(FixedChoiceProvider {
1616 choice: "degrade".into(),
1617 });
1618 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1619 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1620 let handler = adaptive::<i32, SimpleError>(vec![escalate(), degrade(123)])
1621 .with_provider(provider)
1622 .build();
1623 let op = op_always_fail::<i32>(0);
1624 let cancel = CancelCheckpoint::new();
1625 let res = handler
1626 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1627 .await
1628 .expect("ok");
1629 assert_eq!(res.selection.selected, "degrade");
1630 match res.outcome {
1631 StrategyOutcome::Ok(v) => assert_eq!(v, 123),
1632 other => panic!("expected Ok(123), got {other:?}"),
1633 }
1634 }
1635
1636 struct FailingProvider;
1638 #[async_trait]
1639 impl AiProvider for FailingProvider {
1640 async fn generate(
1641 &self,
1642 _r: &bock_ai::GenerateRequest,
1643 ) -> Result<bock_ai::GenerateResponse, AiError> {
1644 unreachable!()
1645 }
1646 async fn repair(
1647 &self,
1648 _r: &bock_ai::RepairRequest,
1649 ) -> Result<bock_ai::RepairResponse, AiError> {
1650 unreachable!()
1651 }
1652 async fn optimize(
1653 &self,
1654 _r: &bock_ai::OptimizeRequest,
1655 ) -> Result<bock_ai::OptimizeResponse, AiError> {
1656 unreachable!()
1657 }
1658 async fn select(&self, _r: &SelectRequest) -> Result<SelectResponse, AiError> {
1659 Err(AiError::Unavailable("test: offline".into()))
1660 }
1661 fn model_id(&self) -> String {
1662 "test:failing".into()
1663 }
1664 }
1665
1666 #[tokio::test]
1667 async fn provider_failure_falls_back_to_first() {
1668 let provider: Arc<dyn AiProvider> = Arc::new(FailingProvider);
1669 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1670 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1671 let handler = adaptive::<i32, SimpleError>(vec![degrade(9), escalate()])
1672 .with_provider(provider)
1673 .build();
1674 let op = op_always_fail::<i32>(0);
1675 let cancel = CancelCheckpoint::new();
1676 let res = handler
1677 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1678 .await
1679 .expect("ok");
1680 assert_eq!(res.selection.selected, "degrade");
1681 assert_eq!(res.selection.source, SelectionSource::FirstStrategy);
1682 }
1683
1684 #[test]
1685 fn assert_ai_error_variants_are_stable() {
1686 let _e = AiError::Unavailable("sanity".into());
1690 }
1691
1692 #[tokio::test]
1693 async fn cancelled_before_on_cancel_called() {
1694 struct CountCancel {
1698 fired: Arc<AtomicU32>,
1699 }
1700 #[async_trait]
1701 impl RecoveryStrategy<i32, SimpleError> for CountCancel {
1702 fn name(&self) -> String {
1703 "count_cancel".into()
1704 }
1705 fn description(&self) -> String {
1706 "degrade to 0".into()
1707 }
1708 async fn attempt(
1709 &self,
1710 _e: &SimpleError,
1711 _c: &RecoveryContext,
1712 _op: RecoveryOperation<i32, SimpleError>,
1713 _cancel: &CancelCheckpoint,
1714 ) -> StrategyOutcome<i32, SimpleError> {
1715 StrategyOutcome::Ok(0)
1716 }
1717 async fn on_cancel(&self, _c: &RecoveryContext) {
1718 self.fired.fetch_add(1, Ordering::SeqCst);
1719 }
1720 }
1721 let fired = Arc::new(AtomicU32::new(0));
1722 let strat: BoxedStrategy<i32, SimpleError> = Arc::new(CountCancel {
1723 fired: fired.clone(),
1724 });
1725 let e = Arc::new(err("X", "x", Vec::new())) as Arc<dyn ErrorValue>;
1726 let ctx = RecoveryContext::first_attempt(e.clone(), "op", Annotations::default());
1727 let handler = adaptive::<i32, SimpleError>(vec![strat])
1728 .context_aware(false)
1729 .build();
1730 let op = op_always_fail::<i32>(0);
1731 let cancel = CancelCheckpoint::new();
1732 let _ = handler
1733 .recover(err("X", "x", Vec::new()), "op", ctx, op, &cancel)
1734 .await
1735 .expect("ok");
1736 assert_eq!(fired.load(Ordering::SeqCst), 0);
1737 }
1738}