1use std::future::Future;
11use std::result::Result as StdResult;
12use std::time::Duration;
13
14use crate::error::{ErrorCategory, VtCodeError};
15use crate::retry_after::retry_after_from_llm_metadata;
16use crate::tools::registry::ToolExecutionError;
17use crate::tools::tool_intent::is_command_tool;
18use crate::tools::unified_error::UnifiedToolError;
19use vtcode_commons::llm::LLMError;
20use vtcode_commons::misconfiguration::detect_misconfiguration;
21
22pub use vtcode_commons::retry::{RetryDecision, RetryPolicy};
23
24pub trait RetryPolicyCoreExt {
29 fn decision_for_vtcode_error(
30 &self,
31 error: &VtCodeError,
32 attempt_index: u32,
33 tool_name: Option<&str>,
34 ) -> RetryDecision;
35
36 fn decision_for_anyhow(&self, error: &anyhow::Error, attempt_index: u32, tool_name: Option<&str>) -> RetryDecision;
37
38 fn decision_for_llm_error(&self, error: &LLMError, attempt_index: u32) -> RetryDecision;
39
40 fn decision_for_tool_error(&self, error: &UnifiedToolError, attempt_index: u32) -> RetryDecision;
41
42 fn decision_for_tool_execution_error(&self, error: &ToolExecutionError, attempt_index: u32) -> RetryDecision;
43
44 fn step_for_vtcode_error(&self, error: VtCodeError, attempt_index: u32, tool_name: Option<&str>) -> RetryStep;
51
52 fn apply_to_tool_execution_error(
53 &self,
54 error: ToolExecutionError,
55 attempt_index: u32,
56 tool_name: Option<&str>,
57 ) -> ToolExecutionError;
58}
59
60impl RetryPolicyCoreExt for RetryPolicy {
61 fn decision_for_vtcode_error(
62 &self,
63 error: &VtCodeError,
64 attempt_index: u32,
65 tool_name: Option<&str>,
66 ) -> RetryDecision {
67 if error.is_misconfiguration() {
69 return misconfiguration_decision(error.category, error.retry_after());
70 }
71 decision_for_category_with_tool(self, error.category, attempt_index, error.retry_after(), tool_name)
72 }
73
74 fn decision_for_anyhow(&self, error: &anyhow::Error, attempt_index: u32, tool_name: Option<&str>) -> RetryDecision {
75 if let Some(vtcode_error) = error.downcast_ref::<VtCodeError>() {
76 return self.decision_for_vtcode_error(vtcode_error, attempt_index, tool_name);
77 }
78 if let Some(llm_error) = error.downcast_ref::<LLMError>() {
79 return self.decision_for_llm_error(llm_error, attempt_index);
80 }
81 if let Some(tool_error) = error.downcast_ref::<UnifiedToolError>() {
82 let category = tool_error.category();
83 if detect_misconfiguration(category, &unified_tool_text(tool_error)).is_some() {
84 return misconfiguration_decision(category, None);
85 }
86 let effective_tool = tool_name.or_else(|| {
88 tool_error
89 .debug_context
90 .as_ref()
91 .map(|ctx| ctx.tool_name.as_str())
92 .filter(|name| !name.is_empty())
93 });
94 return decision_for_category_with_tool(self, category, attempt_index, None, effective_tool);
95 }
96
97 let category = vtcode_commons::classify_anyhow_error(error);
98 let message = format!("{error:#}");
99 if detect_misconfiguration(category, &message).is_some() {
100 return misconfiguration_decision(category, None);
101 }
102 decision_for_category_with_tool(self, category, attempt_index, None, tool_name)
103 }
104
105 fn decision_for_llm_error(&self, error: &LLMError, attempt_index: u32) -> RetryDecision {
106 let retry_after = error.metadata().and_then(retry_after_from_llm_metadata);
107 let category = ErrorCategory::from(error);
108 if vtcode_commons::detect_misconfiguration_in_llm_error(error).is_some() {
109 return misconfiguration_decision(category, retry_after);
110 }
111 decision_for_category_with_tool(self, category, attempt_index, retry_after, None)
112 }
113
114 fn decision_for_tool_error(&self, error: &UnifiedToolError, attempt_index: u32) -> RetryDecision {
115 let category = error.category();
116 if detect_misconfiguration(category, &unified_tool_text(error)).is_some() {
117 return misconfiguration_decision(category, None);
118 }
119 let tool_name = error
120 .debug_context
121 .as_ref()
122 .map(|ctx| ctx.tool_name.as_str())
123 .filter(|tool_name| !tool_name.is_empty());
124 decision_for_category_with_tool(self, category, attempt_index, None, tool_name)
125 }
126
127 fn decision_for_tool_execution_error(&self, error: &ToolExecutionError, attempt_index: u32) -> RetryDecision {
128 if error.patch_failure.is_some() {
129 return RetryDecision {
130 category: error.category,
131 retryable: false,
132 delay: None,
133 retry_after: None,
134 };
135 }
136 if detect_misconfiguration(error.category, &tool_execution_text(error)).is_some() {
137 return misconfiguration_decision(error.category, error.retry_after());
138 }
139 decision_for_category_with_tool(
140 self,
141 error.category,
142 attempt_index,
143 error.retry_after(),
144 Some(error.tool_name.as_str()),
145 )
146 }
147
148 fn step_for_vtcode_error(&self, error: VtCodeError, attempt_index: u32, tool_name: Option<&str>) -> RetryStep {
149 let error = error.with_misconfiguration_guidance();
152 let decision = self.decision_for_vtcode_error(&error, attempt_index, tool_name);
153 if decision.retryable {
154 let delay = decision.delay.unwrap_or_else(|| self.delay_for_attempt(attempt_index));
155 RetryStep::Backoff { delay, decision, error }
156 } else {
157 RetryStep::GiveUp { decision, error }
158 }
159 }
160
161 fn apply_to_tool_execution_error(
162 &self,
163 error: ToolExecutionError,
164 attempt_index: u32,
165 tool_name: Option<&str>,
166 ) -> ToolExecutionError {
167 if error.patch_failure.is_some() {
168 return error;
169 }
170 if let Some(guidance) = detect_misconfiguration(error.category, &tool_execution_text(&error)) {
171 let decision = misconfiguration_decision(error.category, error.retry_after());
172 let mut guided = error.with_retry_decision(decision);
173 let mut suggestions = vec![std::borrow::Cow::Owned(guidance.user_message())];
175 suggestions.append(&mut guided.recovery_suggestions);
176 guided.recovery_suggestions = suggestions;
177 guided.retryable = false;
178 guided.is_recoverable = false;
179 guided.circuit_breaker_impact = false;
180 return guided;
181 }
182 let decision = decision_for_category_with_tool(
183 self,
184 error.category,
185 attempt_index,
186 error.retry_after(),
187 tool_name.or(Some(error.tool_name.as_str())),
188 );
189 error.with_retry_decision(decision)
190 }
191}
192
193fn decision_for_category_with_tool(
194 policy: &RetryPolicy,
195 category: ErrorCategory,
196 attempt_index: u32,
197 retry_after: Option<Duration>,
198 tool_name: Option<&str>,
199) -> RetryDecision {
200 if is_non_retryable_command_timeout(category, tool_name) {
201 return RetryDecision {
202 category,
203 retryable: false,
204 delay: None,
205 retry_after,
206 };
207 }
208
209 policy.decision_for_category(category, attempt_index, retry_after)
210}
211
212pub(crate) fn category_was_retryable(error: &VtCodeError) -> bool {
213 error.category.is_retryable() && !error.is_misconfiguration()
214}
215
216fn misconfiguration_decision(category: ErrorCategory, retry_after: Option<Duration>) -> RetryDecision {
219 RetryDecision {
220 category,
221 retryable: false,
222 delay: None,
223 retry_after,
224 }
225}
226
227pub(crate) fn is_non_retryable_command_timeout(category: ErrorCategory, tool_name: Option<&str>) -> bool {
231 matches!(category, ErrorCategory::Timeout) && tool_name.is_some_and(is_command_tool)
232}
233
234fn unified_tool_text(error: &UnifiedToolError) -> String {
235 let mut text = error.user_message.clone();
236 if let Some(source) = error.source.as_ref() {
237 text.push('\n');
238 text.push_str(&format!("{source:#}"));
239 }
240 text
241}
242
243fn tool_execution_text(error: &ToolExecutionError) -> String {
244 let mut text = error.message.clone();
245 if let Some(original) = error.original_error.as_deref() {
246 text.push('\n');
247 text.push_str(original);
248 }
249 text
250}
251
252#[derive(Debug)]
257pub enum RetryStep {
258 Backoff {
260 delay: Duration,
261 decision: RetryDecision,
262 error: VtCodeError,
263 },
264 GiveUp {
266 decision: RetryDecision,
267 error: VtCodeError,
268 },
269}
270
271#[derive(Debug)]
280pub enum RetryEvent<'a> {
281 AttemptStart { attempt: u32, max_attempts: u32 },
283 Success { attempt: u32 },
285 GiveUp {
287 attempt: u32,
288 error: &'a VtCodeError,
289 decision: &'a RetryDecision,
290 category_was_retryable: bool,
291 },
292 Backoff {
295 attempt: u32,
296 error: &'a VtCodeError,
297 decision: &'a RetryDecision,
298 delay: Duration,
299 category_was_retryable: bool,
300 },
301 Exhausted { last_error: Option<&'a VtCodeError> },
304}
305
306#[allow(
326 clippy::too_many_arguments,
327 reason = "Intentional compatibility, platform, or test-only suppression."
328)]
329pub async fn run_with_retry<T, E, S, F, OnEvent, Synthesize>(
330 policy: &RetryPolicy,
331 state: &mut S,
332 mut on_event: OnEvent,
333 mut operation: F,
334 synthesize_exhausted_error: Synthesize,
335) -> crate::error::Result<T>
336where
337 F: for<'a> FnMut(&'a mut S) -> std::pin::Pin<Box<dyn Future<Output = StdResult<T, E>> + Send + 'a>>,
338 E: Into<VtCodeError>,
339 OnEvent: FnMut(&mut S, RetryEvent<'_>),
340 Synthesize: FnOnce(&RetryPolicy) -> VtCodeError,
341{
342 use tokio::time::sleep;
343
344 let mut last_error: Option<VtCodeError> = None;
345 for attempt in 0..policy.max_attempts {
346 on_event(state, RetryEvent::AttemptStart { attempt, max_attempts: policy.max_attempts });
347 match operation(state).await {
348 Ok(value) => {
349 on_event(state, RetryEvent::Success { attempt });
350 return Ok(value);
351 }
352 Err(err) => {
353 let err: VtCodeError = err.into();
354 let category_was_retryable = category_was_retryable(&err);
355 let step = policy.step_for_vtcode_error(err, attempt, None);
356 match step {
357 RetryStep::GiveUp { decision, error } => {
358 on_event(
359 state,
360 RetryEvent::GiveUp {
361 attempt,
362 error: &error,
363 decision: &decision,
364 category_was_retryable,
365 },
366 );
367 return Err(error);
368 }
369 RetryStep::Backoff { delay, decision, error } => {
370 on_event(
371 state,
372 RetryEvent::Backoff {
373 attempt,
374 error: &error,
375 decision: &decision,
376 delay,
377 category_was_retryable,
378 },
379 );
380 last_error = Some(error);
381 sleep(delay).await;
382 }
383 }
384 }
385 }
386 }
387 let final_error = last_error.unwrap_or_else(|| synthesize_exhausted_error(policy));
388 on_event(state, RetryEvent::Exhausted { last_error: Some(&final_error) });
389 Err(final_error)
390}
391
392pub fn decision_for_vtcode_error(
393 error: &VtCodeError,
394 attempt_index: u32,
395 tool_name: Option<&str>,
396 policy_override: Option<&RetryPolicy>,
397) -> RetryDecision {
398 let policy = policy_override.unwrap_or(&RetryPolicy::DEFAULT);
399 policy.decision_for_vtcode_error(error, attempt_index, tool_name)
400}
401
402pub fn decision_for_anyhow_error(
403 error: &anyhow::Error,
404 attempt_index: u32,
405 tool_name: Option<&str>,
406 policy_override: Option<&RetryPolicy>,
407) -> RetryDecision {
408 let policy = policy_override.unwrap_or(&RetryPolicy::DEFAULT);
409 policy.decision_for_anyhow(error, attempt_index, tool_name)
410}
411
412#[cfg(test)]
413mod tests {
414 use super::*;
415 use crate::config::constants::tools;
416 use crate::error::{ErrorCode, VtCodeError};
417 use vtcode_commons::llm::LLMErrorMetadata;
418
419 #[test]
420 fn non_retryable_categories_stop_immediately() {
421 let policy = RetryPolicy::from_retries(2, Duration::from_secs(1), Duration::from_secs(8), 2.0);
422 let err = VtCodeError::security(ErrorCode::PermissionDenied, "blocked by policy");
423
424 let decision = policy.decision_for_vtcode_error(&err, 0, None);
425 assert_eq!(decision.category, ErrorCategory::PolicyViolation);
426 assert!(!decision.retryable);
427 assert!(decision.delay.is_none());
428 }
429
430 #[test]
431 fn retry_after_header_overrides_backoff_delay() {
432 let policy = RetryPolicy::from_retries(3, Duration::from_secs(1), Duration::from_secs(8), 2.0);
433 let err = LLMError::RateLimit {
434 metadata: Some(LLMErrorMetadata::new(
435 "Anthropic",
436 Some(429),
437 Some("rate_limit_error".to_string()),
438 None,
439 None,
440 Some("7".to_string()),
441 Some("too many requests".to_string()),
442 )),
443 };
444
445 let decision = policy.decision_for_llm_error(&err, 0);
446 assert!(decision.retryable);
447 assert_eq!(decision.retry_after, Some(Duration::from_secs(7)));
448 assert_eq!(decision.delay, Some(Duration::from_secs(7)));
449 }
450
451 #[test]
452 fn quota_exhaustion_is_not_retryable() {
453 let policy = RetryPolicy::from_retries(3, Duration::from_secs(1), Duration::from_secs(8), 2.0);
454 let err = LLMError::RateLimit {
455 metadata: Some(LLMErrorMetadata::new(
456 "OpenAI",
457 Some(429),
458 Some("insufficient_quota".to_string()),
459 None,
460 None,
461 None,
462 Some("quota exceeded".to_string()),
463 )),
464 };
465
466 let decision = policy.decision_for_llm_error(&err, 0);
467 assert_eq!(decision.category, ErrorCategory::ResourceExhausted);
468 assert!(!decision.retryable);
469 }
470
471 #[test]
472 fn anyhow_fallback_uses_shared_classifier() {
473 let policy = RetryPolicy::from_retries(1, Duration::from_secs(1), Duration::from_secs(8), 2.0);
474
475 let decision = policy.decision_for_anyhow(&anyhow::anyhow!("HTTP 503 Service Unavailable"), 0, None);
476 assert_eq!(decision.category, ErrorCategory::ServiceUnavailable);
477 assert!(decision.retryable);
478 assert_eq!(decision.delay, Some(Duration::from_secs(1)));
479 }
480
481 #[test]
482 fn anyhow_prefers_typed_llm_errors() {
483 let policy = RetryPolicy::from_retries(3, Duration::from_secs(1), Duration::from_secs(8), 2.0);
484 let err = anyhow::Error::new(LLMError::RateLimit {
485 metadata: Some(LLMErrorMetadata::new(
486 "Anthropic",
487 Some(429),
488 Some("rate_limit_error".to_string()),
489 None,
490 None,
491 Some("9".to_string()),
492 Some("too many requests".to_string()),
493 )),
494 });
495
496 let decision = policy.decision_for_anyhow(&err, 0, None);
497 assert!(decision.retryable);
498 assert_eq!(decision.retry_after, Some(Duration::from_secs(9)));
499 assert_eq!(decision.delay, Some(Duration::from_secs(9)));
500 }
501
502 #[test]
503 fn canonical_exec_aliases_are_command_tools() {
504 for alias in [
505 tools::RUN_PTY_CMD,
506 tools::EXEC_COMMAND,
507 tools::WRITE_STDIN,
508 tools::UNIFIED_EXEC,
509 "shell",
510 "bash",
511 "container.exec",
512 ] {
513 assert!(is_command_tool(alias), "expected {alias} to be a command tool");
514 }
515 }
516
517 #[test]
518 fn typed_tool_timeout_for_command_tools_is_not_retryable() {
519 let policy = RetryPolicy::from_retries(2, Duration::from_secs(1), Duration::from_secs(8), 2.0);
520 let err = UnifiedToolError::new(crate::tools::unified_error::UnifiedErrorKind::Timeout, "timed out")
521 .with_tool_name(tools::RUN_PTY_CMD);
522
523 let decision = policy.decision_for_tool_error(&err, 0);
524 assert_eq!(decision.category, ErrorCategory::Timeout);
525 assert!(!decision.retryable);
526 }
527
528 #[test]
529 fn anyhow_typed_tool_timeout_uses_fallback_tool_name() {
530 let policy = RetryPolicy::from_retries(2, Duration::from_secs(1), Duration::from_secs(8), 2.0);
531 let err = anyhow::Error::new(UnifiedToolError::new(
532 crate::tools::unified_error::UnifiedErrorKind::Timeout,
533 "timed out",
534 ));
535
536 let decision = policy.decision_for_anyhow(&err, 0, Some(tools::RUN_PTY_CMD));
537 assert_eq!(decision.category, ErrorCategory::Timeout);
538 assert!(!decision.retryable);
539 }
540
541 #[test]
542 fn command_timeouts_do_not_retry() {
543 let policy = RetryPolicy::from_retries(2, Duration::from_secs(1), Duration::from_secs(8), 2.0);
544 let err = VtCodeError::new(ErrorCategory::Timeout, ErrorCode::Timeout, "timed out");
545
546 let decision = policy.decision_for_vtcode_error(&err, 0, Some(tools::RUN_PTY_CMD));
547 assert_eq!(decision.category, ErrorCategory::Timeout);
548 assert!(!decision.retryable);
549 }
550
551 #[tokio::test]
552 async fn run_with_retry_returns_first_success() {
553 use std::sync::Arc;
554 use std::sync::atomic::{AtomicU32, Ordering};
555 let policy = RetryPolicy::from_retries(3, Duration::from_millis(0), Duration::from_millis(1), 2.0);
556 let attempts = Arc::new(AtomicU32::new(0));
557 let attempts_for_op = attempts.clone();
558 let result: crate::error::Result<String> = run_with_retry(
559 &policy,
560 &mut (),
561 |_: &mut (), _| {},
562 |_| {
563 let attempts = attempts_for_op.clone();
564 Box::pin(async move {
565 let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
566 if n < 2 {
567 Err(VtCodeError::network(ErrorCode::ConnectionFailed, "transient"))
568 } else {
569 Ok("ok".to_string())
570 }
571 })
572 },
573 |_: &RetryPolicy| VtCodeError::execution(ErrorCode::ToolExecutionFailed, "exhausted"),
574 )
575 .await;
576 assert_eq!(result.unwrap(), "ok");
577 assert_eq!(attempts.load(Ordering::SeqCst), 2);
578 }
579
580 #[tokio::test]
581 async fn run_with_retry_surfaces_give_up_immediately() {
582 use std::sync::Arc;
583 use std::sync::atomic::{AtomicU32, Ordering};
584 let policy = RetryPolicy::from_retries(5, Duration::from_millis(0), Duration::from_millis(1), 2.0);
585 let attempts = Arc::new(AtomicU32::new(0));
586 let attempts_for_op = attempts.clone();
587 let result: crate::error::Result<String> = run_with_retry(
588 &policy,
589 &mut (),
590 |_: &mut (), _| {},
591 |_| {
592 let attempts = attempts_for_op.clone();
593 Box::pin(async move {
594 attempts.fetch_add(1, Ordering::SeqCst);
595 Err::<String, _>(VtCodeError::input(ErrorCode::InvalidArgument, "bad input"))
596 })
597 },
598 |_: &RetryPolicy| VtCodeError::execution(ErrorCode::ToolExecutionFailed, "exhausted"),
599 )
600 .await;
601 assert!(result.is_err());
602 assert_eq!(attempts.load(Ordering::SeqCst), 1, "GiveUp should short-circuit retries");
603 }
604
605 #[test]
606 fn misconfiguration_fails_fast_with_guidance() {
607 let policy = RetryPolicy::from_retries(5, Duration::from_secs(1), Duration::from_secs(8), 2.0);
608 let err = VtCodeError::new(
609 ErrorCategory::Authentication,
610 ErrorCode::AuthenticationFailed,
611 "Authentication failed: invalid api key",
612 );
613
614 let decision = policy.decision_for_vtcode_error(&err, 0, None);
615 assert!(!decision.retryable);
616 assert!(decision.delay.is_none());
617
618 let step = policy.step_for_vtcode_error(err, 0, None);
619 match step {
620 RetryStep::GiveUp { error, .. } => {
621 assert!(error.message.contains("Check settings/config first"));
622 assert!(error.message.contains("before retrying"));
623 }
624 RetryStep::Backoff { .. } => panic!("misconfiguration must not back off"),
625 }
626 }
627
628 #[test]
629 fn misconfiguration_model_error_does_not_retry() {
630 let policy = RetryPolicy::from_retries(5, Duration::from_secs(1), Duration::from_secs(8), 2.0);
631 let err = VtCodeError::input(ErrorCode::InvalidArgument, "unknown model 'gpt-99' in agent.model");
632
633 assert!(err.is_misconfiguration());
634 let decision = policy.decision_for_vtcode_error(&err, 0, None);
635 assert!(!decision.retryable);
636 }
637
638 #[test]
639 fn transient_errors_still_retry_without_guidance() {
640 let policy = RetryPolicy::from_retries(3, Duration::from_secs(1), Duration::from_secs(8), 2.0);
641 let err = VtCodeError::network(ErrorCode::ConnectionFailed, "connection reset by peer");
642
643 assert!(!err.is_misconfiguration());
644 let decision = policy.decision_for_vtcode_error(&err, 0, None);
645 assert!(decision.retryable);
646 }
647
648 #[test]
649 fn misconfiguration_does_not_count_as_retryable_category() {
650 let err = VtCodeError::network(ErrorCode::ConnectionFailed, "invalid provider_overrides configuration");
651
652 assert!(err.is_misconfiguration());
653 assert!(!category_was_retryable(&err));
654 }
655
656 #[test]
657 fn llm_auth_error_is_misconfiguration() {
658 let policy = RetryPolicy::from_retries(3, Duration::from_secs(1), Duration::from_secs(8), 2.0);
659 let err = LLMError::Authentication {
660 message: "invalid api key".to_string(),
661 metadata: None,
662 };
663
664 let decision = policy.decision_for_llm_error(&err, 0);
665 assert!(!decision.retryable);
666 assert_eq!(decision.category, ErrorCategory::Authentication);
667 }
668
669 #[test]
670 fn tool_execution_misconfiguration_prepends_guidance() {
671 use crate::tools::registry::ToolErrorType;
672
673 let policy = RetryPolicy::from_retries(3, Duration::from_secs(1), Duration::from_secs(8), 2.0);
674 let err = ToolExecutionError::new(
675 "test_tool".to_string(),
676 ToolErrorType::ExecutionError,
677 "unknown model 'foo' in agent.model".to_string(),
678 );
679
680 let guided = policy.apply_to_tool_execution_error(err, 0, None);
681 assert!(!guided.retryable);
682 assert!(!guided.is_recoverable);
683 assert!(!guided.circuit_breaker_impact);
684 assert!(
685 guided
686 .recovery_suggestions
687 .first()
688 .is_some_and(|s| s.contains("Check settings/config first"))
689 );
690 }
691}